def do_anova(domain, dep): if 'every' == domain: for dom in FOND_DOMAINS: do_anova(dom) return elif 'all' in domain: prp_data = [] dom_type = [] if 'fond' in domain: dom_type = FOND_DOMAINS elif 'new' in domain: dom_type = NEW_DOMAINS elif 'ipc' in domain: dom_type = IPC06_DOMAINS elif 'interesting' in domain: dom_type = INTERESTING_DOMAINS for dom in dom_type: prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom)) else: prp_data = load_CSV("RESULTS/prp-%s-results.csv" % domain) solved_prp_data = filter(lambda x: x[-2] == 'True', prp_data) anova([prp_data[0]] + solved_prp_data, ['fullstate', 'planlocal', 'usepolicy'], dependent=dep)
def prp_compare_two(domain, type1, type2, name1, name2): if 'every' == domain: for dom in FOND_DOMAINS: prp_compare_two(dom, type1, type2) return elif ('all' in domain) and ('redundant' not in domain): prp_data = [] dom_type = [] if 'fond' in domain: dom_type = FOND_DOMAINS elif 'new' in domain: dom_type = NEW_DOMAINS elif 'ipc' in domain: dom_type = IPC06_DOMAINS elif 'interesting' in domain: dom_type = INTERESTING_DOMAINS for dom in dom_type: prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom)) elif 'all-redundant' == domain: prp_data = [] for i in range(1, 6): prp_data.extend( load_CSV("RESULTS/prp-blocksworld-redundant-%d-results.csv" % i)) else: prp_data = load_CSV("RESULTS/prp-%s-results.csv" % domain) return _prp_compare_two(domain, type1, type2, name1, name2, prp_data)
def checkit(filename): data_kd = load_CSV(filename + "_kd.csv")[1:] data_kt = load_CSV(filename + "_kt.csv")[1:] # Merge the two sets row by row assert len(data_kd) == len(data_kt) data = [] for i in range(0, len(data_kd)): data.append(data_kd[i] + data_kt[i]) for row in data: for i in range(len(row)): if row[i] == '0.000000': row[i] = '0.000001' def plot_data(data, inds, labels, zlabel, fname): data_map = {} for size in range(SIZE[0], SIZE[1] + 10, 10): data_map[size] = {} for dep in range(DEPTH[0], DEPTH[1]+1): data_map[size][dep] = {} for row in data: data_map[int(row[0])][int(row[1])][inds[0]] = float(row[inds[0]]) data_map[int(row[0])][int(row[1])][inds[1]] = float(row[inds[1]]) data_map[int(row[0])][int(row[1])][inds[2]] = float(row[inds[2]]) data_map[int(row[0])][int(row[1])][inds[3]] = float(row[inds[3]]) from mpl_toolkits.mplot3d import axes3d import matplotlib.pyplot as plt import matplotlib import numpy as np X, Y = np.meshgrid(np.arange(SIZE[0], SIZE[1] + 10, 10), np.arange(DEPTH[0], DEPTH[1]+1)) zs0 = np.array([data_map[x][y][inds[0]] for x,y in zip(np.ravel(X), np.ravel(Y))]) zs1 = np.array([data_map[x][y][inds[1]] for x,y in zip(np.ravel(X), np.ravel(Y))]) zs2 = np.array([data_map[x][y][inds[2]] for x,y in zip(np.ravel(X), np.ravel(Y))]) zs3 = np.array([data_map[x][y][inds[3]] for x,y in zip(np.ravel(X), np.ravel(Y))]) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') #if 'Query Time ($log_e(sec)$)' == zlabel or 'Update Time ($log_e(sec)$)' == zlabel: Z0 = np.log(zs0).reshape(X.shape) Z1 = np.log(zs1).reshape(X.shape) Z2 = np.log(zs2).reshape(X.shape) Z3 = np.log(zs3).reshape(X.shape) # else: #ax.set_zticks([]) ''' Z0 = (zs0 / 1).reshape(X.shape) Z1 = (zs1 / 1).reshape(X.shape) Z2 = (zs2 / 1).reshape(X.shape) Z3 = (zs3 / 1).reshape(X.shape) ''' cols = [] for i in range(0, len(labels)): cols.append(COLOURS[labels[i]]) ax.plot_wireframe(X, Y, Z0, color=cols[0]) ax.plot_wireframe(X, Y, Z1, color=cols[1]) ax.plot_wireframe(X, Y, Z2, color=cols[2]) ax.plot_wireframe(X, Y, Z3, color=cols[3]) #cset = ax.contourf(X, Y, Z0, zdir='z', offset=-100, cmap=matplotlib.cm.coolwarm) #cset = ax.contourf(X, Y, Z0, zdir='x', offset=0, cmap=matplotlib.cm.coolwarm) #cset = ax.contourf(X, Y, Z0, zdir='z', offset=0, cmap=matplotlib.cm.coolwarm) #cset = ax.contourf(X, Y, Z0, zdir='y', offset=40, cmap=cm.coolwarm) ax.set_xlabel('# of Agents') ax.set_ylabel('Maximum Depth') ax.set_zlabel(zlabel) scatter1_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[0], marker = 's') scatter2_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[1], marker = 's') scatter3_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[2], marker = 's') scatter4_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[3], marker = 's') ax.legend([scatter1_proxy, scatter2_proxy, scatter3_proxy, scatter4_proxy], labels, numpoints = 1) ax.get_xaxis().set_major_locator(matplotlib.ticker.MaxNLocator(integer=True)) ax.get_yaxis().set_major_locator(matplotlib.ticker.MaxNLocator(integer=True)) plt.show() print "Plotting query time..." plot_data(data, [V_KT_QUERY, V_KD_QUERY, CLOSURE_KD_QUERY, CLOSURE_KT_QUERY], [V_KT, V_KD, CLOSURE_KD, CLOSURE_KT], 'Query Time ($log_e(sec)$)', 'time.eps') print "Plotting size..." plot_data(data, [CLOSURE_KT_SIZE, CLOSURE_KD_SIZE, V_KD_SIZE, V_KT_SIZE], [CLOSURE_KT, CLOSURE_KD, V_KD, V_KT], 'Size (x1000)', 'size.eps') print "Plotting update time..." plot_data(data, [CLOSURE_KT_UPDATE, CLOSURE_KD_UPDATE, V_KT_UPDATE, V_KD_UPDATE], [CLOSURE_KT, CLOSURE_KD, V_KT, V_KD], 'Update Time ($log_e(sec)$)', 'update_time.eps')
def plot_from_file(csvfile, title=None, use_header=True, xlog=False, ylog=False, no_x_tics=False, noxy=False, no_scatter=False, bw=False, mult_plots=False, xlabel="", ylabel="", legendname="Legend", nosymbols=False): from krrt.utils import load_CSV data = load_CSV(csvfile) if use_header: header = data.pop(0) else: header = ['x-axis', 'y-axis', 'Legend'] if mult_plots: num_plots = len(data[0]) - 1 x = [float(item[0]) for item in data] xs = [] ys = [] ps = [] assert use_header for i in range(1, num_plots + 1): y = [float(item[i]) for item in filter(lambda x: x[i] != '', data)] xs.append(x[:len(y)]) ys.append(y) ps.append(header[i]) plot(xs, ys, x_label=header[0], y_label=ylabel, x_log=xlog, y_log=ylog, names=ps, graph_name=title, legend_name=legendname, disable_x_tics=no_x_tics, xyline=not noxy, no_scatter=no_scatter, col=not bw, nosymbols=nosymbols) return if len(data[0]) > 2: data_keys = sorted(list(set([item[2] for item in data]))) xs = [] ys = [] ps = [] for key in data_keys: relevant_data = filter(lambda x: x[2] == key, data) xs.append([]) ys.append([]) ps.append(key) for line in relevant_data: xs[-1].append(float(line[0])) ys[-1].append(float(line[1])) plot(xs, ys, x_label=header[0], y_label=header[1], x_log=xlog, y_log=ylog, names=ps, graph_name=title, legend_name=header[2], disable_x_tics=no_x_tics, xyline=not noxy, no_scatter=no_scatter, col=not bw, nosymbols=nosymbols) else: xs = [] ys = [] for d in data: xs.append(float(d[0])) ys.append(float(d[1])) plot([xs], [ys], x_label=header[0], y_label=header[1], x_log=xlog, y_log=ylog, graph_name=title, disable_x_tics=no_x_tics, xyline=not noxy, no_scatter=no_scatter, col=not bw, nosymbols=nosymbols)
def compare_results(dirs): from krrt.utils import load_CSV print print "Comparing results for %s..." % (dirs) coverage = [0] * len(dirs) time_better = [0] * len(dirs) time_score = [0] * len(dirs) pruned_score = [0] * len(dirs) generated_score = [0] * len(dirs) quality_better = [0] * len(dirs) quality_score = [0] * len(dirs) nodes_expanded = [0] * len(dirs) nodes_generated = [0] * len(dirs) nodes_pruned = [0] * len(dirs) time_total = [0] * len(dirs) search_time = [0] * len(dirs) pruned_avg = [0] * len(dirs) baseline = None stats_results = False if 'stats' in dirs[0]: stats_results = True for i in range(len(dirs)): if dirs[i] == 'results_brfs': baseline = i break for dom in domains: data = [] cov = [] t_score = [] st_score = [] q_score = [] p_score = [] g_score = [] p_avg = [] for i in range(len(dirs)): direc = dirs[i] data.append(load_CSV("%s/%s.csv" % (direc, dom))[1:]) cov.append(len(filter(lambda x: 'ok' == x[1], data[-1]))) coverage[i] += cov[-1] shared_data = [] for prob in zip(*data): if 0 != len(filter(lambda x: 'ok' == x[1], prob)): for x in prob: if x[1] != 'ok': x[2] = sys.maxint if x[1] != 'ok': x[3] = sys.maxint shared_data.append(list(prob)) for i in range(len(dirs)): tsum = 0 for x in shared_data: if x[i][1] != 'ok': continue tstar = max(1.0, min([float(x[j][2]) for j in range(len(x))])) t = float(x[i][2]) if float(x[i][2]) < 1.0: tsum+=1.0 else: tsum += 1/(1+log10(t/tstar)) #tsum += float(tstar/t) #tsum += log10(1+tstar)/log10(1+t) t_score.append(tsum) time_score[i] += t_score[i] for i in range(len(dirs)): tsum = 0 for x in shared_data: if x[i][1] != 'ok': continue tstar = float("inf") for j in range(len(x)): if x[j][1] != 'ok':continue if baseline == j: if tstar > float(x[j][2]): tstar = float(x[j][2]) else: if tstar > float(x[j][11]): tstar = float(x[j][11]) tstar = max(1.0, tstar) t = 0 if baseline == i: t = float(x[i][2]) else: t = float(x[i][11]) if t < 1.0: tsum+=1.0 else: tsum += 1/(1+log10(t/tstar)) #tsum += float(tstar/t) #tsum += log10(1+tstar)/log10(1+t) st_score.append(tsum) search_time[i] += st_score[i] for i in range(len(dirs)): tsum = 0 for x in shared_data: if x[i][1] != 'ok': continue all_solved = True for j in range(len(dirs)): if x[j][1] != 'ok': all_solved = False break if all_solved is False: continue tstar = 1.0 for j in range(len(x)): if x[j][1] != 'ok':continue if tstar < float(x[j][6]): tstar = float(x[j][6]) t = float(x[i][6]) if float(x[i][6]) < 1.0: tsum+=0.0 else: tsum += (1+log10(t/tstar))/(1) #tsum += float(t/tstar) #tsum += log10(1+tstar)/log10(1+t) p_score.append(tsum) pruned_score[i] += p_score[i] #Pruned Average for i in range(len(dirs)): tsum = 0 total_solved = 0 xprun = 0.0 for x in shared_data: if x[i][1] != 'ok': continue all_solved = True for j in range(len(dirs)): if x[j][1] != 'ok': all_solved = False break if all_solved is False: continue total_solved+=1 xprun += float(x[i][6]) if total_solved == 0: p_avg.append( 0 ) else: p_avg.append( xprun / total_solved ) pruned_avg[i] += p_avg[i] if baseline is None: for i in range(len(dirs)): tsum = 0 for x in shared_data: if x[i][1] != 'ok': continue tstar = float("inf") for j in range(len(x)): if x[j][1] != 'ok':continue if tstar > float(x[j][4]): tstar = float(x[j][4]) tstar = max(1.0, tstar) t = float(x[i][4]) if t < 1.0: tsum+=1.0 else: tsum += 1/(1+log10(t/tstar)) #tsum += float(t/tstar) #tsum += log10(1+tstar)/log10(1+t) g_score.append(tsum) generated_score[i] += g_score[i] else: #Generated Reduction percentage for i in range(len(dirs)): tsum = 0 total_solved = 0 baseline_generated = 0.0 xgen = 0.0 for x in shared_data: if x[baseline][1] != 'ok': continue if x[i][1] != 'ok': continue total_solved+=1 baseline_generated += float(x[baseline][4]) xgen += float(x[i][4]) if xgen == 0: g_score.append( 1 ) else: g_score.append( baseline_generated / xgen ) generated_score[i] += g_score[i] #n_gen_1 = sum([int(x[0][4]) for x in shared_data]) #n_gen_2 = sum([int(x[1][4]) for x in shared_data]) #nodes_generated[0] += n_gen_1 #nodes_generated[1] += n_gen_2 #n_exp_1 = sum([int(x[0][5]) for x in shared_data]) #n_exp_2 = sum([int(x[1][5]) for x in shared_data]) #nodes_expanded[0] += n_exp_1 #nodes_expanded[1] += n_exp_2 #time_1 = sum([float(x[0][2]) for x in shared_data]) #time_2 = sum([float(x[1][2]) for x in shared_data]) #time_total[0] += time_1 #time_total[1] += time_2 print "\nDomain: %s" % dom print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in cov]) print "Search Time score: %s" % ' -vs- '.join(["%.2f" % t for t in st_score]) print "Total Time score: %s" % ' -vs- '.join(["%.2f" % t for t in t_score]) if stats_results is False: print "Generated Reduction Factor: %s" % ' -vs- '.join(["%.2f" % t for t in g_score]) else: print "Avg. States Pruned: %s" % ' -vs- '.join(["%.2f" % (t/float(len(dirs))) for t in p_avg]) if baseline is None: print "Pruned score: %s" % ' -vs- '.join(["%.2f" % t for t in p_score]) print "\nDomain: all" print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in coverage]) print "Search Time score: %s" % ' -vs- '.join(["%.2f" % t for t in search_time]) print "Total Time score: %s" % ' -vs- '.join(["%.2f" % t for t in time_score]) if stats_results is False: print "Generated Reduction Factor: %s" % ' -vs- '.join(["%.2f" % t for t in generated_score]) else: print "Avg. States Pruned: %s" % ' -vs- '.join(["%.2f" % (t/float(len(dirs))) for t in pruned_avg]) if baseline is None: print "Pruned score: %s" % ' -vs- '.join(["%.2f" % t for t in pruned_score])
def checkit(filename): data = load_CSV(filename)[1:] for row in data: for i in range(len(row)): if row[i] == '0.000000': row[i] = '0.000001' def plot_data(data, inds, labs, cols, zlabel, fname): data_map = {} for ag in range(AGENTS[0], AGENTS[1] + 1): data_map[ag] = {} for dep in range(DEPTH[0], DEPTH[1] + 1): data_map[ag][dep] = {} for row in data: data_map[int(row[0])][int(row[1])][inds[0]] = float(row[inds[0]]) data_map[int(row[0])][int(row[1])][inds[1]] = float(row[inds[1]]) data_map[int(row[0])][int(row[1])][inds[2]] = float(row[inds[2]]) from mpl_toolkits.mplot3d import axes3d import matplotlib.pyplot as plt import matplotlib import numpy as np X, Y = np.meshgrid(np.arange(AGENTS[0], AGENTS[1] + 1), np.arange(DEPTH[0], DEPTH[1] + 1)) #zs0 = np.array([1 for x,y in zip(np.ravel(X), np.ravel(Y))]) #zs1 = np.array([data_map[x][y][ind1] / data_map[x][y][indnorm] for x,y in zip(np.ravel(X), np.ravel(Y))]) #zs2 = np.array([data_map[x][y][ind2] / data_map[x][y][indnorm] for x,y in zip(np.ravel(X), np.ravel(Y))]) zs0 = np.array([ data_map[x][y][inds[0]] for x, y in zip(np.ravel(X), np.ravel(Y)) ]) zs1 = np.array([ data_map[x][y][inds[1]] for x, y in zip(np.ravel(X), np.ravel(Y)) ]) zs2 = np.array([ data_map[x][y][inds[2]] for x, y in zip(np.ravel(X), np.ravel(Y)) ]) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') if 'Query Time ($log_e(sec)$)' == zlabel or 'Update Time ($log_e(sec)$)' == zlabel: print "za = " + str(zs0) Z0 = np.log(zs0).reshape(X.shape) print "Z0 = " + str(Z0) Z1 = np.log(zs1).reshape(X.shape) Z2 = np.log(zs2).reshape(X.shape) else: #ax.set_zticks([]) Z0 = (zs0 / 1000).reshape(X.shape) Z1 = (zs1 / 1000).reshape(X.shape) Z2 = (zs2 / 1000).reshape(X.shape) #ax.plot_wireframe(X, Y, Z0, color='0.75') ax.plot_wireframe(X, Y, Z0, color=cols[0]) ax.plot_wireframe(X, Y, Z1, color=cols[1]) ax.plot_wireframe(X, Y, Z2, color=cols[2]) #cset = ax.contourf(X, Y, Z0, zdir='z', offset=-100, cmap=matplotlib.cm.coolwarm) #cset = ax.contourf(X, Y, Z0, zdir='x', offset=0, cmap=matplotlib.cm.coolwarm) #cset = ax.contourf(X, Y, Z0, zdir='z', offset=0, cmap=matplotlib.cm.coolwarm) #cset = ax.contourf(X, Y, Z0, zdir='y', offset=40, cmap=cm.coolwarm) ax.set_xlabel('# of Agents') ax.set_ylabel('Maximum Depth') ax.set_zlabel(zlabel) scatter1_proxy = matplotlib.lines.Line2D([0], [0], linestyle="none", c=cols[0], marker='s') scatter2_proxy = matplotlib.lines.Line2D([0], [0], linestyle="none", c=cols[1], marker='s') scatter3_proxy = matplotlib.lines.Line2D([0], [0], linestyle="none", c=cols[2], marker='s') ax.legend([scatter1_proxy, scatter2_proxy, scatter3_proxy], [labs[0], labs[1], labs[2]], numpoints=1) ax.get_xaxis().set_major_locator( matplotlib.ticker.MaxNLocator(integer=True)) ax.get_yaxis().set_major_locator( matplotlib.ticker.MaxNLocator(integer=True)) plt.show() col1 = '#1b9e77' col2 = '#d95f02' col3 = '#7570b3' print "Plotting query time..." plot_data(data, [6, 8, 7], ['INF', '$V_{RML}$', 'Closure'], [col1, col3, col2], 'Query Time ($log_e(sec)$)', 'time.eps') print "Plotting size..." plot_data(data, [4, 3, 5], ['Closure', 'INF', '$V_{RML}$'], [col2, col1, col3], 'Size (x1000)', 'size.eps') print "Plotting update time..." plot_data(data, [9, 11, 10], ['INF', '$V_{RML}$', 'Closure'], [col1, col3, col2], 'Update Time ($log_e(sec)$)', 'update_time.eps')
def compare_results(dirs): from krrt.utils import load_CSV print print "Comparing results for %s..." % (dirs) coverage = [0] * len(dirs) time_better = [0] * len(dirs) time_score = [0] * len(dirs) quality_better = [0] * len(dirs) quality_score = [0] * len(dirs) nodes_expanded = [0] * len(dirs) nodes_generated = [0] * len(dirs) time_total = [0] * len(dirs) for dom in domains: data = [] cov = [] t_score = [] q_score = [] for i in range(len(dirs)): direc = dirs[i] data.append(load_CSV("%s/%s.csv" % (direc, dom))[1:]) cov.append(len(filter(lambda x: 'ok' == x[1], data[-1]))) coverage[i] += cov[-1] shared_data = [] for prob in zip(*data): if 0 != len(filter(lambda x: 'ok' == x[1], prob)): for x in prob: if x[1] != 'ok': x[2] = sys.maxint if x[1] != 'ok': x[3] = sys.maxint shared_data.append(list(prob)) for i in range(len(dirs)): tsum = 0 for x in shared_data: if x[i][1] != 'ok': continue tstar = max(1.0, min([float(x[j][2]) for j in range(len(x))])) t = float(x[i][2]) if float(x[i][2]) < 1.0: tsum+=1.0 else: tsum += 1/(1+log10(t/tstar)) #tsum += float(tstar/t) #tsum += log10(1+tstar)/log10(1+t) t_score.append(tsum) time_score[i] += t_score[i] for i in range(len(dirs)): q_score.append( sum([ min([float(x[j][3]) for j in range(len(x)) ]) / float(x[i][3]) for x in shared_data ])) quality_score[i] += q_score[i] #n_gen_1 = sum([int(x[0][4]) for x in shared_data]) #n_gen_2 = sum([int(x[1][4]) for x in shared_data]) #nodes_generated[0] += n_gen_1 #nodes_generated[1] += n_gen_2 #n_exp_1 = sum([int(x[0][5]) for x in shared_data]) #n_exp_2 = sum([int(x[1][5]) for x in shared_data]) #nodes_expanded[0] += n_exp_1 #nodes_expanded[1] += n_exp_2 #time_1 = sum([float(x[0][2]) for x in shared_data]) #time_2 = sum([float(x[1][2]) for x in shared_data]) #time_total[0] += time_1 #time_total[1] += time_2 print "\nDomain: %s" % dom print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in cov]) #print "Time better: %d -vs- %d" % (time_better_1, time_better_2) print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in t_score]) #print "Quality better: %d -vs- %d" % (quality_better_1, quality_better_2) print "Quality score: %s" % ' -vs- '.join(["%.2f" % t for t in q_score]) #print "Time per node generated: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_gen_1))), (float(time_2) / float(max(1,n_gen_2)))) #print "Time per node expanded: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_exp_1))), (float(time_2) / float(max(1,n_exp_2)))) print "\nDomain: all" print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in coverage]) #print "Time better: %d -vs- %d" % (time_better[0], time_better[1]) print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in time_score]) #print "Quality better: %d -vs- %d" % (quality_better[0], quality_better[1]) print "Quality score: %s" % ' -vs- '.join(["%.2f" % t for t in quality_score])
def compare_results(dirs): from krrt.utils import load_CSV print print "Comparing results for %s..." % (dirs) coverage = [0] * len(dirs) time_better = [0] * len(dirs) time_score = [0] * len(dirs) quality_better = [0] * len(dirs) quality_score = [0] * len(dirs) nodes_expanded = [0] * len(dirs) nodes_generated = [0] * len(dirs) time_total = [0] * len(dirs) for dom in domains: data = [] cov = [] t_score = [] q_score = [] for i in range(len(dirs)): direc = dirs[i] data.append(load_CSV("%s/%s.csv" % (direc, dom))[1:]) cov.append(len(filter(lambda x: 'ok' == x[1], data[-1]))) coverage[i] += cov[-1] shared_data = [] for prob in zip(*data): if 0 != len(filter(lambda x: 'ok' == x[1], prob)): for x in prob: if x[1] != 'ok': x[2] = sys.maxint if x[1] != 'ok': x[3] = sys.maxint shared_data.append(list(prob)) for i in range(len(dirs)): tsum = 0 for x in shared_data: if x[i][1] != 'ok': continue tstar = max(1.0, min([float(x[j][2]) for j in range(len(x))])) t = float(x[i][2]) if float(x[i][2]) < 1.0: tsum += 1.0 else: tsum += 1 / (1 + log10(t / tstar)) #tsum += float(tstar/t) #tsum += log10(1+tstar)/log10(1+t) t_score.append(tsum) time_score[i] += t_score[i] for i in range(len(dirs)): q_score.append( sum([ min([float(x[j][3]) for j in range(len(x))]) / float(x[i][3]) for x in shared_data ])) quality_score[i] += q_score[i] #n_gen_1 = sum([int(x[0][4]) for x in shared_data]) #n_gen_2 = sum([int(x[1][4]) for x in shared_data]) #nodes_generated[0] += n_gen_1 #nodes_generated[1] += n_gen_2 #n_exp_1 = sum([int(x[0][5]) for x in shared_data]) #n_exp_2 = sum([int(x[1][5]) for x in shared_data]) #nodes_expanded[0] += n_exp_1 #nodes_expanded[1] += n_exp_2 #time_1 = sum([float(x[0][2]) for x in shared_data]) #time_2 = sum([float(x[1][2]) for x in shared_data]) #time_total[0] += time_1 #time_total[1] += time_2 print "\nDomain: %s" % dom print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in cov]) #print "Time better: %d -vs- %d" % (time_better_1, time_better_2) print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in t_score]) #print "Quality better: %d -vs- %d" % (quality_better_1, quality_better_2) print "Quality score: %s" % ' -vs- '.join( ["%.2f" % t for t in q_score]) #print "Time per node generated: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_gen_1))), (float(time_2) / float(max(1,n_gen_2)))) #print "Time per node expanded: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_exp_1))), (float(time_2) / float(max(1,n_exp_2)))) print "\nDomain: all" print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in coverage]) #print "Time better: %d -vs- %d" % (time_better[0], time_better[1]) print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in time_score]) #print "Quality better: %d -vs- %d" % (quality_better[0], quality_better[1]) print "Quality score: %s" % ' -vs- '.join( ["%.2f" % t for t in quality_score])
def ab_compare(res1, res2): def get_time(line): if line[4] == '-': return max(0.1, float(line[2])) else: return float('inf') def get_size(line): if line[4] == '-': return float(line[3]) else: return float('inf') def get_score(_x, y): x = max(_x, 0.001) if x == float('inf') and y == float('inf'): return 0.0 else: return min(x, y) / x res1_experiments = set( map(lambda x: x.split('/')[-1], get_file_list(res1, match_list=['.csv']))) res2_experiments = set( map(lambda x: x.split('/')[-1], get_file_list(res2, match_list=['.csv']))) shared_results = list(res1_experiments & res2_experiments) print print "Found %d overlapping experiments to compare." % len(shared_results) time_score = [0.0, 0.0] size_score = [0.0, 0.0] coverage = [0, 0] for res in shared_results: data1 = load_CSV(res1 + '/' + res)[1:] data2 = load_CSV(res2 + '/' + res)[1:] if len(data1) != len(data2): print "Error with %s experiment -- different number of data points." % res continue times1 = [get_time(line) for line in data1] times2 = [get_time(line) for line in data2] time1 = sum( [get_score(times1[i], times2[i]) for i in range(len(data1))]) time2 = sum( [get_score(times2[i], times1[i]) for i in range(len(data1))]) sizes1 = [get_size(line) for line in data1] sizes2 = [get_size(line) for line in data2] size1 = sum( [get_score(sizes1[i], sizes2[i]) for i in range(len(data1))]) size2 = sum( [get_score(sizes2[i], sizes1[i]) for i in range(len(data1))]) cov1 = len(filter(lambda x: x[4] == '-', data1)) cov2 = len(filter(lambda x: x[4] == '-', data2)) time_score[0] += time1 time_score[1] += time2 size_score[0] += size1 size_score[1] += size2 coverage[0] += cov1 coverage[1] += cov2 print "\n [ %s ]\n" % res.split('-results.csv')[0] print " Coverage: %d -vs- %d" % (cov1, cov2) print "Size Score: %.2f -vs- %.2f" % (size1, size2) print "Time Score: %.2f -vs- %.2f\n" % (time1, time2) print "\n [ OVERALL ]" print " Coverage: %d -vs- %d" % (coverage[0], coverage[1]) print "Size Score: %.2f -vs- %.2f" % (size_score[0], size_score[1]) print "Time Score: %.2f -vs- %.2f" % (time_score[0], time_score[1]) print
def online_compare(domain): if 'every' in domain: dom_type = [] if 'fond' in domain: dom_type = FOND_DOMAINS elif 'new' in domain: dom_type = NEW_DOMAINS elif 'ipc' in domain: dom_type = IPC06_DOMAINS elif 'interesting' in domain: dom_type = INTERESTING_DOMAINS for dom in dom_type: online_compare(dom) return elif 'all' in domain: prp_data = [] ffr_data = [] dom_type = [] if 'fond' in domain: dom_type = FOND_DOMAINS elif 'new' in domain: dom_type = NEW_DOMAINS elif 'ipc' in domain: dom_type = IPC06_DOMAINS elif 'interesting' in domain: dom_type = INTERESTING_DOMAINS for dom in dom_type: prp_data.extend(load_CSV("RESULTS/bo-prp-%s-results.csv" % dom)) ffr_data.extend(load_CSV("RESULTS/ffr-prp-%s-results.csv" % dom)) else: prp_data = load_CSV("RESULTS/bo-prp-%s-results.csv" % domain) ffr_data = load_CSV("RESULTS/ffr-prp-%s-results.csv" % domain) # Load both sets solved_prp_data = filter(lambda x: x[4] == '-', prp_data) solved_ffr_data = filter(lambda x: x[4] == '-', ffr_data) prp_mapping = {} ffr_mapping = {} for line in solved_ffr_data: assert (line[0], line[1]) not in ffr_mapping ffr_mapping[(line[0], line[1])] = line for line in solved_prp_data: assert (line[0], line[1]) not in prp_mapping prp_mapping[(line[0], line[1])] = line ffr_solved = set(ffr_mapping.keys()) prp_solved = set(prp_mapping.keys()) both_solved = ffr_solved & prp_solved ffr_actions = [] prp_actions = [] ffr_replans = [] prp_replans = [] ffr_times = [] prp_times = [] ffr_success = [] prp_success = [] for (dom, prob) in both_solved: ffr_actions.append(float(ffr_mapping[(dom, prob)][-4])) prp_actions.append(float(prp_mapping[(dom, prob)][-4])) ffr_replans.append(float(ffr_mapping[(dom, prob)][-5])) prp_replans.append(float(prp_mapping[(dom, prob)][-5])) ffr_success.append(float(ffr_mapping[(dom, prob)][-1])) prp_success.append(float(prp_mapping[(dom, prob)][-1])) ffr_times.append( float(ffr_mapping[(dom, prob)][-7]) / float(ffr_mapping[(dom, prob)][6])) prp_times.append( float(prp_mapping[(dom, prob)][-7]) / float(prp_mapping[(dom, prob)][6])) print "Online replanning for domain %s:\n" % domain print "Total ffr successful runs: %d" % len(ffr_solved) print "Total prp successful runs: %d" % len(prp_solved) print "Total shared successful runs: %d\n" % len(both_solved) print "FFR Average Actions: %f" % (sum(ffr_actions) / float(len(ffr_actions))) print "PRP Average Actions: %f\n" % (sum(prp_actions) / float(len(prp_actions))) print "FFR Average Replans: %f" % (sum(ffr_replans) / float(len(ffr_replans))) print "PRP Average Replans: %f\n" % (sum(prp_replans) / float(len(prp_replans))) print "FFR Average Success: %f" % (sum(ffr_success) / float(len(ffr_success))) print "PRP Average Success: %f\n" % (sum(prp_success) / float(len(prp_success))) print "FFR Total Success: %d" % int(sum(ffr_success)) print "PRP Total Success: %d\n" % int(sum(prp_success)) print "FFR Average Time: %f" % (sum(ffr_times) / float(len(ffr_times))) print "PRP Average Time: %f\n" % (sum(prp_times) / float(len(prp_times))) return
def fip_vs_prp(domain): if 'every' in domain: dom_type = [] if 'fond' in domain: dom_type = FOND_DOMAINS elif 'new' in domain: dom_type = NEW_DOMAINS elif 'ipc' in domain: dom_type = IPC06_DOMAINS elif 'interesting' in domain: dom_type = INTERESTING_DOMAINS for dom in dom_type: fip_vs_prp(dom) return elif 'all' in domain: fip_data = [] prp_data = [] dom_type = [] if 'fond' in domain: dom_type = FOND_DOMAINS elif 'new' in domain: dom_type = NEW_DOMAINS elif 'ipc' in domain: dom_type = IPC06_DOMAINS elif 'interesting' in domain: dom_type = INTERESTING_DOMAINS for dom in dom_type: prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom)) fip_data = [] prp_data = [] for dom in dom_type: fip_data.extend(load_CSV("RESULTS/fip-%s-results.csv" % dom)) prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom)) else: fip_data = load_CSV("RESULTS/fip-%s-results.csv" % domain) prp_data = load_CSV("RESULTS/prp-%s-results.csv" % domain) print "\nAnalyzing FIP vs PRP for %s:" % domain # Load both sets solved_fip_data = filter(lambda x: x[-1] == '-' and x[-2] != '0', fip_data) nosol_fip_data = filter(lambda x: x[-1] == 'N', fip_data) prp_data = [prp_data[0]] + filter_prp_settings( prp_data, '18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1') solved_prp_data = filter(lambda x: x[-2] == 'True', prp_data) solved_prp_data = average_prp_data(solved_prp_data) nosol_prp_data = filter(lambda x: x[4] == 'N' or x[-2] == 'False', prp_data) nosol_prp_data = average_prp_data(nosol_prp_data) fip_mapping = {} prp_mapping = {} for line in solved_fip_data: assert (line[0], line[1]) not in fip_mapping fip_mapping[(line[0], line[1])] = line for line in solved_prp_data: assert (line[0], line[1]) not in prp_mapping prp_mapping[(line[0], line[1])] = line fip_solved = set(fip_mapping.keys()) prp_solved = set(prp_mapping.keys()) both_solved = fip_solved & prp_solved print "FIP Coverage: %d / %d" % (len(fip_solved), len(nosol_fip_data)) print "PRP Coverage: %d / %d" % (len(prp_solved), len(nosol_prp_data)) print "Combined Coverage: %d" % len(both_solved) print "FIP - PRP: %s" % str(fip_solved - prp_solved) all_time_data = [] time_data = [] size_data = [] probs = [] for (dom, prob) in both_solved: probs.append(prob) time_data.append((float(fip_mapping[(dom, prob)][2]), float(prp_mapping[(dom, prob)][17]))) size_data.append((float(fip_mapping[(dom, prob)][3]), float(prp_mapping[(dom, prob)][3]))) #HACK: We assign 0.001 seconds to a reported time of 0: This is needed for log plots to work plot([max(0.001, item[0]) for item in time_data], [max(0.001, item[1]) for item in time_data], x_label="FIP Time (s)", y_label="PRP Time (s)", makesquare=True, x_log=True, y_log=True, col=False) plot([item[0] for item in size_data], [item[1] for item in size_data], x_label="FIP Policy Size", y_label="PRP Policy Size", makesquare=True, x_log=True, y_log=True, col=False) x1, y1 = create_time_profile([ max(0.001, float(fip_mapping[(dom, prob)][2])) for (dom, prob) in fip_solved ]) x2, y2 = create_time_profile([ max(0.001, float(prp_mapping[(dom, prob)][17])) for (dom, prob) in prp_solved ]) plot([x1, x2], [y1, y2], x_label="Time", y_label="Problems Solved", no_scatter=True, xyline=False, names=["FIP", "PRP"], x_log=True, col=False) print
('18000', '0', '1', '1', '0', '1', '0', '0', '0', '0', '0'), ('18000', '0', '0', '1', '0', '1', '0', '0', '0', '0', '0'), 'PRP$_{\\textrm{Full}}$', 'PRP') if 'ffreplan-vs-prp' in flags: online_compare(myargs['-domain']) if 'anova-time' in flags: do_anova(myargs['-domain'], 'runtime') if 'anova-size' in flags: do_anova(myargs['-domain'], 'size') if 'ablation' in flags: prp_data = load_CSV("RESULTS/best-prp-%s-results.csv" % myargs['-domain']) + \ load_CSV("RESULTS/no-local-prp-%s-results.csv" % myargs['-domain']) _prp_compare_two( myargs['-domain'], ('18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1'), ('18000', '0', '0', '1', '0', '1', '0', '1', '1', '1', '1'), 'PRP', 'PRP$_{\\textrm{-local}}$', prp_data) prp_data = load_CSV("RESULTS/best-prp-%s-results.csv" % myargs['-domain']) + \ load_CSV("RESULTS/no-scd-prp-%s-results.csv" % myargs['-domain']) _prp_compare_two( myargs['-domain'], ('18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1'), ('18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '0'), 'PRP', 'PRP$_{\\textrm{-scd}}$', prp_data)