def dojic(dom, prob, max_jic): print "Running a varied jic experiment for the following domain / problem:" print dom print prob exp_name = "jic" domain = dom.split('/')[-2] PRP_PARAMS['jic']['--jic-limit'] = list(range(1, max_jic + 1)) prp_results = run_experiment( base_command='./../../src/prp', single_arguments={'domprob': ["../%s ../%s" % (dom, prob)]}, parameters=PRP_PARAMS['jic'], time_limit=TIME_LIMIT, memory_limit=MEM_LIMIT, results_dir="RESULTS/%s-%s" % (exp_name, domain), progress_file=None, processors=CORES, sandbox=exp_name, clean_sandbox=True, trials=TRIALS, output_file_func=(lambda res: res.single_args['domprob'].split(' ')[1]. split('/')[-1] + '.' + str(res.id) + '.out'), error_file_func=(lambda res: res.single_args['domprob'].split(' ')[1]. split('/')[-1] + '.' + str(res.id) + '.err')) prp_csv = ['jic limit,jic time,policy score'] for res_id in prp_results.get_ids(): result = prp_results[res_id] jic_lim = result.parameters['--jic-limit'] jic_time = get_value(result.output_file, '.*Just-in-case Repairs: ([0-9]+\.?[0-9]*)s\n.*', float) policy_score = get_value(result.output_file, '.*Policy Score: ([0-9]+\.?[0-9]*)\n.*', float) prp_csv.append("%s,%f,%f" % (jic_lim, jic_time, policy_score)) append_file("RESULTS/%s-%s-results.csv" % (exp_name, domain), prp_csv)
] results = run_experiment( base_directory=".", base_command='./../../src/plan-prp', single_arguments={'domprob': domprobs}, time_limit=1800, # 15minute time limit (900 seconds) memory_limit=1000, # 1gig memory limit (1000 megs) results_dir="%s/results" % domain, progress_file=None, # Print the progress to stdout processors=6, # You've got 8 cores, right? sandbox='fd_out', clean_sandbox=False) good_results = results.filter(lambda result: not result.timed_out) good_results = [good_results[i] for i in good_results.get_ids()] print "Coverage: %d / %d" % (len(good_results), len(domprobs)) try: data = ['runtime(s),size(nodes)'] + [ "%f,%d" % (res.runtime, get_value(res.output_file, '.*State-Action Pairs: (\d+)\n.*', int)) for res in good_results ] except: pass write_file("%s.csv" % domain, data)
def benchmark_domain(planner, dom, extra_args, lmcut=False): from krrt.utils import get_value, match_value, run_experiment, write_file, get_lines print "Benchmarking %s..." % dom global ipc if lmcut and '../' != ipc[:3]: ipc = '../' + ipc if TYPE == OLD: domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]] elif TYPE == NEW: #domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] domprob_args = ["%s/%s/%s %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) sand = None if lmcut: sand = 'lmcut-'+dom results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, clean_sandbox=True, sandbox=sand) data = [] for resid in results.get_ids(): res = results[resid] if lmcut: prob = res.single_args['domprob'].split()[1].split('/')[-1] elif TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-4].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file os.system("cp %s %s"%(outfile, outfile+dom)) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*This configuration does not support axioms.*'): data.append("%s,ax,-1,-1,-1,-1" % prob) else: if match_value(outfile, '.*Plan found with cost: ([0-9]+).*'): quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int) generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int) expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int) data.append("%s,ok,%.2f,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded)) else: print "Error with %s" % prob data.append("%s,err,%.2f,-1,-1,-1" % (prob, res.runtime)) if 'old-' in planner: cmd("rm TMP_OUTPUT") data.sort() header = ['problem,status,runtime,quality,generated,expanded'] data = header + data write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom): from krrt.utils import get_value, match_value, run_experiment, write_file print print "Benchmarking %s..." % dom if TYPE == OLD: domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] elif TYPE == NEW: domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, sandbox=None) data = [] for resid in results.get_ids(): res = results[resid] if TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file os.system("cp %s %s"%(outfile, outfile+dom)) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) else: if match_value(outfile, '.*Effective Width during search*'): lines = open(outfile) quality = 0 generated = 0 expanded = 0 width = 0 for line in lines: if 'Plan found with cost:' in line: quality = int(line.split(':')[-1]) if 'Nodes generated during search:' in line: generated = int(line.split(':')[-1]) if 'Nodes expanded during search:' in line: expanded = int(line.split(':')[-1]) if 'Effective Width during search:' in line: width = int(line.split(':')[-1]) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, width)) elif match_value(outfile, '([0-9]+).* .*actions in the plan.'): quality = get_value(outfile, '([0-9]+).* .*actions in the plan.', int) generated = 0 expanded = 0 backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'): quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int) generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int) expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Length : ([0-9]+).*'): quality = get_value(outfile, '.*Length : ([0-9]+).*', int) generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*', int) expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*'): quality = get_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*', float) generated = get_value(outfile, '.*Generated: ([0-9]+).*', int) expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*NOT I-REACHABLE.*'): backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,not-i,%f,-1,-1,-1,%d" % (prob, res.runtime, backtracks)) else: print "Error with %s" % prob data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime)) if 'old-' in planner: cmd("rm TMP_OUTPUT") data.sort() data = ['problem,status,runtime,quality,generated,expanded'] + data write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom): from krrt.utils import get_value, match_value, run_experiment, write_file print print "Benchmarking %s..." % dom if TYPE == OLD: domprob_args = [ "--domain %s/%s/%s --problem %s/%s/%s" % (ipc, dom, domain, ipc, dom, problem) for (domain, problem) in benchmark[dom] ] elif TYPE == NEW: domprob_args = [ "%s/%s/%s %s/%s/%s o/dev/null" % (ipc, dom, domain, ipc, dom, problem) for (domain, problem) in benchmark[dom] ] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, sandbox=None) data = [] for resid in results.get_ids(): res = results[resid] if TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file os.system("cp %s %s" % (outfile, outfile + dom)) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) else: if match_value(outfile, '.*Effective Width during search*'): lines = open(outfile) quality = 0 generated = 0 expanded = 0 width = 0 for line in lines: if 'Plan found with cost:' in line: quality = int(line.split(':')[-1]) if 'Nodes generated during search:' in line: generated = int(line.split(':')[-1]) if 'Nodes expanded during search:' in line: expanded = int(line.split(':')[-1]) if 'Effective Width during search:' in line: width = int(line.split(':')[-1]) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, width)) elif match_value(outfile, '([0-9]+).* .*actions in the plan.'): quality = get_value(outfile, '([0-9]+).* .*actions in the plan.', int) generated = 0 expanded = 0 backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'): quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int) generated = get_value( outfile, '.*Nodes generated during search: ([0-9]+).*', int) expanded = get_value( outfile, '.*Nodes expanded during search: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Length : ([0-9]+).*'): quality = get_value(outfile, '.*Length : ([0-9]+).*', int) generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*', int) expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*'): quality = get_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*', float) generated = get_value(outfile, '.*Generated: ([0-9]+).*', int) expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*NOT I-REACHABLE.*'): backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,not-i,%f,-1,-1,-1,%d" % (prob, res.runtime, backtracks)) else: print "Error with %s" % prob data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime)) if 'old-' in planner: cmd("rm TMP_OUTPUT") data.sort() data = ['problem,status,runtime,quality,generated,expanded'] + data write_file("%s/%s.csv" % (results_directory, dom), data)
def parse_prp(outfile): runtime = get_value(outfile, '.*Total time: ([0-9]+\.?[0-9]*)s\n.*', float) jic_time = get_value(outfile, '.*Just-in-case Repairs: ([0-9]+\.?[0-9]*)s\n.*', float) policy_use_time = get_value(outfile, '.*Using the policy: ([0-9]+\.?[0-9]*)s\n.*', float) policy_construction_time = get_value( outfile, '.*Policy Construction: ([0-9]+\.?[0-9]*)s\n.*', float) policy_eval_time = get_value( outfile, '.*Evaluating the policy quality: ([0-9]+\.?[0-9]*)s\n.*', float) search_time = get_value(outfile, '.*Search Time: ([0-9]+\.?[0-9]*)s\n.*', float) engine_init_time = get_value( outfile, '.*Engine Initialization: ([0-9]+\.?[0-9]*)s\n.*', float) regression_time = get_value( outfile, '.*Regression Computation: ([0-9]+\.?[0-9]*)s\n.*', float) simulator_time = get_value(outfile, '.*Simulator time: ([0-9]+\.?[0-9]*)s\n.*', float) successful_states = get_value( outfile, '.*Successful states: ([0-9]+\.?[0-9]*) \+.*', float) replans = get_value(outfile, '.*Replans: ([0-9]+\.?[0-9]*) \+.*', float) actions = get_value(outfile, '.*Actions: ([0-9]+\.?[0-9]*) \+.*', float) size = get_value(outfile, '.*State-Action Pairs: (\d+)\n.*', int) strongly_cyclic = match_value(outfile, '.*Strongly Cyclic: True.*') succeeded = get_value(outfile, '.*Succeeded: (\d+) .*', int) policy_score = get_value(outfile, '.*Policy Score: ([0-9]+\.?[0-9]*)\n.*', float) return runtime, jic_time, policy_eval_time, policy_construction_time, policy_use_time, \ search_time, engine_init_time, regression_time, simulator_time, successful_states, \ replans, actions, size, strongly_cyclic, succeeded, policy_score
def parse_fip(outfile): runtime = get_value(outfile, '.* ([0-9]+\.?[0-9]+) seconds searching.*', float) policy_size = len(filter(lambda x: 'case S' in x, read_file(outfile))) return runtime, policy_size