def doit_fip(domain, dom_probs, exp_name='fip'): print "\n\nRunning FIP experiments on domain, %s" % domain fip_args = [ "-o ../%s.fip -f ../%s" % (item[0], item[1]) for item in dom_probs ] fip_results = run_experiment( base_command='./../fip', single_arguments={'domprob': fip_args}, time_limit=TIME_LIMIT, memory_limit=MEM_LIMIT, results_dir="RESULTS/%s-%s" % (exp_name, domain), progress_file=None, processors=CORES, sandbox=exp_name, clean_sandbox=True, output_file_func=( lambda res: res.single_args['domprob'].split('/')[-1] + '.out'), error_file_func=( lambda res: res.single_args['domprob'].split('/')[-1] + '.err')) timeouts = 0 memouts = 0 errorouts = 0 fip_csv = ['domain,problem,runtime,size,status'] for res_id in fip_results.get_ids(): result = fip_results[res_id] prob = result.single_args['domprob'].split(' ')[3].split('/')[-1] if match_value(result.output_file, 'No plan will solve it' ) or 'No solutions are found!' in read_file( result.output_file)[-1]: fip_csv.append("%s,%s,-1,-1,N" % (domain, prob)) else: if result.timed_out: timeouts += 1 fip_csv.append("%s,%s,-1,-1,T" % (domain, prob)) elif result.mem_out: memouts += 1 fip_csv.append("%s,%s,-1,-1,M" % (domain, prob)) elif not result.clean_run or check_segfault(result.output_file): errorouts += 1 fip_csv.append("%s,%s,-1,-1,E" % (domain, prob)) else: run, size = parse_fip(result.output_file) fip_csv.append("%s,%s,%f,%d,-" % (domain, prob, run, size)) print "\nTimed out %d times." % timeouts print "Ran out of memory %d times." % memouts print "Unknown error %d times." % errorouts append_file("RESULTS/%s-%s-results.csv" % (exp_name, domain), fip_csv)
def parse_output_FF(file_name): from krrt.utils import match_value, get_lines # Check for the failed solution if match_value(file_name, '.* No plan will solve it.*'): print "No solution." return None # Get the plan ff_actions = get_lines(file_name, lower_bound='.*found legal plan.*', upper_bound='.*time spent.*') ff_actions = filter(lambda x: x != '', [item.strip(' ') for item in ff_actions]) ff_actions = [item.split(':')[1].strip(' ').lower() for item in ff_actions] actions = [Action(item) for item in ff_actions] return Plan(actions)
def parse_prp(outfile): runtime = get_value(outfile, '.*Total time: ([0-9]+\.?[0-9]*)s\n.*', float) jic_time = get_value(outfile, '.*Just-in-case Repairs: ([0-9]+\.?[0-9]*)s\n.*', float) policy_use_time = get_value(outfile, '.*Using the policy: ([0-9]+\.?[0-9]*)s\n.*', float) policy_construction_time = get_value( outfile, '.*Policy Construction: ([0-9]+\.?[0-9]*)s\n.*', float) policy_eval_time = get_value( outfile, '.*Evaluating the policy quality: ([0-9]+\.?[0-9]*)s\n.*', float) search_time = get_value(outfile, '.*Search Time: ([0-9]+\.?[0-9]*)s\n.*', float) engine_init_time = get_value( outfile, '.*Engine Initialization: ([0-9]+\.?[0-9]*)s\n.*', float) regression_time = get_value( outfile, '.*Regression Computation: ([0-9]+\.?[0-9]*)s\n.*', float) simulator_time = get_value(outfile, '.*Simulator time: ([0-9]+\.?[0-9]*)s\n.*', float) successful_states = get_value( outfile, '.*Successful states: ([0-9]+\.?[0-9]*) \+.*', float) replans = get_value(outfile, '.*Replans: ([0-9]+\.?[0-9]*) \+.*', float) actions = get_value(outfile, '.*Actions: ([0-9]+\.?[0-9]*) \+.*', float) size = get_value(outfile, '.*State-Action Pairs: (\d+)\n.*', int) strongly_cyclic = match_value(outfile, '.*Strongly Cyclic: True.*') succeeded = get_value(outfile, '.*Succeeded: (\d+) .*', int) policy_score = get_value(outfile, '.*Policy Score: ([0-9]+\.?[0-9]*)\n.*', float) return runtime, jic_time, policy_eval_time, policy_construction_time, policy_use_time, \ search_time, engine_init_time, regression_time, simulator_time, successful_states, \ replans, actions, size, strongly_cyclic, succeeded, policy_score
def benchmark_domain(planner, bound, dom): from krrt.utils import get_value, match_value, run_experiment, write_file print print "Benchmarking %s..." % dom if TYPE == OLD: domprob_args = [" --bound %s --domain %s/%s/%s --problem %s/%s/%s" % (bound,ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] elif TYPE == NEW: domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, sandbox=None) data = [] for resid in results.get_ids(): res = results[resid] if TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file path, filename = os.path.split(outfile) os.system("cp %s %s"%(outfile, path+"/"+dom+"_"+prob+".log")) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) lines = open(outfile) quality = 0 generated = 0 expanded = 0 pruned = 0 width = 0 h2=0 h1=0 time=0.0 plan="" goal="" for line in lines: if 'Plan found with cost:' in line: quality = int(line.split(':')[-1]) if 'Nodes generated during search:' in line: generated = int(line.split(':')[-1]) if 'Nodes expanded during search:' in line: expanded = int(line.split(':')[-1]) if 'Nodes pruned by bound:' in line: pruned = int(line.split(':')[-1]) if 'Effective width:' in line: width = int(line.split(':')[-1]) if 'Total time:' in line: time = float(line.split(':')[-1]) if 'h1:' in line: if "inf" in line: h1 = 99999 else: h1 = int(line.split(':')[-1]) if 'h2:' in line: if "inf" in line: h2 = 99999 else: h2 = int(line.split(':')[-1]) if 'plan:' in line: plan = (line.split(':')[-1]).strip() if 'Goal:' in line: goal = (line.split(':')[-1]).strip() if '****' in line: data.append("%s,%s,ok,%f,%d,%d,%d,%d,%d,%d,%d,%s" % (prob, goal, time, generated, pruned, expanded, width, quality,h1,h2,plan)) quality = 0 generated = 0 expanded = 0 pruned = 0 width = 0 h2=0 h1=0 time=0.0 plan="" goal="" os.system("rm "+res.output_file) os.system("rm %s.err"%res.output_file) data.sort() data = ['problem,status,goal,runtime,generated,pruned,expanded,width,quality,h1,h2,plan'] + data write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, bound, dom): from krrt.utils import get_value, match_value, run_experiment, write_file print print "Benchmarking %s..." % dom if TYPE == OLD: domprob_args = [ " --bound %s --domain %s/%s/%s --problem %s/%s/%s" % (bound, ipc, dom, domain, ipc, dom, problem) for (domain, problem) in benchmark[dom] ] elif TYPE == NEW: domprob_args = [ "%s/%s/%s %s/%s/%s o/dev/null" % (ipc, dom, domain, ipc, dom, problem) for (domain, problem) in benchmark[dom] ] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, sandbox=None) data = [] for resid in results.get_ids(): res = results[resid] if TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file path, filename = os.path.split(outfile) os.system("cp %s %s" % (outfile, path + "/" + dom + "_" + prob + ".log")) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) lines = open(outfile) quality = 0 generated = 0 expanded = 0 pruned = 0 width = 0 h2 = 0 h1 = 0 time = 0.0 plan = "" goal = "" for line in lines: if 'Plan found with cost:' in line: quality = int(line.split(':')[-1]) if 'Nodes generated during search:' in line: generated = int(line.split(':')[-1]) if 'Nodes expanded during search:' in line: expanded = int(line.split(':')[-1]) if 'Nodes pruned by bound:' in line: pruned = int(line.split(':')[-1]) if 'Effective width:' in line: width = int(line.split(':')[-1]) if 'Total time:' in line: time = float(line.split(':')[-1]) if 'h1:' in line: if "inf" in line: h1 = 99999 else: h1 = int(line.split(':')[-1]) if 'h2:' in line: if "inf" in line: h2 = 99999 else: h2 = int(line.split(':')[-1]) if 'plan:' in line: plan = (line.split(':')[-1]).strip() if 'Goal:' in line: goal = (line.split(':')[-1]).strip() if '****' in line: data.append("%s,%s,ok,%f,%d,%d,%d,%d,%d,%d,%d,%s" % (prob, goal, time, generated, pruned, expanded, width, quality, h1, h2, plan)) quality = 0 generated = 0 expanded = 0 pruned = 0 width = 0 h2 = 0 h1 = 0 time = 0.0 plan = "" goal = "" os.system("rm " + res.output_file) os.system("rm %s.err" % res.output_file) data.sort() data = [ 'problem,status,goal,runtime,generated,pruned,expanded,width,quality,h1,h2,plan' ] + data write_file("%s/%s.csv" % (results_directory, dom), data)
def benchmark_domain(planner, dom, extra_args, lmcut=False): from krrt.utils import get_value, match_value, run_experiment, write_file, get_lines print "Benchmarking %s..." % dom global ipc if lmcut and '../' != ipc[:3]: ipc = '../' + ipc if TYPE == OLD: domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]] elif TYPE == NEW: #domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] domprob_args = ["%s/%s/%s %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) sand = None if lmcut: sand = 'lmcut-'+dom results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, clean_sandbox=True, sandbox=sand) data = [] for resid in results.get_ids(): res = results[resid] if lmcut: prob = res.single_args['domprob'].split()[1].split('/')[-1] elif TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-4].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file os.system("cp %s %s"%(outfile, outfile+dom)) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*This configuration does not support axioms.*'): data.append("%s,ax,-1,-1,-1,-1" % prob) else: if match_value(outfile, '.*Plan found with cost: ([0-9]+).*'): quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int) generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int) expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int) data.append("%s,ok,%.2f,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded)) else: print "Error with %s" % prob data.append("%s,err,%.2f,-1,-1,-1" % (prob, res.runtime)) if 'old-' in planner: cmd("rm TMP_OUTPUT") data.sort() header = ['problem,status,runtime,quality,generated,expanded'] data = header + data write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom): from krrt.utils import get_value, match_value, run_experiment, write_file print print "Benchmarking %s..." % dom if TYPE == OLD: domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] elif TYPE == NEW: domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, sandbox=None) data = [] for resid in results.get_ids(): res = results[resid] if TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file os.system("cp %s %s"%(outfile, outfile+dom)) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) else: if match_value(outfile, '.*Effective Width during search*'): lines = open(outfile) quality = 0 generated = 0 expanded = 0 width = 0 for line in lines: if 'Plan found with cost:' in line: quality = int(line.split(':')[-1]) if 'Nodes generated during search:' in line: generated = int(line.split(':')[-1]) if 'Nodes expanded during search:' in line: expanded = int(line.split(':')[-1]) if 'Effective Width during search:' in line: width = int(line.split(':')[-1]) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, width)) elif match_value(outfile, '([0-9]+).* .*actions in the plan.'): quality = get_value(outfile, '([0-9]+).* .*actions in the plan.', int) generated = 0 expanded = 0 backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'): quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int) generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int) expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Length : ([0-9]+).*'): quality = get_value(outfile, '.*Length : ([0-9]+).*', int) generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*', int) expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*'): quality = get_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*', float) generated = get_value(outfile, '.*Generated: ([0-9]+).*', int) expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*NOT I-REACHABLE.*'): backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,not-i,%f,-1,-1,-1,%d" % (prob, res.runtime, backtracks)) else: print "Error with %s" % prob data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime)) if 'old-' in planner: cmd("rm TMP_OUTPUT") data.sort() data = ['problem,status,runtime,quality,generated,expanded'] + data write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom): from krrt.utils import get_value, match_value, run_experiment, write_file print print "Benchmarking %s..." % dom if TYPE == OLD: domprob_args = [ "--domain %s/%s/%s --problem %s/%s/%s" % (ipc, dom, domain, ipc, dom, problem) for (domain, problem) in benchmark[dom] ] elif TYPE == NEW: domprob_args = [ "%s/%s/%s %s/%s/%s o/dev/null" % (ipc, dom, domain, ipc, dom, problem) for (domain, problem) in benchmark[dom] ] else: assert False, "What the deuce?" if os.path.exists(results_directory) is False: os.mkdir(results_directory) results = run_experiment(base_directory=".", base_command=planner, single_arguments={'domprob': domprob_args}, time_limit=timelimit, memory_limit=memorylimit, results_dir=results_directory, progress_file=None, processors=cores, sandbox=None) data = [] for resid in results.get_ids(): res = results[resid] if TYPE == OLD: prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1] elif TYPE == NEW: prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1] else: assert False, "What the deuce?" if 'old-' in planner: cmd("tail -26 %s > TMP_OUTPUT" % res.output_file) outfile = "TMP_OUTPUT" else: outfile = res.output_file os.system("cp %s %s" % (outfile, outfile + dom)) if res.timed_out: data.append("%s,time,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*MemoryError.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'): data.append("%s,mem,-1,-1,-1,-1" % prob) elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'): data.append("%s,seg,-1,-1,-1,-1" % prob) else: if match_value(outfile, '.*Effective Width during search*'): lines = open(outfile) quality = 0 generated = 0 expanded = 0 width = 0 for line in lines: if 'Plan found with cost:' in line: quality = int(line.split(':')[-1]) if 'Nodes generated during search:' in line: generated = int(line.split(':')[-1]) if 'Nodes expanded during search:' in line: expanded = int(line.split(':')[-1]) if 'Effective Width during search:' in line: width = int(line.split(':')[-1]) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, width)) elif match_value(outfile, '([0-9]+).* .*actions in the plan.'): quality = get_value(outfile, '([0-9]+).* .*actions in the plan.', int) generated = 0 expanded = 0 backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'): quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int) generated = get_value( outfile, '.*Nodes generated during search: ([0-9]+).*', int) expanded = get_value( outfile, '.*Nodes expanded during search: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Length : ([0-9]+).*'): quality = get_value(outfile, '.*Length : ([0-9]+).*', int) generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*', int) expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*'): quality = get_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*', float) generated = get_value(outfile, '.*Generated: ([0-9]+).*', int) expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int) backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks)) elif match_value(outfile, '.*NOT I-REACHABLE.*'): backtracks = 0 if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'): backtracks = get_value( outfile, '.*Backtracks during search: ([0-9]+).*', int) data.append("%s,not-i,%f,-1,-1,-1,%d" % (prob, res.runtime, backtracks)) else: print "Error with %s" % prob data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime)) if 'old-' in planner: cmd("rm TMP_OUTPUT") data.sort() data = ['problem,status,runtime,quality,generated,expanded'] + data write_file("%s/%s.csv" % (results_directory, dom), data)
def doit_prp(domain, dom_probs, prp_params, exp_name='prp'): print "\n\nRunning %s experiments on domain, %s" % (exp_name, domain) prp_args = ["../%s ../%s" % (item[0], item[1]) for item in dom_probs] prp_results = run_experiment( base_command='./../../src/prp', single_arguments={'domprob': prp_args}, parameters=prp_params, time_limit=TIME_LIMIT, memory_limit=MEM_LIMIT, results_dir="RESULTS/%s-%s" % (exp_name, domain), progress_file=None, processors=CORES, sandbox=exp_name, clean_sandbox=True, trials=TRIALS, output_file_func=(lambda res: res.single_args['domprob'].split(' ')[1]. split('/')[-1] + '.' + str(res.id) + '.out'), error_file_func=(lambda res: res.single_args['domprob'].split(' ')[1]. split('/')[-1] + '.' + str(res.id) + '.err')) timeouts = 0 memouts = 0 errorouts = 0 parametererrors = 0 prp_csv = [ 'domain,problem,runtime,size,status,%s,jic time,policy eval,policy creation,policy use,search time,engine time,regression time,simulator time,successful states,replans,actions,policy score,strongly cyclic,succeeded' % ','.join(PARAMETERS) ] for res_id in prp_results.get_ids(): result = prp_results[res_id] prob = result.single_args['domprob'].split(' ')[1].split('/')[-1] if match_value(result.output_file, 'No solution -- aborting repairs.'): prp_csv.append("%s,%s,-1,-1,N,%s,%s" % (domain, prob, parse_prp_settings(result), ','.join( ['-'] * 14))) else: if result.mem_out or check_memout(result.output_file): memouts += 1 prp_csv.append("%s,%s,-1,-1,M,%s,%s" % (domain, prob, parse_prp_settings(result), ','.join(['-'] * 14))) elif result.timed_out: timeouts += 1 prp_csv.append("%s,%s,-1,-1,T,%s,%s" % (domain, prob, parse_prp_settings(result), ','.join(['-'] * 14))) #elif not result.clean_run or check_segfault(result.output_file): elif check_segfault(result.output_file): errorouts += 1 prp_csv.append("%s,%s,-1,-1,E,%s,%s" % (domain, prob, parse_prp_settings(result), ','.join(['-'] * 14))) elif match_value(result.output_file, '.*Parameter Error.*'): parametererrors += 1 prp_csv.append("%s,%s,-1,-1,P,%s,%s" % (domain, prob, parse_prp_settings(result), ','.join(['-'] * 14))) else: runtime, jic_time, policy_eval_time, policy_construction_time, policy_use_time, \ search_time, engine_init_time, regression_time, simulator_time, successful_states, \ replans, actions, size, strongly_cyclic, succeeded, policy_score = parse_prp(result.output_file) prp_csv.append( "%s,%s,%f,%d,-,%s,%f,%f,%f,%f,%f,%f,%f,%f,%d,%d,%d,%f,%s,%s" % (domain, prob, runtime, size, parse_prp_settings(result), jic_time, policy_eval_time, policy_construction_time, policy_use_time, search_time, engine_init_time, regression_time, simulator_time, successful_states, replans, actions, policy_score, str(strongly_cyclic), str(succeeded))) print "\nTimed out %d times." % timeouts print "Ran out of memory %d times." % memouts print "Unknown error %d times." % errorouts print "Invalid parameter settings %d times." % parametererrors append_file("RESULTS/%s-%s-results.csv" % (exp_name, domain), prp_csv)
def check_memout(outfile): return match_value(outfile, '.*Memory limit has been reached.*')
def check_segfault(outfile): return match_value(outfile, '.*Segmentation fault.*')