def doit_fip(domain, dom_probs, exp_name='fip'):
print "\n\nRunning FIP experiments on domain, %s" % domain
fip_args = [
"-o ../%s.fip -f ../%s" % (item[0], item[1]) for item in dom_probs
]
fip_results = run_experiment(
base_command='./../fip',
single_arguments={'domprob': fip_args},
time_limit=TIME_LIMIT,
memory_limit=MEM_LIMIT,
results_dir="RESULTS/%s-%s" % (exp_name, domain),
progress_file=None,
processors=CORES,
sandbox=exp_name,
clean_sandbox=True,
output_file_func=(
lambda res: res.single_args['domprob'].split('/')[-1] + '.out'),
error_file_func=(
lambda res: res.single_args['domprob'].split('/')[-1] + '.err'))
timeouts = 0
memouts = 0
errorouts = 0
fip_csv = ['domain,problem,runtime,size,status']
for res_id in fip_results.get_ids():
result = fip_results[res_id]
prob = result.single_args['domprob'].split(' ')[3].split('/')[-1]
if match_value(result.output_file, 'No plan will solve it'
) or 'No solutions are found!' in read_file(
result.output_file)[-1]:
fip_csv.append("%s,%s,-1,-1,N" % (domain, prob))
else:
if result.timed_out:
timeouts += 1
fip_csv.append("%s,%s,-1,-1,T" % (domain, prob))
elif result.mem_out:
memouts += 1
fip_csv.append("%s,%s,-1,-1,M" % (domain, prob))
elif not result.clean_run or check_segfault(result.output_file):
errorouts += 1
fip_csv.append("%s,%s,-1,-1,E" % (domain, prob))
else:
run, size = parse_fip(result.output_file)
fip_csv.append("%s,%s,%f,%d,-" % (domain, prob, run, size))
print "\nTimed out %d times." % timeouts
print "Ran out of memory %d times." % memouts
print "Unknown error %d times." % errorouts
append_file("RESULTS/%s-%s-results.csv" % (exp_name, domain), fip_csv)
def parse_output_FF(file_name):
from krrt.utils import match_value, get_lines
# Check for the failed solution
if match_value(file_name, '.* No plan will solve it.*'):
print "No solution."
return None
# Get the plan
ff_actions = get_lines(file_name,
lower_bound='.*found legal plan.*',
upper_bound='.*time spent.*')
ff_actions = filter(lambda x: x != '',
[item.strip(' ') for item in ff_actions])
ff_actions = [item.split(':')[1].strip(' ').lower() for item in ff_actions]
actions = [Action(item) for item in ff_actions]
return Plan(actions)
def parse_prp(outfile):
runtime = get_value(outfile, '.*Total time: ([0-9]+\.?[0-9]*)s\n.*', float)
jic_time = get_value(outfile,
'.*Just-in-case Repairs: ([0-9]+\.?[0-9]*)s\n.*',
float)
policy_use_time = get_value(outfile,
'.*Using the policy: ([0-9]+\.?[0-9]*)s\n.*',
float)
policy_construction_time = get_value(
outfile, '.*Policy Construction: ([0-9]+\.?[0-9]*)s\n.*', float)
policy_eval_time = get_value(
outfile, '.*Evaluating the policy quality: ([0-9]+\.?[0-9]*)s\n.*',
float)
search_time = get_value(outfile, '.*Search Time: ([0-9]+\.?[0-9]*)s\n.*',
float)
engine_init_time = get_value(
outfile, '.*Engine Initialization: ([0-9]+\.?[0-9]*)s\n.*', float)
regression_time = get_value(
outfile, '.*Regression Computation: ([0-9]+\.?[0-9]*)s\n.*', float)
simulator_time = get_value(outfile,
'.*Simulator time: ([0-9]+\.?[0-9]*)s\n.*',
float)
successful_states = get_value(
outfile, '.*Successful states: ([0-9]+\.?[0-9]*) \+.*', float)
replans = get_value(outfile, '.*Replans: ([0-9]+\.?[0-9]*) \+.*', float)
actions = get_value(outfile, '.*Actions: ([0-9]+\.?[0-9]*) \+.*', float)
size = get_value(outfile, '.*State-Action Pairs: (\d+)\n.*', int)
strongly_cyclic = match_value(outfile, '.*Strongly Cyclic: True.*')
succeeded = get_value(outfile, '.*Succeeded: (\d+) .*', int)
policy_score = get_value(outfile, '.*Policy Score: ([0-9]+\.?[0-9]*)\n.*',
float)
return runtime, jic_time, policy_eval_time, policy_construction_time, policy_use_time, \
search_time, engine_init_time, regression_time, simulator_time, successful_states, \
replans, actions, size, strongly_cyclic, succeeded, policy_score
def benchmark_domain(planner, bound, dom):
from krrt.utils import get_value, match_value, run_experiment, write_file
print
print "Benchmarking %s..." % dom
if TYPE == OLD:
domprob_args = [" --bound %s --domain %s/%s/%s --problem %s/%s/%s" % (bound,ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
elif TYPE == NEW:
domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
sandbox=None)
data = []
for resid in results.get_ids():
res = results[resid]
if TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
path, filename = os.path.split(outfile)
os.system("cp %s %s"%(outfile, path+"/"+dom+"_"+prob+".log"))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
lines = open(outfile)
quality = 0
generated = 0
expanded = 0
pruned = 0
width = 0
h2=0
h1=0
time=0.0
plan=""
goal=""
for line in lines:
if 'Plan found with cost:' in line:
quality = int(line.split(':')[-1])
if 'Nodes generated during search:' in line:
generated = int(line.split(':')[-1])
if 'Nodes expanded during search:' in line:
expanded = int(line.split(':')[-1])
if 'Nodes pruned by bound:' in line:
pruned = int(line.split(':')[-1])
if 'Effective width:' in line:
width = int(line.split(':')[-1])
if 'Total time:' in line:
time = float(line.split(':')[-1])
if 'h1:' in line:
if "inf" in line:
h1 = 99999
else:
h1 = int(line.split(':')[-1])
if 'h2:' in line:
if "inf" in line:
h2 = 99999
else:
h2 = int(line.split(':')[-1])
if 'plan:' in line:
plan = (line.split(':')[-1]).strip()
if 'Goal:' in line:
goal = (line.split(':')[-1]).strip()
if '****' in line:
data.append("%s,%s,ok,%f,%d,%d,%d,%d,%d,%d,%d,%s" % (prob, goal, time, generated, pruned, expanded, width, quality,h1,h2,plan))
quality = 0
generated = 0
expanded = 0
pruned = 0
width = 0
h2=0
h1=0
time=0.0
plan=""
goal=""
os.system("rm "+res.output_file)
os.system("rm %s.err"%res.output_file)
data.sort()
data = ['problem,status,goal,runtime,generated,pruned,expanded,width,quality,h1,h2,plan'] + data
write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, bound, dom):
from krrt.utils import get_value, match_value, run_experiment, write_file
print
print "Benchmarking %s..." % dom
if TYPE == OLD:
domprob_args = [
" --bound %s --domain %s/%s/%s --problem %s/%s/%s" %
(bound, ipc, dom, domain, ipc, dom, problem)
for (domain, problem) in benchmark[dom]
]
elif TYPE == NEW:
domprob_args = [
"%s/%s/%s %s/%s/%s o/dev/null" %
(ipc, dom, domain, ipc, dom, problem)
for (domain, problem) in benchmark[dom]
]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
sandbox=None)
data = []
for resid in results.get_ids():
res = results[resid]
if TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
path, filename = os.path.split(outfile)
os.system("cp %s %s" %
(outfile, path + "/" + dom + "_" + prob + ".log"))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file,
'.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
lines = open(outfile)
quality = 0
generated = 0
expanded = 0
pruned = 0
width = 0
h2 = 0
h1 = 0
time = 0.0
plan = ""
goal = ""
for line in lines:
if 'Plan found with cost:' in line:
quality = int(line.split(':')[-1])
if 'Nodes generated during search:' in line:
generated = int(line.split(':')[-1])
if 'Nodes expanded during search:' in line:
expanded = int(line.split(':')[-1])
if 'Nodes pruned by bound:' in line:
pruned = int(line.split(':')[-1])
if 'Effective width:' in line:
width = int(line.split(':')[-1])
if 'Total time:' in line:
time = float(line.split(':')[-1])
if 'h1:' in line:
if "inf" in line:
h1 = 99999
else:
h1 = int(line.split(':')[-1])
if 'h2:' in line:
if "inf" in line:
h2 = 99999
else:
h2 = int(line.split(':')[-1])
if 'plan:' in line:
plan = (line.split(':')[-1]).strip()
if 'Goal:' in line:
goal = (line.split(':')[-1]).strip()
if '****' in line:
data.append("%s,%s,ok,%f,%d,%d,%d,%d,%d,%d,%d,%s" %
(prob, goal, time, generated, pruned, expanded,
width, quality, h1, h2, plan))
quality = 0
generated = 0
expanded = 0
pruned = 0
width = 0
h2 = 0
h1 = 0
time = 0.0
plan = ""
goal = ""
os.system("rm " + res.output_file)
os.system("rm %s.err" % res.output_file)
data.sort()
data = [
'problem,status,goal,runtime,generated,pruned,expanded,width,quality,h1,h2,plan'
] + data
write_file("%s/%s.csv" % (results_directory, dom), data)
def benchmark_domain(planner, dom, extra_args, lmcut=False):
from krrt.utils import get_value, match_value, run_experiment, write_file, get_lines
print "Benchmarking %s..." % dom
global ipc
if lmcut and '../' != ipc[:3]:
ipc = '../' + ipc
if TYPE == OLD:
domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]]
elif TYPE == NEW:
#domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
domprob_args = ["%s/%s/%s %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
sand = None
if lmcut:
sand = 'lmcut-'+dom
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
clean_sandbox=True,
sandbox=sand)
data = []
for resid in results.get_ids():
res = results[resid]
if lmcut:
prob = res.single_args['domprob'].split()[1].split('/')[-1]
elif TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-4].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
os.system("cp %s %s"%(outfile, outfile+dom))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*This configuration does not support axioms.*'):
data.append("%s,ax,-1,-1,-1,-1" % prob)
else:
if match_value(outfile, '.*Plan found with cost: ([0-9]+).*'):
quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int)
generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int)
data.append("%s,ok,%.2f,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded))
else:
print "Error with %s" % prob
data.append("%s,err,%.2f,-1,-1,-1" % (prob, res.runtime))
if 'old-' in planner:
cmd("rm TMP_OUTPUT")
data.sort()
header = ['problem,status,runtime,quality,generated,expanded']
data = header + data
write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom):
from krrt.utils import get_value, match_value, run_experiment, write_file
print
print "Benchmarking %s..." % dom
if TYPE == OLD:
domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
elif TYPE == NEW:
domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
sandbox=None)
data = []
for resid in results.get_ids():
res = results[resid]
if TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
os.system("cp %s %s"%(outfile, outfile+dom))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
else:
if match_value(outfile, '.*Effective Width during search*'):
lines = open(outfile)
quality = 0
generated = 0
expanded = 0
width = 0
for line in lines:
if 'Plan found with cost:' in line:
quality = int(line.split(':')[-1])
if 'Nodes generated during search:' in line:
generated = int(line.split(':')[-1])
if 'Nodes expanded during search:' in line:
expanded = int(line.split(':')[-1])
if 'Effective Width during search:' in line:
width = int(line.split(':')[-1])
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, width))
elif match_value(outfile, '([0-9]+).* .*actions in the plan.'):
quality = get_value(outfile, '([0-9]+).* .*actions in the plan.', int)
generated = 0
expanded = 0
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'):
quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int)
generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*Length : ([0-9]+).*'):
quality = get_value(outfile, '.*Length : ([0-9]+).*', int)
generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*', int)
expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*', int)
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*'):
quality = get_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*', float)
generated = get_value(outfile, '.*Generated: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*NOT I-REACHABLE.*'):
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,not-i,%f,-1,-1,-1,%d" % (prob, res.runtime, backtracks))
else:
print "Error with %s" % prob
data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime))
if 'old-' in planner:
cmd("rm TMP_OUTPUT")
data.sort()
data = ['problem,status,runtime,quality,generated,expanded'] + data
write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom):
from krrt.utils import get_value, match_value, run_experiment, write_file
print
print "Benchmarking %s..." % dom
if TYPE == OLD:
domprob_args = [
"--domain %s/%s/%s --problem %s/%s/%s" %
(ipc, dom, domain, ipc, dom, problem)
for (domain, problem) in benchmark[dom]
]
elif TYPE == NEW:
domprob_args = [
"%s/%s/%s %s/%s/%s o/dev/null" %
(ipc, dom, domain, ipc, dom, problem)
for (domain, problem) in benchmark[dom]
]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
sandbox=None)
data = []
for resid in results.get_ids():
res = results[resid]
if TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
os.system("cp %s %s" % (outfile, outfile + dom))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file,
'.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
else:
if match_value(outfile, '.*Effective Width during search*'):
lines = open(outfile)
quality = 0
generated = 0
expanded = 0
width = 0
for line in lines:
if 'Plan found with cost:' in line:
quality = int(line.split(':')[-1])
if 'Nodes generated during search:' in line:
generated = int(line.split(':')[-1])
if 'Nodes expanded during search:' in line:
expanded = int(line.split(':')[-1])
if 'Effective Width during search:' in line:
width = int(line.split(':')[-1])
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated,
expanded, width))
elif match_value(outfile, '([0-9]+).* .*actions in the plan.'):
quality = get_value(outfile,
'([0-9]+).* .*actions in the plan.', int)
generated = 0
expanded = 0
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'):
quality = get_value(outfile,
'.*Plan found with cost: ([0-9]+).*', int)
generated = get_value(
outfile, '.*Nodes generated during search: ([0-9]+).*',
int)
expanded = get_value(
outfile, '.*Nodes expanded during search: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile, '.*Length : ([0-9]+).*'):
quality = get_value(outfile, '.*Length : ([0-9]+).*', int)
generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*',
int)
expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*',
int)
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile,
'.*Plan cost: ([0-9]+\.[0-9]+), steps.*'):
quality = get_value(outfile,
'.*Plan cost: ([0-9]+\.[0-9]+), steps.*',
float)
generated = get_value(outfile, '.*Generated: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile, '.*NOT I-REACHABLE.*'):
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,not-i,%f,-1,-1,-1,%d" %
(prob, res.runtime, backtracks))
else:
print "Error with %s" % prob
data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime))
if 'old-' in planner:
cmd("rm TMP_OUTPUT")
data.sort()
data = ['problem,status,runtime,quality,generated,expanded'] + data
write_file("%s/%s.csv" % (results_directory, dom), data)
def doit_prp(domain, dom_probs, prp_params, exp_name='prp'):
print "\n\nRunning %s experiments on domain, %s" % (exp_name, domain)
prp_args = ["../%s ../%s" % (item[0], item[1]) for item in dom_probs]
prp_results = run_experiment(
base_command='./../../src/prp',
single_arguments={'domprob': prp_args},
parameters=prp_params,
time_limit=TIME_LIMIT,
memory_limit=MEM_LIMIT,
results_dir="RESULTS/%s-%s" % (exp_name, domain),
progress_file=None,
processors=CORES,
sandbox=exp_name,
clean_sandbox=True,
trials=TRIALS,
output_file_func=(lambda res: res.single_args['domprob'].split(' ')[1].
split('/')[-1] + '.' + str(res.id) + '.out'),
error_file_func=(lambda res: res.single_args['domprob'].split(' ')[1].
split('/')[-1] + '.' + str(res.id) + '.err'))
timeouts = 0
memouts = 0
errorouts = 0
parametererrors = 0
prp_csv = [
'domain,problem,runtime,size,status,%s,jic time,policy eval,policy creation,policy use,search time,engine time,regression time,simulator time,successful states,replans,actions,policy score,strongly cyclic,succeeded'
% ','.join(PARAMETERS)
]
for res_id in prp_results.get_ids():
result = prp_results[res_id]
prob = result.single_args['domprob'].split(' ')[1].split('/')[-1]
if match_value(result.output_file, 'No solution -- aborting repairs.'):
prp_csv.append("%s,%s,-1,-1,N,%s,%s" %
(domain, prob, parse_prp_settings(result), ','.join(
['-'] * 14)))
else:
if result.mem_out or check_memout(result.output_file):
memouts += 1
prp_csv.append("%s,%s,-1,-1,M,%s,%s" %
(domain, prob, parse_prp_settings(result),
','.join(['-'] * 14)))
elif result.timed_out:
timeouts += 1
prp_csv.append("%s,%s,-1,-1,T,%s,%s" %
(domain, prob, parse_prp_settings(result),
','.join(['-'] * 14)))
#elif not result.clean_run or check_segfault(result.output_file):
elif check_segfault(result.output_file):
errorouts += 1
prp_csv.append("%s,%s,-1,-1,E,%s,%s" %
(domain, prob, parse_prp_settings(result),
','.join(['-'] * 14)))
elif match_value(result.output_file, '.*Parameter Error.*'):
parametererrors += 1
prp_csv.append("%s,%s,-1,-1,P,%s,%s" %
(domain, prob, parse_prp_settings(result),
','.join(['-'] * 14)))
else:
runtime, jic_time, policy_eval_time, policy_construction_time, policy_use_time, \
search_time, engine_init_time, regression_time, simulator_time, successful_states, \
replans, actions, size, strongly_cyclic, succeeded, policy_score = parse_prp(result.output_file)
prp_csv.append(
"%s,%s,%f,%d,-,%s,%f,%f,%f,%f,%f,%f,%f,%f,%d,%d,%d,%f,%s,%s"
% (domain, prob, runtime, size, parse_prp_settings(result),
jic_time, policy_eval_time, policy_construction_time,
policy_use_time, search_time, engine_init_time,
regression_time, simulator_time, successful_states,
replans, actions, policy_score, str(strongly_cyclic),
str(succeeded)))
print "\nTimed out %d times." % timeouts
print "Ran out of memory %d times." % memouts
print "Unknown error %d times." % errorouts
print "Invalid parameter settings %d times." % parametererrors
append_file("RESULTS/%s-%s-results.csv" % (exp_name, domain), prp_csv)
def check_memout(outfile):
return match_value(outfile, '.*Memory limit has been reached.*')
def check_segfault(outfile):
return match_value(outfile, '.*Segmentation fault.*')
