def dojic(dom, prob, max_jic):
print "Running a varied jic experiment for the following domain / problem:"
print dom
print prob
exp_name = "jic"
domain = dom.split('/')[-2]
PRP_PARAMS['jic']['--jic-limit'] = list(range(1, max_jic + 1))
prp_results = run_experiment(
base_command='./../../src/prp',
single_arguments={'domprob': ["../%s ../%s" % (dom, prob)]},
parameters=PRP_PARAMS['jic'],
time_limit=TIME_LIMIT,
memory_limit=MEM_LIMIT,
results_dir="RESULTS/%s-%s" % (exp_name, domain),
progress_file=None,
processors=CORES,
sandbox=exp_name,
clean_sandbox=True,
trials=TRIALS,
output_file_func=(lambda res: res.single_args['domprob'].split(' ')[1].
split('/')[-1] + '.' + str(res.id) + '.out'),
error_file_func=(lambda res: res.single_args['domprob'].split(' ')[1].
split('/')[-1] + '.' + str(res.id) + '.err'))
prp_csv = ['jic limit,jic time,policy score']
for res_id in prp_results.get_ids():
result = prp_results[res_id]
jic_lim = result.parameters['--jic-limit']
jic_time = get_value(result.output_file,
'.*Just-in-case Repairs: ([0-9]+\.?[0-9]*)s\n.*',
float)
policy_score = get_value(result.output_file,
'.*Policy Score: ([0-9]+\.?[0-9]*)\n.*',
float)
prp_csv.append("%s,%f,%f" % (jic_lim, jic_time, policy_score))
append_file("RESULTS/%s-%s-results.csv" % (exp_name, domain), prp_csv)
]
results = run_experiment(
base_directory=".",
base_command='./../../src/plan-prp',
single_arguments={'domprob': domprobs},
time_limit=1800, # 15minute time limit (900 seconds)
memory_limit=1000, # 1gig memory limit (1000 megs)
results_dir="%s/results" % domain,
progress_file=None, # Print the progress to stdout
processors=6, # You've got 8 cores, right?
sandbox='fd_out',
clean_sandbox=False)
good_results = results.filter(lambda result: not result.timed_out)
good_results = [good_results[i] for i in good_results.get_ids()]
print "Coverage: %d / %d" % (len(good_results), len(domprobs))
try:
data = ['runtime(s),size(nodes)'] + [
"%f,%d" %
(res.runtime,
get_value(res.output_file, '.*State-Action Pairs: (\d+)\n.*', int))
for res in good_results
]
except:
pass
write_file("%s.csv" % domain, data)
def benchmark_domain(planner, dom, extra_args, lmcut=False):
from krrt.utils import get_value, match_value, run_experiment, write_file, get_lines
print "Benchmarking %s..." % dom
global ipc
if lmcut and '../' != ipc[:3]:
ipc = '../' + ipc
if TYPE == OLD:
domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]]
elif TYPE == NEW:
#domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
domprob_args = ["%s/%s/%s %s/%s/%s %s" % (ipc,dom,domain,ipc,dom,problem,extra_args) for (domain, problem) in benchmark[dom]]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
sand = None
if lmcut:
sand = 'lmcut-'+dom
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
clean_sandbox=True,
sandbox=sand)
data = []
for resid in results.get_ids():
res = results[resid]
if lmcut:
prob = res.single_args['domprob'].split()[1].split('/')[-1]
elif TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-4].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
os.system("cp %s %s"%(outfile, outfile+dom))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*This configuration does not support axioms.*'):
data.append("%s,ax,-1,-1,-1,-1" % prob)
else:
if match_value(outfile, '.*Plan found with cost: ([0-9]+).*'):
quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int)
generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int)
data.append("%s,ok,%.2f,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded))
else:
print "Error with %s" % prob
data.append("%s,err,%.2f,-1,-1,-1" % (prob, res.runtime))
if 'old-' in planner:
cmd("rm TMP_OUTPUT")
data.sort()
header = ['problem,status,runtime,quality,generated,expanded']
data = header + data
write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom):
from krrt.utils import get_value, match_value, run_experiment, write_file
print
print "Benchmarking %s..." % dom
if TYPE == OLD:
domprob_args = ["--domain %s/%s/%s --problem %s/%s/%s" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
elif TYPE == NEW:
domprob_args = ["%s/%s/%s %s/%s/%s o/dev/null" % (ipc,dom,domain,ipc,dom,problem) for (domain, problem) in benchmark[dom]]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
sandbox=None)
data = []
for resid in results.get_ids():
res = results[resid]
if TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
os.system("cp %s %s"%(outfile, outfile+dom))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
else:
if match_value(outfile, '.*Effective Width during search*'):
lines = open(outfile)
quality = 0
generated = 0
expanded = 0
width = 0
for line in lines:
if 'Plan found with cost:' in line:
quality = int(line.split(':')[-1])
if 'Nodes generated during search:' in line:
generated = int(line.split(':')[-1])
if 'Nodes expanded during search:' in line:
expanded = int(line.split(':')[-1])
if 'Effective Width during search:' in line:
width = int(line.split(':')[-1])
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, width))
elif match_value(outfile, '([0-9]+).* .*actions in the plan.'):
quality = get_value(outfile, '([0-9]+).* .*actions in the plan.', int)
generated = 0
expanded = 0
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'):
quality = get_value(outfile, '.*Plan found with cost: ([0-9]+).*', int)
generated = get_value(outfile, '.*Nodes generated during search: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Nodes expanded during search: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*Length : ([0-9]+).*'):
quality = get_value(outfile, '.*Length : ([0-9]+).*', int)
generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*', int)
expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*', int)
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*'):
quality = get_value(outfile, '.*Plan cost: ([0-9]+\.[0-9]+), steps.*', float)
generated = get_value(outfile, '.*Generated: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" % (prob, res.runtime, quality, generated, expanded, backtracks))
elif match_value(outfile, '.*NOT I-REACHABLE.*'):
backtracks = 0
if match_value(outfile, '.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,not-i,%f,-1,-1,-1,%d" % (prob, res.runtime, backtracks))
else:
print "Error with %s" % prob
data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime))
if 'old-' in planner:
cmd("rm TMP_OUTPUT")
data.sort()
data = ['problem,status,runtime,quality,generated,expanded'] + data
write_file("%s/%s.csv" %( results_directory, dom), data)
def benchmark_domain(planner, dom):
from krrt.utils import get_value, match_value, run_experiment, write_file
print
print "Benchmarking %s..." % dom
if TYPE == OLD:
domprob_args = [
"--domain %s/%s/%s --problem %s/%s/%s" %
(ipc, dom, domain, ipc, dom, problem)
for (domain, problem) in benchmark[dom]
]
elif TYPE == NEW:
domprob_args = [
"%s/%s/%s %s/%s/%s o/dev/null" %
(ipc, dom, domain, ipc, dom, problem)
for (domain, problem) in benchmark[dom]
]
else:
assert False, "What the deuce?"
if os.path.exists(results_directory) is False:
os.mkdir(results_directory)
results = run_experiment(base_directory=".",
base_command=planner,
single_arguments={'domprob': domprob_args},
time_limit=timelimit,
memory_limit=memorylimit,
results_dir=results_directory,
progress_file=None,
processors=cores,
sandbox=None)
data = []
for resid in results.get_ids():
res = results[resid]
if TYPE == OLD:
prob = res.single_args['domprob'].split(' ')[-1].split('/')[-1]
elif TYPE == NEW:
prob = res.single_args['domprob'].split(' ')[-2].split('/')[-1]
else:
assert False, "What the deuce?"
if 'old-' in planner:
cmd("tail -26 %s > TMP_OUTPUT" % res.output_file)
outfile = "TMP_OUTPUT"
else:
outfile = res.output_file
os.system("cp %s %s" % (outfile, outfile + dom))
if res.timed_out:
data.append("%s,time,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*std::bad_alloc.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*MemoryError.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file,
'.*cannot allocate memory.*'):
data.append("%s,mem,-1,-1,-1,-1" % prob)
elif match_value("%s.err" % res.output_file, '.*Segmentation fault.*'):
data.append("%s,seg,-1,-1,-1,-1" % prob)
else:
if match_value(outfile, '.*Effective Width during search*'):
lines = open(outfile)
quality = 0
generated = 0
expanded = 0
width = 0
for line in lines:
if 'Plan found with cost:' in line:
quality = int(line.split(':')[-1])
if 'Nodes generated during search:' in line:
generated = int(line.split(':')[-1])
if 'Nodes expanded during search:' in line:
expanded = int(line.split(':')[-1])
if 'Effective Width during search:' in line:
width = int(line.split(':')[-1])
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated,
expanded, width))
elif match_value(outfile, '([0-9]+).* .*actions in the plan.'):
quality = get_value(outfile,
'([0-9]+).* .*actions in the plan.', int)
generated = 0
expanded = 0
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile, '.*Plan found with cost: ([0-9]+).*'):
quality = get_value(outfile,
'.*Plan found with cost: ([0-9]+).*', int)
generated = get_value(
outfile, '.*Nodes generated during search: ([0-9]+).*',
int)
expanded = get_value(
outfile, '.*Nodes expanded during search: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile, '.*Length : ([0-9]+).*'):
quality = get_value(outfile, '.*Length : ([0-9]+).*', int)
generated = get_value(outfile, '.*Expanded nodes : ([0-9]+).*',
int)
expanded = get_value(outfile, '.*Evaluated nodes : ([0-9]+).*',
int)
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile,
'.*Plan cost: ([0-9]+\.[0-9]+), steps.*'):
quality = get_value(outfile,
'.*Plan cost: ([0-9]+\.[0-9]+), steps.*',
float)
generated = get_value(outfile, '.*Generated: ([0-9]+).*', int)
expanded = get_value(outfile, '.*Expanded: ([0-9]+).*', int)
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,ok,%f,%d,%d,%d,%d" %
(prob, res.runtime, quality, generated, expanded,
backtracks))
elif match_value(outfile, '.*NOT I-REACHABLE.*'):
backtracks = 0
if match_value(outfile,
'.*Backtracks during search: ([0-9]+).*'):
backtracks = get_value(
outfile, '.*Backtracks during search: ([0-9]+).*', int)
data.append("%s,not-i,%f,-1,-1,-1,%d" %
(prob, res.runtime, backtracks))
else:
print "Error with %s" % prob
data.append("%s,err,%f,-1,-1,-1" % (prob, res.runtime))
if 'old-' in planner:
cmd("rm TMP_OUTPUT")
data.sort()
data = ['problem,status,runtime,quality,generated,expanded'] + data
write_file("%s/%s.csv" % (results_directory, dom), data)
def parse_prp(outfile):
runtime = get_value(outfile, '.*Total time: ([0-9]+\.?[0-9]*)s\n.*', float)
jic_time = get_value(outfile,
'.*Just-in-case Repairs: ([0-9]+\.?[0-9]*)s\n.*',
float)
policy_use_time = get_value(outfile,
'.*Using the policy: ([0-9]+\.?[0-9]*)s\n.*',
float)
policy_construction_time = get_value(
outfile, '.*Policy Construction: ([0-9]+\.?[0-9]*)s\n.*', float)
policy_eval_time = get_value(
outfile, '.*Evaluating the policy quality: ([0-9]+\.?[0-9]*)s\n.*',
float)
search_time = get_value(outfile, '.*Search Time: ([0-9]+\.?[0-9]*)s\n.*',
float)
engine_init_time = get_value(
outfile, '.*Engine Initialization: ([0-9]+\.?[0-9]*)s\n.*', float)
regression_time = get_value(
outfile, '.*Regression Computation: ([0-9]+\.?[0-9]*)s\n.*', float)
simulator_time = get_value(outfile,
'.*Simulator time: ([0-9]+\.?[0-9]*)s\n.*',
float)
successful_states = get_value(
outfile, '.*Successful states: ([0-9]+\.?[0-9]*) \+.*', float)
replans = get_value(outfile, '.*Replans: ([0-9]+\.?[0-9]*) \+.*', float)
actions = get_value(outfile, '.*Actions: ([0-9]+\.?[0-9]*) \+.*', float)
size = get_value(outfile, '.*State-Action Pairs: (\d+)\n.*', int)
strongly_cyclic = match_value(outfile, '.*Strongly Cyclic: True.*')
succeeded = get_value(outfile, '.*Succeeded: (\d+) .*', int)
policy_score = get_value(outfile, '.*Policy Score: ([0-9]+\.?[0-9]*)\n.*',
float)
return runtime, jic_time, policy_eval_time, policy_construction_time, policy_use_time, \
search_time, engine_init_time, regression_time, simulator_time, successful_states, \
replans, actions, size, strongly_cyclic, succeeded, policy_score
def parse_fip(outfile):
runtime = get_value(outfile, '.* ([0-9]+\.?[0-9]+) seconds searching.*',
float)
policy_size = len(filter(lambda x: 'case S' in x, read_file(outfile)))
return runtime, policy_size