def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks')
suite = [
'airport',
'depot',
'driverlog',
'elevators-opt08-strips',
'elevators-opt11-strips',
'freecell',
'hiking-opt14-strips',
'pipesworld-tankage',
]
configs = {
IssueConfig(
'astar-blind-ssec',
['--search', 'astar(blind(), pruning=stubborn_sets_ec())']),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
exp()
def main(revisions=None):
benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks')
suite=suites.suite_optimal_strips()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-blind-sss', ['--search', 'astar(blind(), pruning=stubborn_sets_simple())']),
IssueConfig('astar-blind-ssec', ['--search', 'astar(blind(), pruning=stubborn_sets_ec())']),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())'],
driver_options=['--search-time-limit', '5m']),
}
exp = IssueExperiment(
benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks'),
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
attribute = "total_time"
config_nick = 'astar-blind'
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}".format(rev, config_nick) for rev in revisions],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(exp.name, attribute, config_nick)
)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
def main(revisions=None):
suite = SUITE_SAT14
configs = [
IssueConfig("astar_goalcount", ["--search", "astar(goalcount)"]),
IssueConfig("eager_greedy_ff", [
"--heuristic", "h=ff()", "--search", "eager_greedy(h, preferred=h)"
]),
IssueConfig("eager_greedy_add", [
"--heuristic", "h=add()", "--search",
"eager_greedy(h, preferred=h)"
]),
IssueConfig("eager_greedy_cg", [
"--heuristic", "h=cg()", "--search", "eager_greedy(h, preferred=h)"
]),
IssueConfig("eager_greedy_cea", [
"--heuristic", "h=cea()", "--search",
"eager_greedy(h, preferred=h)"
]),
IssueConfig("lazy_greedy_ff", [
"--heuristic", "h=ff()", "--search", "lazy_greedy(h, preferred=h)"
]),
IssueConfig("lazy_greedy_add", [
"--heuristic", "h=add()", "--search", "lazy_greedy(h, preferred=h)"
]),
IssueConfig("lazy_greedy_cg", [
"--heuristic", "h=cg()", "--search", "lazy_greedy(h, preferred=h)"
]),
IssueConfig("seq_sat_lama_2011", [],
driver_options=["--alias", "seq-sat-lama-2011"]),
IssueConfig("seq_sat_fdss_1", [],
driver_options=["--alias", "seq-sat-fdss-1"]),
IssueConfig("seq_sat_fdss_2", [],
driver_options=["--alias", "seq-sat-fdss-2"]),
]
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=[
#'cavediving-sat14-adl:testing01_easy.pddl',
#'childsnack-sat14-strips:child-snack_pfile05.pddl',
#'citycar-sat14-adl:p3-2-2-0-1.pddl',
#'ged-sat14-strips:d-3-6.pddl',
'hiking-sat14-strips:ptesting-1-2-7.pddl',
#'maintenance-sat14-adl:maintenance-1-3-060-180-5-000.pddl',
#'tetris-sat14-strips:p020.pddl',
#'thoughtful-sat14-strips:bootstrap-typed-01.pddl',
#'transport-sat14-strips:p01.pddl',
],
processes=4,
email='*****@*****.**',
)
exp.add_absolute_report_step()
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks')
suite = [
'assembly',
'miconic-fulladl',
'openstacks',
'openstacks-sat08-adl',
'optical-telegraphs',
'philosophers',
'psr-large',
'psr-middle',
'trucks',
]
configs = {
IssueConfig('lazy-greedy-ff', [
'--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)'
]),
IssueConfig('lama-first', [], driver_options=['--alias',
'lama-first']),
IssueConfig('eager_greedy_cg', [
'--heuristic', 'h=cg()', '--search', 'eager_greedy(h, preferred=h)'
]),
IssueConfig('eager_greedy_cea', [
'--heuristic', 'h=cea()', '--search',
'eager_greedy(h, preferred=h)'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
IssueConfig('astar-ipdb', ['--search', 'astar(ipdb())']),
IssueConfig('astar-cegar-original',
['--search', 'astar(cegar(subtasks=[original()]))']),
IssueConfig(
'astar-cegar-lm-goals',
['--search', 'astar(cegar(subtasks=[landmarks(),goals()]))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=[
"issue627-base-%s" % config.nick,
"issue627-v2-%s" % config.nick
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue627_base_v2_memory_%s.png' % config.nick)
exp.add_report(RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=[
"issue627-base-%s" % config.nick,
"issue627-v2-%s" % config.nick
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue627_base_v2_total_time_%s.png' %
config.nick)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
exp.add_report(
RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue499-base-astar-lmcut", "issue499-v1-astar-lmcut"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue499_base_v1_memory.png'
)
exp.add_report(
RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue499-base-astar-lmcut", "issue499-v1-astar-lmcut"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue499_base_v1_total_time.png'
)
exp.add_report(
RelativeScatterPlotReport(
attributes=["expansions_until_last_jump"],
filter_config=["issue499-base-astar-lmcut", "issue499-v1-astar-lmcut"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue499_base_v1_expansions_until_last_jump.png'
)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
IssueConfig('astar-ipdb', ['--search', 'astar(ipdb())']),
IssueConfig('astar-seq_opt_bjolp', [
'--search',
'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true), mpd=true)'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
nick = config.nick
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue416-base-%s" % nick,
"issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_memory_%s.png' % nick)
exp.add_report(RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue416-base-%s" % nick,
"issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_total_time_%s.png' % nick)
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_all()
configs = {
IssueConfig('blind', ['--search', 'astar(blind())'],
driver_options=['--search-time-limit', '60s']),
IssueConfig('lama-first', [],
driver_options=[
'--alias', 'lama-first', '--search-time-limit', '60s'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl', 'gripper:prob01.pddl'],
processes=4,
email='*****@*****.**',
)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.append('translator_*')
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
suite = suites.suite_satisficing_with_ipc11()
configs = {
IssueConfig('seq_sat_lama_2011', [], driver_options=['--alias', 'seq-sat-lama-2011']),
IssueConfig('lama_first', [], driver_options=['--alias', 'lama-first']),
IssueConfig('ehc_lm_zhu', ['--search', 'ehc(lmcount(lm_zg()))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
nick = config.nick
exp.add_report(
RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue416-base-%s" % nick, "issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_memory_%s.png' % nick
)
exp.add_report(
RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue416-base-%s" % nick, "issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_total_time_%s.png' % nick
)
exp()
def generate_experiments(configs):
SUITE = [
"gripper:prob01.pddl", "blocks:probBLOCKS-5-0.pddl",
"visitall-sat11-strips:problem12.pddl", "airport:p01-airport1-p1.pddl"
]
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**",
export=EXPORTS)
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=2)
exp = IssueExperiment(
revisions=REVISIONS,
configs=configs,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.run_steps()
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
import suites
configs = [
IssueConfig(
"cegar-10K-original",
["--search", "astar(cegar(subtasks=[original()],max_states=10000,max_time=infinity))"]),
]
revisions = ["issue632-base", "issue632-v1"]
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suites.suite_optimal_with_ipc11(),
test_suite=["depot:pfile1"],
email="*****@*****.**",
)
exp.add_comparison_table_step()
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('dfp-reg-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-allrnd-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(randomized_order=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step()
exp()
import common_setup
from common_setup import IssueConfig, IssueExperiment
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=[],
configs=[],
environment=ENVIRONMENT,
)
compared_algorithms = []
for search in ["eager_greedy", "lazy_greedy"]:
for h1, h2 in itertools.permutations(["cea", "cg", "ff"], 2):
rev = "issue714-base"
config_nick = "-".join([search, h1, h2])
algo1 = common_setup.get_algo_nick(rev, config_nick)
exp.add_algorithm(
algo1,
common_setup.get_repo_base(),
rev, [
"--heuristic", "h{h1}={h1}".format(**locals()), "--heuristic",
"h{h2}={h2}".format(**locals()), "--search",
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
REVS = ["issue592-base", "issue592-v1"]
SUITE = suites.suite_satisficing()
CONFIGS = [
IssueConfig("seq-sat-lama-2011", [],
driver_options=["--alias", "seq-sat-lama-2011"]),
IssueConfig("lama-first", [], driver_options=["--alias", "lama-first"]),
]
exp = IssueExperiment(revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**")
exp.add_comparison_table_step()
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('sbf-miasm-rl-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-allrnd-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),single_random])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step(
name='issue668-v1-abp',
filter_config=[
'%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v1',
])
exp.add_absolute_report_step(
name='issue668-v1-pba',
filter_config=[
'%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-rnd-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-rnd-pba-b50k' % 'issue668-v1',
])
exp.add_absolute_report_step(
name='issue668-v1-paper',
filter_config=[
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-allrnd-b50k' % 'issue668-v1',
])
#if matplotlib:
#for attribute in ["memory", "total_time"]:
#for config in configs:
#exp.add_report(
#RelativeScatterPlotReport(
#attributes=[attribute],
#filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
#get_category=lambda run1, run2: run1.get("domain"),
#),
#outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
#)
exp()
heuristics = [
"{}(cache_estimates=false)".format(h)
for h in ("pdb", "cpdbs", "diverse_potentials", "all_states_potential",
"initial_state_potential", "sample_based_potentials")
]
max_eval = "max([{}])".format(",".join(heuristics))
ipc_max = "ipc_max([{}],cache_estimates=false)".format(",".join(heuristics))
configs = [
IssueConfig(name, ["--search", "astar({})".format(eval_)])
for name, eval_ in [("max", max_eval), ("ipc_max", ipc_max)]
]
revision = "1e84d77e4e37"
exp = IssueExperiment(
revisions=[revision],
configs=configs,
suite=suites.suite_optimal_strips(),
test_suite=["depot:pfile1"],
email="*****@*****.**",
)
exp.add_absolute_report_step()
exp.add_report(common_setup.CompareConfigsReport(
[(revision + "-" + "ipc_max", revision + "-" + "max")],
attributes=IssueExperiment.DEFAULT_TABLE_ATTRIBUTES),
name=common_setup.get_experiment_name() + "-compare")
exp()
# Test sampling
IssueConfig('ipdb', ['--search', 'astar(ipdb)']),
# Test genetic pattern generation
IssueConfig('genetic', ['--search', 'astar(zopdbs(patterns=genetic))']),
# Test cegar
IssueConfig(
"cegar-10K-goals-randomorder",
["--search", "astar(cegar(subtasks=[goals(order=random)],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-10K-original-randomorder",
["--search", "astar(cegar(subtasks=[original],max_states=10000,max_time=infinity,pick=random))"]),
]
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**"
)
exp.add_fetcher('data/issue648-v1-opt-test', parsers=['parser.py'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
out_of_memory = Attribute('out_of_memory', absolute=True, min_wins=True)
out_of_time = Attribute('out_of_time', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
out_of_memory,
def main(revisions=None):
benchmarks_dir = os.environ["DOWNWARD_BENCHMARKS"]
suite = [
"agricola-sat18-strips", "airport", "barman-sat11-strips",
"barman-sat14-strips", "blocks", "childsnack-sat14-strips",
"data-network-sat18-strips", "depot", "driverlog",
"elevators-sat08-strips", "elevators-sat11-strips",
"floortile-sat11-strips", "floortile-sat14-strips", "freecell",
"ged-sat14-strips", "grid", "gripper", "hiking-sat14-strips",
"logistics00", "logistics98", "miconic", "movie", "mprime", "mystery",
"nomystery-sat11-strips", "openstacks-sat08-strips",
"openstacks-sat11-strips", "openstacks-sat14-strips",
"openstacks-strips", "organic-synthesis-sat18-strips",
"organic-synthesis-split-sat18-strips", "parcprinter-08-strips",
"parcprinter-sat11-strips", "parking-sat11-strips",
"parking-sat14-strips", "pathways", "pegsol-08-strips",
"pegsol-sat11-strips", "pipesworld-notankage", "pipesworld-tankage",
"psr-small", "rovers", "satellite", "scanalyzer-08-strips",
"scanalyzer-sat11-strips", "snake-sat18-strips",
"sokoban-sat08-strips", "sokoban-sat11-strips", "spider-sat18-strips",
"storage", "termes-sat18-strips", "tetris-sat14-strips",
"thoughtful-sat14-strips", "tidybot-sat11-strips", "tpp",
"transport-sat08-strips", "transport-sat11-strips",
"transport-sat14-strips", "trucks-strips", "visitall-sat11-strips",
"visitall-sat14-strips", "woodworking-sat08-strips",
"woodworking-sat11-strips", "zenotravel"
]
# suite = ["elevators-sat08-strips:p01.pddl"]
environment = LocalEnvironment(processes=48)
BUILD_OPTIONS = ["--build", "release64"]
DRIVER_OPTIONS = [
"--transform-task", "builds/h2-mutexes/bin/preprocess",
"--overall-time-limit", "30m", "--overall-memory-limit", "4096M",
"--alias", "seq-sat-fdss-2018"
]
configs = {
IssueConfig("fdss", [],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS)
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
environment=environment,
)
exp.add_suite(benchmarks_dir, suite)
#exp.add_parser(exp.LAB_STATIC_PROPERTIES_PARSER)
#exp.add_parser(exp.LAB_DRIVER_PARSER)
#exp.add_parser(exp.EXITCODE_PARSER)
#exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
#exp.add_parser(exp.PLANNER_PARSER)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
# exp.add_comparison_table_step(attributes=attributes)
exp.add_absolute_report_step(attributes=attributes)
exp.run_steps()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('dfp-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=infinity,threshold_before_merge=1))'
]),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(),label_reduction=exact(before_shrinking=false,before_merging=true),max_states=50000))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks')
suite = suites.suite_optimal()
configs = []
for osi in ['103', '107']:
for cplex in ['1251', '1263']:
if osi == '107' and cplex == '1251':
# incompatible versions
continue
configs += [
IssueConfig('astar_initial_state_potential_OSI%s_CPLEX%s' %
(osi, cplex),
['--search', 'astar(initial_state_potential())'],
build_options=[
'issue680_OSI%s_CPLEX%s' % (osi, cplex)
],
driver_options=[
'--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)
]),
IssueConfig('astar_sample_based_potentials_OSI%s_CPLEX%s' %
(osi, cplex),
['--search', 'astar(sample_based_potentials())'],
build_options=[
'issue680_OSI%s_CPLEX%s' % (osi, cplex)
],
driver_options=[
'--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)
]),
IssueConfig(
'astar_all_states_potential_OSI%s_CPLEX%s' % (osi, cplex),
['--search', 'astar(all_states_potential())'],
build_options=['issue680_OSI%s_CPLEX%s' % (osi, cplex)],
driver_options=[
'--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)
]),
]
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl', 'gripper:prob01.pddl'],
processes=4,
email='*****@*****.**',
)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
domains = suites.suite_optimal_strips()
exp.add_absolute_report_step(filter_domain=domains)
for attribute in ["memory", "total_time"]:
for config in [
'astar_initial_state_potential',
'astar_sample_based_potentials', 'astar_all_states_potential'
]:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}_OSI{}_CPLEX1263".format(revisions[0], config, osi)
for osi in ['103', '107']
],
filter_domain=domains,
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}_CPLEX1263.png".format(
exp.name, attribute, config))
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}_OSI103_CPLEX{}".format(revisions[0], config, cplex)
for cplex in ['1251', '1263']
],
filter_domain=domains,
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}_OSI103.png".format(
exp.name, attribute, config))
exp()
"hlm,hff=lm_ff_syn(lm_rhw(reasonable_orders=true,lm_cost_type=one,cost_type=one))",
"--search",
"lazy(alt([single(hff),single(hff, pref_only=true), single(hlm), single(hlm, pref_only=true)]),"
"preferred=[hff,hlm],cost_type=one)"
]),
IssueConfig("lama-first-types-ff-g", [
"--heuristic",
"hlm,hff=lm_ff_syn(lm_rhw(reasonable_orders=true,lm_cost_type=one,cost_type=one))",
"--search",
"lazy(alt([single(hff),single(hff, pref_only=true), single(hlm), single(hlm, pref_only=true), type_based([hff, g()])]),"
"preferred=[hff,hlm],cost_type=one)"
]),
]
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**"
)
# Absolute report commented out because a comparison table is more useful for this issue.
# (It's still in this file because someone might want to use it as a basis.)
# Scatter plots commented out for now because I have no usable matplotlib available.
# exp.add_absolute_report_step()
exp.add_comparison_table_step()
# exp.add_scatter_plot_step()
exp()
def main(revisions=None):
suite = suites.suite_optimal_strips()
suite.extend(suites.suite_ipc14_opt_strips())
# only DFP configs
configs = {
# label reduction with seed 2016
IssueConfig('dfp-b50k-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false,random_seed=2016)))'
]),
IssueConfig('dfp-ginf-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false,random_seed=2016)))'
]),
IssueConfig('dfp-f50k-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true,random_seed=2016)))'
]),
# shrink fh/rnd with seed 2016
IssueConfig('dfp-f50ks2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('dfp-rnd50ks2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_random(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
# shrink fh/rnd with seed 2016 and with label reduction with seed 2016
IssueConfig('dfp-f50ks2016-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true,random_seed=2016)))'
]),
IssueConfig('dfp-rnd50ks2016-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_random(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true,random_seed=2016)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step()
exp()
def main(revisions=None):
benchmarks_dir = os.environ["DOWNWARD_BENCHMARKS_IPC2018"]
# optimal union satisficing
suite = []
#suite.extend(['briefcaseworld', 'cavediving-14-adl', 'citycar-sat14-adl', 'fsc-blocks', 'fsc-grid-a1', 'fsc-grid-a2', 'fsc-grid-r', 'fsc-hall', 'fsc-visualmarker', 'gedp-ds2ndp', 'miconic-simpleadl', 't0-adder', 't0-coins', 't0-comm', 't0-grid-dispose', 't0-grid-push', 't0-grid-trash', 't0-sortnet', 't0-sortnet-alt', 't0-uts'])
suite.extend([
"agricola-sat18", "caldera-sat18", "caldera-split-sat18",
"data-network-sat18", "flashfill-sat18", "nurikabe-sat18",
"organic-synthesis-sat18", "organic-synthesis-split-sat18",
"settlers-sat18", "snake-sat18", "spider-sat18", "termes-sat18"
])
environment = OracleGridEngineEnvironment(queue='all.q')
BUILD_OPTIONS = ["release64"]
DRIVER_OPTIONS = [
"--build", "release64", "--overall-time-limit", "30m",
"--overall-memory-limit", "4096M"
]
configs = {
IssueConfig('rb-ce-cerberus', [
'--heuristic', 'hrb=RB(dag=from_coloring, extract_plan=true)',
'--search', 'lazy_greedy([hrb],reopen_closed=false)'
],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
environment=environment,
)
exp.add_suite(benchmarks_dir, suite)
exp.add_parser(exp.LAB_STATIC_PROPERTIES_PARSER)
exp.add_parser(exp.LAB_DRIVER_PARSER)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
#exp.add_parser(exp.PLANNER_PARSER)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_absolute_report_step(attributes=attributes)
report_name = os.path.basename(exp.path.rstrip('/'))
exp.add_step('copy_report', subprocess.call, [
'cp',
os.path.join(exp.eval_dir, '%s.html' % report_name),
'/storage/US1J6721/EXTERNAL'
])
exp.add_step('print_dest', subprocess.call, [
'echo',
os.path.join("https://syss063.pok.stglabs.ibm.com/users/mkatz/storage",
'%s.html' % report_name)
])
algorithm_nicks = ['translate-symm-stabgoal-stabinit']
OTHER_REV = ''
exp.add_fetcher('data/2018-06-22-lifted-stabinit-stabgoal-order-mk-eval',
filter_algorithm=[
'{}-{}'.format(OTHER_REV, x) for x in algorithm_nicks
])
exp.add_report(
ComparativeReport(
algorithm_pairs=[('{}-{}'.format(OTHER_REV,
x), '{}-{}'.format(REVISION, x))
for x in algorithm_nicks],
attributes=attributes,
),
outfile=os.path.join(exp.eval_dir, 'a' + exp.name + '-compare.html'),
)
exp.run_steps()
def main(revisions=None):
suite=suites.suite_optimal_strips()
suite.extend(suites.suite_ipc14_opt_strips())
configs = {
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
ms_dfp_nogoalrelevantpair_counter = Attribute('ms_dfp_nogoalrelevantpair_counter', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
ms_dfp_nogoalrelevantpair_counter,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
#for attribute in ["memory", "total_time"]:
#for config in configs:
#exp.add_report(
#RelativeScatterPlotReport(
#attributes=[attribute],
#filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
#get_category=lambda run1, run2: run1.get("domain"),
#),
#outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
#)
exp()
CONFIGS = [
IssueConfig("lama-first", [], driver_options=["--alias", "lama-first"]),
IssueConfig("lm_hm", [
"--landmarks", "lm=lm_hm(2)", "--heuristic", "hlm=lmcount(lm)",
"--search", "lazy_greedy(hlm)"
]),
IssueConfig("lm_exhaust", [
"--landmarks", "lm=lm_exhaust()", "--heuristic", "hlm=lmcount(lm)",
"--search", "lazy_greedy(hlm)"
]),
IssueConfig("lm_rhw", [
"--landmarks", "lm=lm_rhw()", "--heuristic", "hlm=lmcount(lm)",
"--search", "lazy_greedy(hlm)"
]),
IssueConfig("lm_zg", [
"--landmarks", "lm=lm_zg()", "--heuristic", "hlm=lmcount(lm)",
"--search", "lazy_greedy(hlm)"
]),
]
exp = IssueExperiment(revisions=REVS,
benchmarks_dir=BENCHMARKS,
suite=SUITE,
configs=CONFIGS,
processes=4,
email="*****@*****.**")
exp.add_comparison_table_step()
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
# dummy configs with correct names so that comparison report works
configs = {
IssueConfig('rl-b50k', []),
IssueConfig('cggl-b50k', []),
IssueConfig('dfp-b50k', []),
IssueConfig('rl-ginf', []),
IssueConfig('cggl-ginf', []),
IssueConfig('dfp-ginf', []),
IssueConfig('rl-f50k', []),
IssueConfig('cggl-f50k', []),
IssueConfig('dfp-f50k', []),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue655-base-eval')
exp.add_fetcher('data/issue655-v1-eval')
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
driver_options=["--build", build]) for rev in REVISIONS
for build in BUILDS for config_nick, config in CONFIG_NICKS
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_absolute_report_step()
#exp.add_comparison_table_step()
def main(revisions=[]):
suite = suites.suite_optimal_with_ipc11()
configs = {}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v1-eval',
filter_config=[
'issue604-base-rl-b50k',
'issue604-base-cggl-b50k',
'issue604-base-dfp-b50k',
'issue604-base-rl-ginf',
'issue604-base-cggl-ginf',
'issue604-base-dfp-ginf',
'issue604-base-rl-f50k',
'issue604-base-cggl-f50k',
'issue604-base-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-eval',
filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-rest-eval',
filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'),
('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'),
('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'),
('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'),
('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'),
('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'),
('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'),
('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'),
('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-base-v7-comparison.html'))
exp()
