def main(revisions=None): suite = suites.suite_optimal_with_ipc11() configs = { IssueConfig('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']), IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']), IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']), IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']), IssueConfig('cggl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']), IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']), IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']), IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']), IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']), } exp = IssueExperiment( revisions=revisions, configs=configs, suite=suite, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True) extra_attributes = [ search_out_of_memory, search_out_of_time, perfect_heuristic, proved_unsolvability, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, ms_memory_delta, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_comparison_table_step() exp()
def main(revisions=None): suite = suites.suite_optimal_with_ipc11() configs = { IssueConfig('astar-blind', ['--search', 'astar(blind())']), IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']), IssueConfig('astar-ipdb', ['--search', 'astar(ipdb())']), IssueConfig('astar-seq_opt_bjolp', [ '--search', 'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true), mpd=true)' ]), } exp = IssueExperiment( revisions=revisions, configs=configs, suite=suite, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_comparison_table_step() for config in configs: nick = config.nick exp.add_report(RelativeScatterPlotReport( attributes=["memory"], filter_config=["issue416-base-%s" % nick, "issue416-v1-%s" % nick], get_category=lambda run1, run2: run1.get("domain"), ), outfile='issue416_base_v1_memory_%s.png' % nick) exp.add_report(RelativeScatterPlotReport( attributes=["total_time"], filter_config=["issue416-base-%s" % nick, "issue416-v1-%s" % nick], get_category=lambda run1, run2: run1.get("domain"), ), outfile='issue416_base_v1_total_time_%s.png' % nick) exp()
#! /usr/bin/env python # -*- coding: utf-8 -*- from downward import suites import common_setup exp = common_setup.IssueExperiment( search_revisions=["issue422-base", "issue422-v1"], configs={"lmcut": ["--search", "astar(lmcut())"]}, suite=suites.suite_optimal_with_ipc11(), ) exp.add_absolute_report_step() exp.add_comparison_table_step() exp.add_scatter_plot_step() exp()
def main(revisions=None): SUITE = suites.suite_optimal_with_ipc11() B_CONFIGS = { 'rl-b50k': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))' ], 'cggl-b50k': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))' ], 'dfp-b50k': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))' ], } G_CONFIGS = { 'rl-ginf': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))' ], 'cggl-ginf': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))' ], 'dfp-ginf': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))' ], } F_CONFIGS = { 'rl-f50k': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))' ], 'cggl-f50k': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))' ], 'dfp-f50k': [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))' ], } CONFIGS = dict(B_CONFIGS) CONFIGS.update(G_CONFIGS) CONFIGS.update(F_CONFIGS) exp = common_setup.IssueExperiment( revisions=revisions, configs=CONFIGS, suite=SUITE, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_comparison_table_step(attributes=attributes) exp()
#! /usr/bin/env python # -*- coding: utf-8 -*- from downward import suites import common_setup CONFIGS = { 'astar_lmcount_lm_merged_rhw_hm': [ '--search', 'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true),mpd=true)'], } exp = common_setup.IssueExperiment( search_revisions=["issue468-base", "issue468-v1"], configs=CONFIGS, suite=suites.suite_optimal_with_ipc11(), ) exp.add_comparison_table_step() exp()
from downward.configs import default_configs_optimal from downward.reports.scatter import ScatterPlotReport import common_setup REVS = ["issue214-base", "issue214-v5"] CONFIGS = {"blind": ["--search", "astar(blind())"]} TEST_RUN = False if TEST_RUN: SUITE = "gripper:prob01.pddl" PRIORITY = None # "None" means local experiment else: SUITE = list(sorted(set(suites.suite_all()) - set(suites.suite_optimal_with_ipc11()))) PRIORITY = 0 # number means maia experiment exp = common_setup.MyExperiment( grid_priority=PRIORITY, revisions=REVS, configs=CONFIGS, suite=SUITE, parsers=['state_size_parser.py'], ) exp.add_comparison_table_step( attributes=common_setup.MyExperiment.DEFAULT_TABLE_ATTRIBUTES + ['bytes_per_state', 'variables', 'state_var_t_size'] )
def main(rev1, rev2): REVS = [rev1, rev2] LIMITS = {"search_time": 1800} SUITE = suites.suite_optimal_with_ipc11() B_CONFIGS = { 'rl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'], 'cggl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'], 'dfp-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'], } G_CONFIGS = { 'rl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'], 'cggl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'], 'dfp-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'], } F_CONFIGS = { 'rl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'], 'cggl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'], 'dfp-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'], } CONFIGS = dict(B_CONFIGS) CONFIGS.update(G_CONFIGS) CONFIGS.update(F_CONFIGS) exp = common_setup.IssueExperiment( search_revisions=REVS, configs=CONFIGS, suite=SUITE, limits=LIMITS, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_search_parser('ms-parser.py') # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, actual_search_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_comparison_table_step(attributes=attributes) exp()
def main(revisions=[]): suite = suites.suite_optimal_with_ipc11() configs = {} exp = IssueExperiment( revisions=revisions, configs=configs, suite=suite, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True) extra_attributes = [ search_out_of_memory, search_out_of_time, perfect_heuristic, proved_unsolvability, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, ms_memory_delta, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_fetcher('data/issue604-v1-eval', filter_config=[ 'issue604-base-rl-b50k', 'issue604-base-cggl-b50k', 'issue604-base-dfp-b50k', 'issue604-base-rl-ginf', 'issue604-base-cggl-ginf', 'issue604-base-dfp-ginf', 'issue604-base-rl-f50k', 'issue604-base-cggl-f50k', 'issue604-base-dfp-f50k', ]) exp.add_fetcher('data/issue604-v7-eval', filter_config=[ 'issue604-v7-rl-b50k', 'issue604-v7-cggl-b50k', 'issue604-v7-dfp-b50k', 'issue604-v7-rl-ginf', 'issue604-v7-cggl-ginf', 'issue604-v7-dfp-ginf', 'issue604-v7-rl-f50k', 'issue604-v7-cggl-f50k', 'issue604-v7-dfp-f50k', ]) exp.add_fetcher('data/issue604-v7-rest-eval', filter_config=[ 'issue604-v7-rl-b50k', 'issue604-v7-cggl-b50k', 'issue604-v7-dfp-b50k', 'issue604-v7-rl-ginf', 'issue604-v7-cggl-ginf', 'issue604-v7-dfp-ginf', 'issue604-v7-rl-f50k', 'issue604-v7-cggl-f50k', 'issue604-v7-dfp-f50k', ]) exp.add_report(CompareConfigsReport(compared_configs=[ ('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'), ('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'), ('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'), ('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'), ('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'), ('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'), ('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'), ('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'), ('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'), ], attributes=attributes), outfile=os.path.join(exp.eval_dir, 'issue604-base-v7-comparison.html')) exp()
def main(revisions=None): SUITE = suites.suite_optimal_with_ipc11() B_CONFIGS = { "dfp-b50k": [ "--search", "astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))", ] } G_CONFIGS = { "dfp-ginf": [ "--search", "astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))", ] } F_CONFIGS = { "dfp-f50k": [ "--search", "astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))", ] } CONFIGS = dict(B_CONFIGS) CONFIGS.update(G_CONFIGS) CONFIGS.update(F_CONFIGS) exp = common_setup.IssueExperiment( revisions=revisions, configs=CONFIGS, suite=SUITE, test_suite=["depot:pfile1"], processes=4, email="*****@*****.**", ) exp.add_resource("ms_parser", "ms-parser.py", dest="ms-parser.py") exp.add_command("ms-parser", ["ms_parser"]) # planner outcome attributes perfect_heuristic = Attribute("perfect_heuristic", absolute=True, min_wins=False) proved_unsolvability = Attribute("proved_unsolvability", absolute=True, min_wins=False) actual_search_time = Attribute("actual_search_time", absolute=False, min_wins=True, functions=[gm]) initial_h_value = Attribute("initial_h_value", absolute=False, min_wins=False) # m&s attributes ms_construction_time = Attribute("ms_construction_time", absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute("ms_abstraction_constructed", absolute=True, min_wins=False) ms_final_size = Attribute("ms_final_size", absolute=False, min_wins=True) ms_out_of_memory = Attribute("ms_out_of_memory", absolute=True, min_wins=True) ms_out_of_time = Attribute("ms_out_of_time", absolute=True, min_wins=True) search_out_of_memory = Attribute("search_out_of_memory", absolute=True, min_wins=True) search_out_of_time = Attribute("search_out_of_time", absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, initial_h_value, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_comparison_table_step(attributes=attributes) exp()
def main(revisions=[]): suite = suites.suite_optimal_with_ipc11() configs = { } exp = IssueExperiment( revisions=revisions, configs=configs, suite=suite, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True) extra_attributes = [ search_out_of_memory, search_out_of_time, perfect_heuristic, proved_unsolvability, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, ms_memory_delta, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_fetcher('data/issue604-v1-eval',filter_config=[ 'issue604-base-rl-b50k', 'issue604-base-cggl-b50k', 'issue604-base-dfp-b50k', 'issue604-base-rl-ginf', 'issue604-base-cggl-ginf', 'issue604-base-dfp-ginf', 'issue604-base-rl-f50k', 'issue604-base-cggl-f50k', 'issue604-base-dfp-f50k', ]) exp.add_fetcher('data/issue604-v7-eval',filter_config=[ 'issue604-v7-rl-b50k', 'issue604-v7-cggl-b50k', 'issue604-v7-dfp-b50k', 'issue604-v7-rl-ginf', 'issue604-v7-cggl-ginf', 'issue604-v7-dfp-ginf', 'issue604-v7-rl-f50k', 'issue604-v7-cggl-f50k', 'issue604-v7-dfp-f50k', ]) exp.add_fetcher('data/issue604-v7-rest-eval',filter_config=[ 'issue604-v7-rl-b50k', 'issue604-v7-cggl-b50k', 'issue604-v7-dfp-b50k', 'issue604-v7-rl-ginf', 'issue604-v7-cggl-ginf', 'issue604-v7-dfp-ginf', 'issue604-v7-rl-f50k', 'issue604-v7-cggl-f50k', 'issue604-v7-dfp-f50k', ]) exp.add_report(CompareConfigsReport(compared_configs=[ ('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'), ('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'), ('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'), ('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'), ('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'), ('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'), ('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'), ('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'), ('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'), ],attributes=attributes),outfile=os.path.join( exp.eval_dir, 'issue604-base-v7-comparison.html')) exp()
def __init__(self, path, repo, opt_or_sat, rev, base_rev=None, use_core_configs=True, use_ipc_configs=True, use_extended_configs=False, **kwargs): """ See :py:class:`DownwardExperiment <downward.experiments.DownwardExperiment>` for inherited parameters. The experiment will be built at *path*. *repo* must be the path to a Fast Downward repository. This repository is used to search for problem files. If *opt_or_sat* is 'opt', configurations for optimal planning will be tested on all domains suited for optimal planning. If it is 'sat', configurations for satisficing planning will be tested on the satisficing suite. *rev* determines the new revision to test. If *base_rev* is None (default), the latest revision on the branch default that is an ancestor of *rev* will be used. *use_core_configs* determines if the most common configurations are tested (default: True). *use_ipc_configs* determines if the configurations used in the IPCs are tested (default: True). *use_extended_configs* determines if some less common configurations are tested (default: False). """ base_rev = checkouts.get_common_ancestor(repo, rev) combos = [(Translator(repo, rev=r), Preprocessor(repo, rev=r), Planner(repo, rev=r)) for r in (base_rev, rev)] DownwardExperiment.__init__(self, path, repo, combinations=combos, **kwargs) # ------ suites and configs ------------------------------------ if opt_or_sat == 'opt': self.add_suite(suite_optimal_with_ipc11()) configs = default_configs_optimal(use_core_configs, use_ipc_configs, use_extended_configs) elif opt_or_sat == 'sat': self.add_suite(suite_satisficing_with_ipc11()) configs = default_configs_satisficing(use_core_configs, use_ipc_configs, use_extended_configs) else: logging.critical('Select to test either \'opt\' or \'sat\' configurations') for nick, command in configs.items(): self.add_config(nick, command) # ------ reports ----------------------------------------------- comparison = CompareRevisionsReport(base_rev, rev, attributes=COMPARED_ATTRIBUTES) self.add_report(comparison, name='report-compare-scores', outfile='report-compare-scores.html') for nick in configs.keys(): config_before = '%s-%s' % (base_rev, nick) config_after = '%s-%s' % (rev, nick) for attribute in SCATTER_PLOT_ATTRIBUTES: name = 'scatter-%s-%s' % (attribute, nick) self.add_report( ScatterPlotReport( filter_config=[config_before, config_after], attributes=[attribute], get_category=lambda run1, run2: run1['domain']), outfile=name)
def main(revisions=None): suite = suites.suite_optimal_with_ipc11() configs = { IssueConfig('rl-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('cggl-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('rl-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('cggl-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('rl-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))' ]), IssueConfig('cggl-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))' ]), IssueConfig('dfp-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))' ]), } exp = IssueExperiment( revisions=revisions, configs=configs, suite=suite, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True) extra_attributes = [ search_out_of_memory, search_out_of_time, perfect_heuristic, proved_unsolvability, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, ms_memory_delta, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_fetcher('data/issue604-v4-eval') exp.add_report(CompareConfigsReport(compared_configs=[ ('issue604-v3-rl-b50k', 'issue604-v5-rl-b50k'), ('issue604-v3-cggl-b50k', 'issue604-v5-cggl-b50k'), ('issue604-v3-dfp-b50k', 'issue604-v5-dfp-b50k'), ('issue604-v3-rl-ginf', 'issue604-v5-rl-ginf'), ('issue604-v3-cggl-ginf', 'issue604-v5-cggl-ginf'), ('issue604-v3-dfp-ginf', 'issue604-v5-dfp-ginf'), ('issue604-v3-rl-f50k', 'issue604-v5-rl-f50k'), ('issue604-v3-cggl-f50k', 'issue604-v5-cggl-f50k'), ('issue604-v3-dfp-f50k', 'issue604-v5-dfp-f50k'), ], attributes=attributes), outfile=os.path.join(exp.eval_dir, 'issue604-v3-v5-comparison.html')) exp.add_report(CompareConfigsReport(compared_configs=[ ('issue604-v4-rl-b50k', 'issue604-v5-rl-b50k'), ('issue604-v4-cggl-b50k', 'issue604-v5-cggl-b50k'), ('issue604-v4-dfp-b50k', 'issue604-v5-dfp-b50k'), ('issue604-v4-rl-ginf', 'issue604-v5-rl-ginf'), ('issue604-v4-cggl-ginf', 'issue604-v5-cggl-ginf'), ('issue604-v4-dfp-ginf', 'issue604-v5-dfp-ginf'), ('issue604-v4-rl-f50k', 'issue604-v5-rl-f50k'), ('issue604-v4-cggl-f50k', 'issue604-v5-cggl-f50k'), ('issue604-v4-dfp-f50k', 'issue604-v5-dfp-f50k'), ], attributes=attributes), outfile=os.path.join(exp.eval_dir, 'issue604-v4-v5-comparison.html')) exp()
def main(revisions=None): suite = suites.suite_optimal_with_ipc11() configs = { IssueConfig('rl-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('cggl-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('rl-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('cggl-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('rl-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))' ]), IssueConfig('cggl-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))' ]), IssueConfig('dfp-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))' ]), } exp = IssueExperiment( revisions=revisions, configs=configs, suite=suite, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_comparison_table_step() exp.add_report(RelativeScatterPlotReport( attributes=["memory"], filter_config=["issue604-base-dfp-ginf", "issue604-v1-dfp-ginf"], get_category=lambda run1, run2: run1.get("domain"), ), outfile='issue604_base_v1_memory_dfp.png') exp.add_report(RelativeScatterPlotReport( attributes=["memory"], filter_config=["issue604-base-rl-ginf", "issue604-v1-rl-ginf"], get_category=lambda run1, run2: run1.get("domain"), ), outfile='issue604_base_v1_memory_rl.png') exp()
#! /usr/bin/env python # -*- coding: utf-8 -*- from downward import suites import common_setup REVS = ["issue546-base", "issue546-v1"] LIMITS = {"search_time": 1800} SUITE = suites.suite_optimal_with_ipc11() CONFIGS = { "seq_opt_fdss_1": ["--alias", "seq-opt-fdss-1"], "seq_opt_fdss_2": ["--alias", "seq-opt-fdss-2"], } exp = common_setup.IssueExperiment( search_revisions=REVS, configs=CONFIGS, suite=SUITE, limits=LIMITS, ) exp.add_comparison_table_step( attributes=common_setup.IssueExperiment.PORTFOLIO_ATTRIBUTES) exp()
def main(revisions=None): SUITE = suites.suite_optimal_with_ipc11() B_CONFIGS = { 'rl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'], 'cggl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'], 'dfp-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'], } G_CONFIGS = { 'rl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'], 'cggl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'], 'dfp-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'], } F_CONFIGS = { 'rl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'], 'cggl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'], 'dfp-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'], } CONFIGS = dict(B_CONFIGS) CONFIGS.update(G_CONFIGS) CONFIGS.update(F_CONFIGS) exp = common_setup.IssueExperiment( revisions=revisions, configs=CONFIGS, suite=SUITE, test_suite=['depot:pfile1'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_fetcher('data/issue601-base-eval') exp.add_report(CompareConfigsReport(compared_configs=[ ('issue601-base-rl-b50k', 'issue601-v1-rl-b50k'), ('issue601-base-cggl-b50k', 'issue601-v1-cggl-b50k'), ('issue601-base-dfp-b50k', 'issue601-v1-dfp-b50k'), ('issue601-base-rl-ginf', 'issue601-v1-rl-ginf'), ('issue601-base-cggl-ginf', 'issue601-v1-cggl-ginf'), ('issue601-base-dfp-ginf', 'issue601-v1-dfp-ginf'), ('issue601-base-rl-f50k', 'issue601-v1-rl-f50k'), ('issue601-base-cggl-f50k', 'issue601-v1-cggl-f50k'), ('issue601-base-dfp-f50k', 'issue601-v1-dfp-f50k'), ],attributes=attributes)) exp()
#! /usr/bin/env python # -*- coding: utf-8 -*- from downward import suites import configs import common_setup REVS = ["issue436-base", "issue436-v2"] LIMITS = {"search_time": 1800} SUITE = suites.suite_optimal_with_ipc11() configs_optimal_core = configs.configs_optimal_core() CONFIGS = {} for name in [ 'astar_merge_and_shrink_greedy_bisim', 'astar_merge_and_shrink_dfp_bisim', 'astar_ipdb', 'astar_hmax', 'astar_blind', 'astar_lmcut', 'astar_merge_and_shrink_bisim', 'astar_lmcount_lm_merged_rhw_hm' ]: CONFIGS[name] = configs_optimal_core[name] exp = common_setup.IssueExperiment( search_revisions=REVS, configs=CONFIGS, suite=SUITE, limits=LIMITS, ) exp.add_absolute_report_step() exp.add_comparison_table_step()
#! /usr/bin/env python # -*- coding: utf-8 -*- from downward import suites import common_setup CONFIGS = { "astar_ipdb": ["--search", "astar(ipdb())"], "astar_pdb": ["--search", "astar(pdb())"], "astar_gapdb": ["--search", "astar(gapdb())"], } exp = common_setup.IssueExperiment( search_revisions=["issue488-base", "issue488-v1"], configs=CONFIGS, suite=suites.suite_optimal_with_ipc11() ) exp.add_comparison_table_step() exp()