def main(revisions=None): benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks') suite=suites.suite_optimal_strips() configs = { IssueConfig('astar-blind', ['--search', 'astar(blind())']), IssueConfig('astar-blind-sss', ['--search', 'astar(blind(), pruning=stubborn_sets_simple())']), IssueConfig('astar-blind-ssec', ['--search', 'astar(blind(), pruning=stubborn_sets_ec())']), } exp = IssueExperiment( benchmarks_dir=benchmarks_dir, suite=suite, revisions=revisions, configs=configs, test_suite=['depot:p01.pddl'], processes=4, email='*****@*****.**', ) exp.add_comparison_table_step() if matplotlib: for attribute in ["memory", "total_time"]: for config in configs: exp.add_report( RelativeScatterPlotReport( attributes=[attribute], filter_config=["{}-{}".format(rev, config.nick) for rev in revisions], get_category=lambda run1, run2: run1.get("domain"), ), outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick) ) exp()
def main(revisions=None): benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks') suite = suites.suite_optimal_strips() configs = { IssueConfig('dfp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('dfp-ginf', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=infinity,threshold_before_merge=1))' ]), IssueConfig('dfp-f50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(),label_reduction=exact(before_shrinking=false,before_merging=true),max_states=50000))' ]), } exp = IssueExperiment( benchmarks_dir=benchmarks_dir, suite=suite, revisions=revisions, configs=configs, test_suite=['depot:p01.pddl'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_comparison_table_step() if matplotlib: for attribute in ["memory", "total_time"]: for config in configs: exp.add_report(RelativeScatterPlotReport( attributes=[attribute], filter_config=[ "{}-{}".format(rev, config.nick) for rev in revisions ], get_category=lambda run1, run2: run1.get("domain"), ), outfile="{}-{}-{}.png".format( exp.name, attribute, config.nick)) exp()
def main(revisions=None): benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks') suite = suites.suite_optimal() configs = [] for osi in ['103', '107']: for cplex in ['1251', '1263']: if osi == '107' and cplex == '1251': # incompatible versions continue configs += [ IssueConfig('astar_initial_state_potential_OSI%s_CPLEX%s' % (osi, cplex), ['--search', 'astar(initial_state_potential())'], build_options=[ 'issue680_OSI%s_CPLEX%s' % (osi, cplex) ], driver_options=[ '--build=issue680_OSI%s_CPLEX%s' % (osi, cplex) ]), IssueConfig('astar_sample_based_potentials_OSI%s_CPLEX%s' % (osi, cplex), ['--search', 'astar(sample_based_potentials())'], build_options=[ 'issue680_OSI%s_CPLEX%s' % (osi, cplex) ], driver_options=[ '--build=issue680_OSI%s_CPLEX%s' % (osi, cplex) ]), IssueConfig( 'astar_all_states_potential_OSI%s_CPLEX%s' % (osi, cplex), ['--search', 'astar(all_states_potential())'], build_options=['issue680_OSI%s_CPLEX%s' % (osi, cplex)], driver_options=[ '--build=issue680_OSI%s_CPLEX%s' % (osi, cplex) ]), ] exp = IssueExperiment( benchmarks_dir=benchmarks_dir, suite=suite, revisions=revisions, configs=configs, test_suite=['depot:p01.pddl', 'gripper:prob01.pddl'], processes=4, email='*****@*****.**', ) attributes = exp.DEFAULT_TABLE_ATTRIBUTES domains = suites.suite_optimal_strips() exp.add_absolute_report_step(filter_domain=domains) for attribute in ["memory", "total_time"]: for config in [ 'astar_initial_state_potential', 'astar_sample_based_potentials', 'astar_all_states_potential' ]: exp.add_report(RelativeScatterPlotReport( attributes=[attribute], filter_config=[ "{}-{}_OSI{}_CPLEX1263".format(revisions[0], config, osi) for osi in ['103', '107'] ], filter_domain=domains, get_category=lambda run1, run2: run1.get("domain"), ), outfile="{}-{}-{}_CPLEX1263.png".format( exp.name, attribute, config)) exp.add_report(RelativeScatterPlotReport( attributes=[attribute], filter_config=[ "{}-{}_OSI103_CPLEX{}".format(revisions[0], config, cplex) for cplex in ['1251', '1263'] ], filter_domain=domains, get_category=lambda run1, run2: run1.get("domain"), ), outfile="{}-{}-{}_OSI103.png".format( exp.name, attribute, config)) exp()
def main(revisions=None): benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks') suite = suites.suite_optimal_strips() configs = { IssueConfig('dfp-reg-otn-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-reg-nto-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-reg-rnd-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-inv-otn-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-inv-nto-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-inv-rnd-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-rnd-otn-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-rnd-nto-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-rnd-rnd-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-reg-otn-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-reg-nto-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-reg-rnd-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-inv-otn-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-inv-nto-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-inv-rnd-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-rnd-otn-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-rnd-nto-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-rnd-rnd-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), IssueConfig('dfp-allrnd-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(randomized_order=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))' ]), } exp = IssueExperiment( benchmarks_dir=benchmarks_dir, suite=suite, revisions=revisions, configs=configs, test_suite=['depot:p01.pddl'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_absolute_report_step() exp()
from lab.environments import LocalEnvironment, MaiaEnvironment from common_setup import IssueConfig, IssueExperiment, is_test_run BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue698-base", "issue698-v1"] CONFIGS = [ IssueConfig("blind", ["--search", "astar(blind())"], driver_options=["--search-time-limit", "60s"]) ] sys.path.append(BENCHMARKS_DIR) import suites SUITE = suites.suite_optimal_strips() ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**") if is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_command("parser", ["custom-parser.py"])
def main(revisions=None): benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks') suite = suites.suite_optimal_strips() configs = { IssueConfig('sbf-miasm-rl-otn-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rl-nto-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rl-rnd-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-l-otn-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-l-nto-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-l-rnd-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rnd-otn-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rnd-nto-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rnd-rnd-abp-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rl-otn-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rl-nto-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rl-rnd-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-l-otn-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-l-nto-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-l-rnd-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rnd-otn-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rnd-nto-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-rnd-rnd-pba-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), IssueConfig('sbf-miasm-allrnd-b50k', [ '--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),single_random])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))' ]), } exp = IssueExperiment( benchmarks_dir=benchmarks_dir, suite=suite, revisions=revisions, configs=configs, test_suite=['depot:p01.pddl'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_absolute_report_step( name='issue668-v1-abp', filter_config=[ '%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v1', ]) exp.add_absolute_report_step( name='issue668-v1-pba', filter_config=[ '%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-l-rnd-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-otn-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-rnd-pba-b50k' % 'issue668-v1', ]) exp.add_absolute_report_step( name='issue668-v1-paper', filter_config=[ '%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1', '%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1', '%s-sbf-miasm-allrnd-b50k' % 'issue668-v1', ]) #if matplotlib: #for attribute in ["memory", "total_time"]: #for config in configs: #exp.add_report( #RelativeScatterPlotReport( #attributes=[attribute], #filter_config=["{}-{}".format(rev, config.nick) for rev in revisions], #get_category=lambda run1, run2: run1.get("domain"), #), #outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick) #) exp()
def main(revisions=None): benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks') suite = suites.suite_optimal_strips() # dummy configs with correct names so that comparison report works configs = { IssueConfig('rl-b50k', []), IssueConfig('cggl-b50k', []), IssueConfig('dfp-b50k', []), IssueConfig('rl-ginf', []), IssueConfig('cggl-ginf', []), IssueConfig('dfp-ginf', []), IssueConfig('rl-f50k', []), IssueConfig('cggl-f50k', []), IssueConfig('dfp-f50k', []), } exp = IssueExperiment( benchmarks_dir=benchmarks_dir, suite=suite, revisions=revisions, configs=configs, test_suite=['depot:p01.pddl'], processes=4, email='*****@*****.**', ) exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py') exp.add_command('ms-parser', ['ms_parser']) # planner outcome attributes perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False) proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False) actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm]) # m&s attributes ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm]) ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False) ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True) ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True) ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True) search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True) search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True) extra_attributes = [ perfect_heuristic, proved_unsolvability, actual_search_time, ms_construction_time, ms_abstraction_constructed, ms_final_size, ms_out_of_memory, ms_out_of_time, search_out_of_memory, search_out_of_time, ] attributes = exp.DEFAULT_TABLE_ATTRIBUTES attributes.extend(extra_attributes) exp.add_fetcher('data/issue655-base-eval') exp.add_fetcher('data/issue655-v1-eval') exp.add_comparison_table_step() if matplotlib: for attribute in ["memory", "total_time"]: for config in configs: exp.add_report(RelativeScatterPlotReport( attributes=[attribute], filter_config=[ "{}-{}".format(rev, config.nick) for rev in revisions ], get_category=lambda run1, run2: run1.get("domain"), ), outfile="{}-{}-{}.png".format( exp.name, attribute, config.nick)) exp()
heuristics = [ "{}(cache_estimates=false)".format(h) for h in ("pdb", "cpdbs", "diverse_potentials", "all_states_potential", "initial_state_potential", "sample_based_potentials") ] max_eval = "max([{}])".format(",".join(heuristics)) ipc_max = "ipc_max([{}],cache_estimates=false)".format(",".join(heuristics)) configs = [ IssueConfig(name, ["--search", "astar({})".format(eval_)]) for name, eval_ in [("max", max_eval), ("ipc_max", ipc_max)] ] revision = "1e84d77e4e37" exp = IssueExperiment( revisions=[revision], configs=configs, suite=suites.suite_optimal_strips(), test_suite=["depot:pfile1"], email="*****@*****.**", ) exp.add_absolute_report_step() exp.add_report(common_setup.CompareConfigsReport( [(revision + "-" + "ipc_max", revision + "-" + "max")], attributes=IssueExperiment.DEFAULT_TABLE_ATTRIBUTES), name=common_setup.get_experiment_name() + "-compare") exp()