def main(revisions=None):
benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks')
suite=suites.suite_optimal_strips()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-blind-sss', ['--search', 'astar(blind(), pruning=stubborn_sets_simple())']),
IssueConfig('astar-blind-ssec', ['--search', 'astar(blind(), pruning=stubborn_sets_ec())']),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
)
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('dfp-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=infinity,threshold_before_merge=1))'
]),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(),label_reduction=exact(before_shrinking=false,before_merging=true),max_states=50000))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks')
suite = suites.suite_optimal()
configs = []
for osi in ['103', '107']:
for cplex in ['1251', '1263']:
if osi == '107' and cplex == '1251':
# incompatible versions
continue
configs += [
IssueConfig('astar_initial_state_potential_OSI%s_CPLEX%s' %
(osi, cplex),
['--search', 'astar(initial_state_potential())'],
build_options=[
'issue680_OSI%s_CPLEX%s' % (osi, cplex)
],
driver_options=[
'--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)
]),
IssueConfig('astar_sample_based_potentials_OSI%s_CPLEX%s' %
(osi, cplex),
['--search', 'astar(sample_based_potentials())'],
build_options=[
'issue680_OSI%s_CPLEX%s' % (osi, cplex)
],
driver_options=[
'--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)
]),
IssueConfig(
'astar_all_states_potential_OSI%s_CPLEX%s' % (osi, cplex),
['--search', 'astar(all_states_potential())'],
build_options=['issue680_OSI%s_CPLEX%s' % (osi, cplex)],
driver_options=[
'--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)
]),
]
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl', 'gripper:prob01.pddl'],
processes=4,
email='*****@*****.**',
)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
domains = suites.suite_optimal_strips()
exp.add_absolute_report_step(filter_domain=domains)
for attribute in ["memory", "total_time"]:
for config in [
'astar_initial_state_potential',
'astar_sample_based_potentials', 'astar_all_states_potential'
]:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}_OSI{}_CPLEX1263".format(revisions[0], config, osi)
for osi in ['103', '107']
],
filter_domain=domains,
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}_CPLEX1263.png".format(
exp.name, attribute, config))
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}_OSI103_CPLEX{}".format(revisions[0], config, cplex)
for cplex in ['1251', '1263']
],
filter_domain=domains,
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}_OSI103.png".format(
exp.name, attribute, config))
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('dfp-reg-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-reg-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-inv-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-rnd-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-allrnd-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp(randomized_order=true),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step()
exp()
from lab.environments import LocalEnvironment, MaiaEnvironment
from common_setup import IssueConfig, IssueExperiment, is_test_run
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue698-base", "issue698-v1"]
CONFIGS = [
IssueConfig("blind", ["--search", "astar(blind())"],
driver_options=["--search-time-limit", "60s"])
]
sys.path.append(BENCHMARKS_DIR)
import suites
SUITE = suites.suite_optimal_strips()
ENVIRONMENT = MaiaEnvironment(priority=0,
email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_command("parser", ["custom-parser.py"])
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('sbf-miasm-rl-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-allrnd-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),single_random])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step(
name='issue668-v1-abp',
filter_config=[
'%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v1',
])
exp.add_absolute_report_step(
name='issue668-v1-pba',
filter_config=[
'%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-rnd-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-rnd-pba-b50k' % 'issue668-v1',
])
exp.add_absolute_report_step(
name='issue668-v1-paper',
filter_config=[
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-allrnd-b50k' % 'issue668-v1',
])
#if matplotlib:
#for attribute in ["memory", "total_time"]:
#for config in configs:
#exp.add_report(
#RelativeScatterPlotReport(
#attributes=[attribute],
#filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
#get_category=lambda run1, run2: run1.get("domain"),
#),
#outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
#)
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
# dummy configs with correct names so that comparison report works
configs = {
IssueConfig('rl-b50k', []),
IssueConfig('cggl-b50k', []),
IssueConfig('dfp-b50k', []),
IssueConfig('rl-ginf', []),
IssueConfig('cggl-ginf', []),
IssueConfig('dfp-ginf', []),
IssueConfig('rl-f50k', []),
IssueConfig('cggl-f50k', []),
IssueConfig('dfp-f50k', []),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue655-base-eval')
exp.add_fetcher('data/issue655-v1-eval')
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
heuristics = [
"{}(cache_estimates=false)".format(h)
for h in ("pdb", "cpdbs", "diverse_potentials", "all_states_potential",
"initial_state_potential", "sample_based_potentials")
]
max_eval = "max([{}])".format(",".join(heuristics))
ipc_max = "ipc_max([{}],cache_estimates=false)".format(",".join(heuristics))
configs = [
IssueConfig(name, ["--search", "astar({})".format(eval_)])
for name, eval_ in [("max", max_eval), ("ipc_max", ipc_max)]
]
revision = "1e84d77e4e37"
exp = IssueExperiment(
revisions=[revision],
configs=configs,
suite=suites.suite_optimal_strips(),
test_suite=["depot:pfile1"],
email="*****@*****.**",
)
exp.add_absolute_report_step()
exp.add_report(common_setup.CompareConfigsReport(
[(revision + "-" + "ipc_max", revision + "-" + "max")],
attributes=IssueExperiment.DEFAULT_TABLE_ATTRIBUTES),
name=common_setup.get_experiment_name() + "-compare")
exp()
