def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
CONFIGS = [
IssueConfig(heuristic, ["--search", "astar({})".format(heuristic)])
for heuristic in
["blind()", "cegar(max_states=10000)", "hm()", "lmcut()", "hmax()"]
]
SUITE = [
'barman-opt14-strips', 'cavediving-14-adl', 'childsnack-opt14-strips',
'citycar-opt14-adl', 'floortile-opt14-strips', 'ged-opt14-strips',
'hiking-opt14-strips', 'maintenance-opt14-adl', 'openstacks-opt14-strips',
'parking-opt14-strips', 'tetris-opt14-strips', 'tidybot-opt14-strips',
'transport-opt14-strips', 'visitall-opt14-strips'
]
ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
exp.add_scatter_plot_step(attributes=["total_time"])
exp()
def main(revisions=None):
benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks')
suite=suites.suite_optimal()
configs = []
for osi in ['103', '107']:
for cplex in ['1251', '1263']:
if osi == '107' and cplex == '1251':
# incompatible versions
continue
configs += [
IssueConfig(
'astar_seq_landmarks_OSI%s_CPLEX%s' % (osi, cplex),
['--search', 'astar(operatorcounting([state_equation_constraints(), lmcut_constraints()]))'],
build_options=['issue680_OSI%s_CPLEX%s' % (osi, cplex)],
driver_options=['--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)]
),
IssueConfig(
'astar_diverse_potentials_OSI%s_CPLEX%s' % (osi, cplex),
['--search', 'astar(diverse_potentials())'],
build_options=['issue680_OSI%s_CPLEX%s' % (osi, cplex)],
driver_options=['--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)]
),
IssueConfig(
'astar_lmcount_OSI%s_CPLEX%s' % (osi, cplex),
['--search', 'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true,optimal=true),mpd=true)'],
build_options=['issue680_OSI%s_CPLEX%s' % (osi, cplex)],
driver_options=['--build=issue680_OSI%s_CPLEX%s' % (osi, cplex)]
),
]
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl', 'gripper:prob01.pddl'],
processes=4,
email='*****@*****.**',
)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
domains = suites.suite_optimal_strips()
exp.add_absolute_report_step(filter_domain=domains)
for attribute in ["memory", "total_time"]:
for config in ['astar_seq_landmarks', 'astar_diverse_potentials', 'astar_lmcount']:
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}_OSI{}_CPLEX1263".format(revisions[0], config, osi) for osi in ['103', '107']],
filter_domain=domains,
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}_CPLEX1263.png".format(exp.name, attribute, config)
)
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}_OSI103_CPLEX{}".format(revisions[0], config, cplex) for cplex in ['1251', '1263']],
filter_domain=domains,
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}_OSI103.png".format(exp.name, attribute, config)
)
exp()
'philosophers',
'psr-large',
'psr-middle',
'trucks',
]
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser("axiom_time_parser.py")
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_comparison_table_step(attributes=exp.DEFAULT_TABLE_ATTRIBUTES +
IssueConfig("cpdbs-sys3", ['--search', 'astar(cpdbs(systematic(3)))']),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser('parser.py')
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
attributes=exp.DEFAULT_TABLE_ATTRIBUTES
"lazy_greedy([heur], preferred=[heur])"
], [], []),
'lama-first': ([], [], ["--alias", "lama-first"])
}
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = [
"depot:p01.pddl", "gripper:prob01.pddl",
"psr-middle:p01-s17-n2-l2-f30.pddl"
]
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(environment=ENVIRONMENT, revisions=[], configs=[])
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_suite(BENCHMARKS_DIR_CUSTOM, SUITE_CUSTOM)
#exp.add_suite(BENCHMARKS_DIR, ["psr-middle:p01-s17-n2-l2-f30.pddl"])
for name, config in CONFIGS.items():
exp.add_algorithm(name + '-base', get_repo_base(), 'issue453-base',
config[0], config[1], config[2])
exp.add_algorithm(name + '-v3', get_repo_base(), 'issue453-v3', config[0],
config[1], config[2])
exp.add_algorithm(
name + '-v3-max-layers', get_repo_base(), 'issue453-v3',
config[0] + ['--translate-options', '--layer-strategy=max'], config[1],
config[2])
exp.add_parser(exp.EXITCODE_PARSER)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
REVS = ["issue592-base", "issue592-v3"]
SUITE = suites.suite_satisficing()
CONFIGS = [
IssueConfig("seq-sat-lama-2011", [],
driver_options=["--alias", "seq-sat-lama-2011"]),
IssueConfig("lama-first", [], driver_options=["--alias", "lama-first"]),
]
exp = IssueExperiment(revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**")
exp.add_comparison_table_step()
exp()
]),
]
SUITE = DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = MaiaEnvironment(priority=0, email='*****@*****.**')
if is_test_run():
SUITE = [
'depot:p01.pddl', 'depot:p02.pddl',
'parcprinter-opt11-strips:p01.pddl',
'parcprinter-opt11-strips:p02.pddl', 'mystery:prob07.pddl'
]
ENVIRONMENT = LocalEnvironment(processes=4)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['{ms_parser}'])
exp.add_suite(BENCHMARKS_DIR, SUITE)
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
'visitall-sat11-strips',
'visitall-sat14-strips',
'woodworking-opt08-strips',
'woodworking-opt11-strips',
'woodworking-sat08-strips',
'woodworking-sat11-strips',
'zenotravel',
]
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=4)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser("translator_additional_parser.py")
del exp.commands['remove-output-sas']
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_parse_again_step()
exp.add_fetcher(name='fetch')
CONFIGS = [
IssueConfig("{nick}-{build}".format(**locals()), ["--search", search],
build_options=[build],
driver_options=["--build", build, "--search-time-limit", "5m"])
for nick, search in SEARCHES for build in BUILDS
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
attributes = IssueExperiment.DEFAULT_TABLE_ATTRIBUTES
# Compare revisions.
for build in BUILDS:
for rev1, rev2 in itertools.combinations(REVISIONS, 2):
algorithm_pairs = [("{rev1}-{config_nick}-{build}".format(**locals()),
"{rev2}-{config_nick}-{build}".format(**locals()),
"Diff ({config_nick}-{build})".format(**locals()))
for config_nick, search in SEARCHES]
# Test sampling
IssueConfig('ipdb', ['--search', 'astar(ipdb)']),
# Test genetic pattern generation
IssueConfig('genetic', ['--search', 'astar(zopdbs(patterns=genetic))']),
# Test cegar
IssueConfig(
"cegar-10K-goals-randomorder",
["--search", "astar(cegar(subtasks=[goals(order=random)],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-10K-original-randomorder",
["--search", "astar(cegar(subtasks=[original],max_states=10000,max_time=infinity,pick=random))"]),
]
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**"
)
exp.add_fetcher('data/issue648-v1-opt-test', parsers=['parser.py'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
out_of_memory = Attribute('out_of_memory', absolute=True, min_wins=True)
out_of_time = Attribute('out_of_time', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
out_of_memory,
driver_options=DRIVER_OPTIONS),
IssueConfig("lama-first", [],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS + ["--alias", "lama-first"]),
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
for attribute in ["total_time"]:
for config in CONFIGS:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_algorithm=[
"{}-{}".format(rev, config.nick) for rev in REVISIONS
],
get_category=lambda run1, run2: run1.get("domain"),
),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = [
#"depot:p02.pddl",
"gripper:prob01.pddl"]
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
#exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser(os.path.join(DIR, "parser.py"))
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
REFINEMENT_ATTRIBUTES = [
SUITE.extend(suites.suite_ipc14_opt_strips())
CONFIGS = [
# Test label reduction, shrink_bucket_based (via shrink_fh and shrink_random)
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-r50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_random(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
# Test sampling
IssueConfig('ipdb', ['--search', 'astar(ipdb)']),
# Test genetic pattern generation
IssueConfig('genetic', ['--search', 'astar(zopdbs(patterns=genetic))']),
# Test cegar
IssueConfig(
"cegar-10K-goals-randomorder",
["--search", "astar(cegar(subtasks=[goals(order=random)],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-10K-original-randomorder",
["--search", "astar(cegar(subtasks=[original],max_states=10000,max_time=infinity,pick=random))"]),
]
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**"
)
exp.add_comparison_table_step()
exp()
REVISIONS = ["issue700-base", "issue700-v1"]
CONFIGS = [
IssueConfig("blind", ["--search", "astar(blind())"]),
IssueConfig("lmcut", ["--search", "astar(lmcut())"]),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
for attr in ["total_time", "search_time", "memory"]:
for rev1, rev2 in [("base", "v1")]:
for config_nick in ["blind", "lmcut"]:
exp.add_report(RelativeScatterPlotReport(
attributes=[attr],
filter_algorithm=[
"issue700-%s-%s" % (rev1, config_nick),
"issue700-%s-%s" % (rev2, config_nick)
build_options=[build],
driver_options=["--build", build, "--search-time-limit", "1m"])
for nick, search in SEARCHES
for build in BUILDS
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = MaiaEnvironment(
priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
# Compare revisions.
for build in BUILDS:
for rev1, rev2 in itertools.combinations(REVISIONS, 2):
algorithm_pairs = [
("{rev1}-{config_nick}-{build}".format(**locals()),
"{rev2}-{config_nick}-{build}".format(**locals()),
"Diff ({config_nick}-{build})".format(**locals()))
for config_nick, search in SEARCHES]
exp.add_report(
ComparativeReport(
algorithm_pairs,
def main(revisions=[]):
suite = suites.suite_optimal_with_ipc11()
configs = {
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v1-eval',filter_config=[
'issue604-base-rl-b50k',
'issue604-base-cggl-b50k',
'issue604-base-dfp-b50k',
'issue604-base-rl-ginf',
'issue604-base-cggl-ginf',
'issue604-base-dfp-ginf',
'issue604-base-rl-f50k',
'issue604-base-cggl-f50k',
'issue604-base-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-eval',filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-rest-eval',filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'),
('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'),
('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'),
('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'),
('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'),
('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'),
('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'),
('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'),
('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'),
],attributes=attributes),outfile=os.path.join(
exp.eval_dir, 'issue604-base-v7-comparison.html'))
exp()
)
]
SUITE = list(
sorted(
set(common_setup.DEFAULT_OPTIMAL_SUITE)
| set(common_setup.DEFAULT_SATISFICING_SUITE)))
ENVIRONMENT = MaiaEnvironment(priority=0,
email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_resource('sg_parser', 'sg-parser.py', dest='sg-parser.py')
exp.add_command('sg-parser', ['{sg_parser}'])
exp.add_absolute_report_step(attributes=[
Attribute(
"sg_construction_time", functions=[arithmetic_mean], min_wins=True),
Attribute("sg_peak_mem_diff", functions=[arithmetic_mean], min_wins=True),
Attribute("sg_counts_empty", functions=[arithmetic_mean], min_wins=True),
Attribute(
"sg_counts_leaf_empty", functions=[arithmetic_mean], min_wins=True),
Attribute(
for build in BUILDS
for search_nick, search in SEARCHES
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
#exp.add_absolute_report_step()
#exp.add_comparison_table_step()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
#IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
#IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
#IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
#IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
#IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
CONFIGS.append(IssueConfig("blind-normal-axiom-rules", ["--search", "astar(blind)"]))
CONFIGS.append(IssueConfig("blind-overapprox-axiom-rules", ["--search", "astar(blind)",
"--translate-options", "--overapproximate-axioms"]),)
SUITE = DERIVED_VARIABLES_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_1",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = ["depot:p01.pddl", "gripper:prob01.pddl", "psr-middle:p01-s17-n2-l2-f30.pddl"]
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser("parser.py")
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
attributes = (['translator_axioms',
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
configs = [
IssueConfig(
"cegar-10K-original",
["--search", "astar(cegar(subtasks=[original()],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-10K-landmarks-goals",
["--search", "astar(cegar(subtasks=[landmarks(), goals()],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-900s-landmarks-goals",
["--search", "astar(cegar(subtasks=[landmarks(), goals()],max_states=infinity,max_time=900))"]),
]
exp = IssueExperiment(
revisions=["issue633-v1", "issue633-v2"],
configs=configs,
suite=suites.suite_optimal_with_ipc11(),
test_suite=["depot:pfile1"],
email="*****@*****.**",
)
exp.add_comparison_table_step()
exp()
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue752-base"]
CONFIGS = [
IssueConfig('astar-blind', ["--search", "astar(blind())"],
build_options=["release64"], driver_options=["--build", "release64"]),
IssueConfig('astar-seq-cplex1251', ["--search", "astar(operatorcounting([state_equation_constraints()], lpsolver=cplex))"],
build_options=["release64"], driver_options=["--build", "release64"]),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.run_steps()
"astar({heuristic}(), pruning=stubborn_sets_{pruning}())".format(
**locals())
]) for heuristic in ["blind", "lmcut"] for pruning in ["queue"]
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser('lab_driver_parser', exp.LAB_DRIVER_PARSER)
exp.add_parser('exitcode_parser', exp.EXITCODE_PARSER)
#exp.add_parser('translator_parser', exp.TRANSLATOR_PARSER)
exp.add_parser('single_search_parser', exp.SINGLE_SEARCH_PARSER)
exp.add_parser('pruning_parser',
os.path.join(common_setup.get_script_dir(), "parser.py"))
exp.add_absolute_report_step(
attributes=IssueExperiment.DEFAULT_TABLE_ATTRIBUTES +
["time_for_pruning_operators"])
exp.add_report(
def main(revisions=None):
suite = suites.suite_optimal_strips()
suite.extend(suites.suite_ipc14_opt_strips())
# only DFP configs
configs = {
# label reduction with seed 2016
IssueConfig('dfp-b50k-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false,random_seed=2016)))'
]),
IssueConfig('dfp-ginf-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false,random_seed=2016)))'
]),
IssueConfig('dfp-f50k-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true,random_seed=2016)))'
]),
# shrink fh/rnd with seed 2016
IssueConfig('dfp-f50ks2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('dfp-rnd50ks2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_random(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
# shrink fh/rnd with seed 2016 and with label reduction with seed 2016
IssueConfig('dfp-f50ks2016-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true,random_seed=2016)))'
]),
IssueConfig('dfp-rnd50ks2016-lrs2016', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_random(max_states=50000,random_seed=2016),label_reduction=exact(before_shrinking=false,before_merging=true,random_seed=2016)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step()
exp()
driver_options=["--build", build, "--overall-time-limit", "30m"])
for rev in REVISIONS for build in BUILDS
for config_nick, config in CONFIG_DICT.items()
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_step("build", exp.build)
exp.add_step("start", exp.start_runs)
exp.add_fetcher(name="fetch")
#exp.add_absolute_report_step()
#exp.add_comparison_table_step()
SUITE = suites.suite_optimal()
CONFIGS = [
IssueConfig("seq-opt-bjolp", [], driver_options=["--alias", "seq-opt-bjolp"]),
IssueConfig("seq-opt-bjolp-ocp", [
"--landmarks", "lm=lm_merged([lm_rhw(),lm_hm(m=1)])",
"--heuristic", "hlm=lmcount(lm,admissible=true,optimal=true)",
"--search", "astar(hlm,mpd=true)"]),
]
ENVIRONMENT = MaiaEnvironment(
priority=0, email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
environment=ENVIRONMENT
)
exp.add_suite(BENCHMARKS, SUITE)
exp.add_comparison_table_step()
exp.add_comparison_table_step(attributes=["memory","total_time", "search_time", "landmarks_generation_time"])
exp.add_scatter_plot_step(relative=True, attributes=["memory","total_time", "search_time", "landmarks_generation_time"])
exp()
def main(revisions=None):
benchmarks_dir = os.environ["DOWNWARD_BENCHMARKS_IPC2018"]
# optimal union satisficing
suite = [
'briefcaseworld', 'cavediving-14-adl', 'citycar-sat14-adl',
'fsc-blocks', 'fsc-grid-a1', 'fsc-grid-a2', 'fsc-grid-r', 'fsc-hall',
'fsc-visualmarker', 'gedp-ds2ndp', 'miconic-simpleadl', 't0-adder',
't0-coins', 't0-comm', 't0-grid-dispose', 't0-grid-push',
't0-grid-trash', 't0-sortnet', 't0-sortnet-alt', 't0-uts',
"agricola-sat18", "caldera-sat18", "caldera-split-sat18",
"data-network-sat18", "flashfill-sat18", "nurikabe-sat18",
"organic-synthesis-sat18", "organic-synthesis-split-sat18",
"settlers-sat18", "snake-sat18", "spider-sat18", "termes-sat18"
]
environment = OracleGridEngineEnvironment(queue='all.q')
BUILD_OPTIONS = ["release64"]
DRIVER_OPTIONS = [
"--build", "release64", "--overall-time-limit", "30m",
"--overall-memory-limit", "4096M"
]
configs = {
IssueConfig('rb-multiply-out-ce', [
'--heuristic',
'hrb=RB(dag=from_coloring, extract_plan=true, transform=multiply_out_conditional_effects())',
'--search', 'lazy_greedy([hrb],reopen_closed=false)'
],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
environment=environment,
)
exp.add_suite(benchmarks_dir, suite)
exp.add_parser(exp.LAB_STATIC_PROPERTIES_PARSER)
exp.add_parser(exp.LAB_DRIVER_PARSER)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
#exp.add_parser(exp.PLANNER_PARSER)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_absolute_report_step(attributes=attributes)
algorithm_nicks = ['translate-symm-stabgoal-stabinit']
OTHER_REV = ''
exp.add_fetcher('data/2018-06-22-lifted-stabinit-stabgoal-order-mk-eval',
filter_algorithm=[
'{}-{}'.format(OTHER_REV, x) for x in algorithm_nicks
])
exp.add_report(
ComparativeReport(
algorithm_pairs=[('{}-{}'.format(OTHER_REV,
x), '{}-{}'.format(REVISION, x))
for x in algorithm_nicks],
attributes=attributes,
),
outfile=os.path.join(exp.eval_dir, 'a' + exp.name + '-compare.html'),
)
exp.run_steps()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('sbf-miasm-rl-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-otn-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-nto-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-rnd-abp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rl-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-l-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-otn-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-nto-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-rnd-rnd-pba-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('sbf-miasm-allrnd-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),single_random])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_absolute_report_step(
name='issue668-v1-abp',
filter_config=[
'%s-sbf-miasm-rl-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-rnd-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-rnd-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-otn-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-rnd-abp-b50k' % 'issue668-v1',
])
exp.add_absolute_report_step(
name='issue668-v1-pba',
filter_config=[
'%s-sbf-miasm-rl-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-rnd-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-rnd-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-otn-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-rnd-pba-b50k' % 'issue668-v1',
])
exp.add_absolute_report_step(
name='issue668-v1-paper',
filter_config=[
'%s-sbf-miasm-rl-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-abp-b50k' % 'issue668-v1',
'%s-sbf-miasm-rl-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-l-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-rnd-nto-pba-b50k' % 'issue668-v1',
'%s-sbf-miasm-allrnd-b50k' % 'issue668-v1',
])
#if matplotlib:
#for attribute in ["memory", "total_time"]:
#for config in configs:
#exp.add_report(
#RelativeScatterPlotReport(
#attributes=[attribute],
#filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
#get_category=lambda run1, run2: run1.get("domain"),
#),
#outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
#)
exp()
]),
] + [
IssueConfig('lm_merged', [
'--landmarks', 'l1=lm_exhaust()', '--landmarks', 'l2=lm_rhw()',
'--landmarks', 'l=lm_merged([l1, l2])', '--heuristic', 'h=lmcount(l)',
'--search', 'eager_greedy([h])'
]),
] + [IssueConfig("lama-first", [], driver_options=["--alias", "lama-first"])]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**")
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=4)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
revision_cache=REVISION_CACHE,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
exp.add_scatter_plot_step()
exp.run_steps()
'--search',
'astar(lmcut(), pruning=stubborn_sets_simple(min_required_pruning_ratio=0.2))'
]),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser('lab_driver_parser', exp.LAB_DRIVER_PARSER)
exp.add_parser('exitcode_parser', exp.EXITCODE_PARSER)
#exp.add_parser('translator_parser', exp.TRANSLATOR_PARSER)
exp.add_parser('single_search_parser', exp.SINGLE_SEARCH_PARSER)
#exp.add_absolute_report_step()
exp.add_comparison_table_step()
for attribute in ["memory", "total_time"]:
for config in CONFIGS:
exp.add_report(RelativeScatterPlotReport(
search,
driver_options=["--overall-time-limit", "30m"])
for rev in REVISIONS for search_nick, search in SEARCHES
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser('custom-parser.py')
exp.add_step("build", exp.build)
exp.add_step("start", exp.start_runs)
exp.add_fetcher(name="fetch")
exp.add_parse_again_step()
def main(revisions=None):
suite = SUITE_MCO14
configs = [
IssueConfig("astar_goalcount", [
"--search",
"astar(goalcount)"]),
IssueConfig("eager_greedy_ff", [
"--heuristic",
"h=ff()",
"--search",
"eager_greedy(h, preferred=h)"]),
IssueConfig("eager_greedy_add", [
"--heuristic",
"h=add()",
"--search",
"eager_greedy(h, preferred=h)"]),
IssueConfig("eager_greedy_cg", [
"--heuristic",
"h=cg()",
"--search",
"eager_greedy(h, preferred=h)"]),
IssueConfig("eager_greedy_cea", [
"--heuristic",
"h=cea()",
"--search",
"eager_greedy(h, preferred=h)"]),
IssueConfig("lazy_greedy_ff", [
"--heuristic",
"h=ff()",
"--search",
"lazy_greedy(h, preferred=h)"]),
IssueConfig("lazy_greedy_add", [
"--heuristic",
"h=add()",
"--search",
"lazy_greedy(h, preferred=h)"]),
IssueConfig("lazy_greedy_cg", [
"--heuristic",
"h=cg()",
"--search",
"lazy_greedy(h, preferred=h)"]),
IssueConfig("seq_sat_lama_2011", [], driver_options=[
"--alias", "seq-sat-lama-2011"]),
IssueConfig("seq_sat_fdss_1", [], driver_options=[
"--alias", "seq-sat-fdss-1"]),
IssueConfig("seq_sat_fdss_2", [], driver_options=[
"--alias", "seq-sat-fdss-2"]),
]
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=[
#'cavediving-sat14-adl:testing01_easy.pddl',
#'childsnack-sat14-strips:child-snack_pfile05.pddl',
#'citycar-sat14-adl:p3-2-2-0-1.pddl',
#'ged-sat14-strips:d-3-6.pddl',
'hiking-sat14-strips:ptesting-1-2-7.pddl',
#'maintenance-sat14-adl:maintenance-1-3-060-180-5-000.pddl',
#'tetris-sat14-strips:p020.pddl',
#'thoughtful-sat14-strips:bootstrap-typed-01.pddl',
#'transport-sat14-strips:p01.pddl',
],
processes=4,
email='*****@*****.**',
)
exp.add_absolute_report_step()
exp()
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
REVS = ["issue481-base", "issue481-v1"]
SUITE = suites.suite_optimal_with_ipc11()
CONFIGS = [
# Greedy (tests single and alternating open lists)
IssueConfig("astar_lmcut", [
"--search",
"astar(lmcut())"
]),
]
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**"
)
exp.add_comparison_table_step()
exp()