def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
IssueConfig('astar-ipdb', ['--search', 'astar(ipdb())']),
IssueConfig('astar-seq_opt_bjolp', [
'--search',
'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true), mpd=true)'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
nick = config.nick
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue416-base-%s" % nick,
"issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_memory_%s.png' % nick)
exp.add_report(RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue416-base-%s" % nick,
"issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_total_time_%s.png' % nick)
exp()
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
import common_setup
exp = common_setup.IssueExperiment(
search_revisions=["issue422-base", "issue422-v1"],
configs={"lmcut": ["--search", "astar(lmcut())"]},
suite=suites.suite_optimal_with_ipc11(),
)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
exp.add_scatter_plot_step()
exp()
def main(revisions=None):
SUITE = suites.suite_optimal_with_ipc11()
B_CONFIGS = {
'rl-b50k': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'
],
'cggl-b50k': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'
],
'dfp-b50k': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'
],
}
G_CONFIGS = {
'rl-ginf': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'
],
'cggl-ginf': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'
],
'dfp-ginf': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'
],
}
F_CONFIGS = {
'rl-f50k': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'
],
'cggl-f50k': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'
],
'dfp-f50k': [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'
],
}
CONFIGS = dict(B_CONFIGS)
CONFIGS.update(G_CONFIGS)
CONFIGS.update(F_CONFIGS)
exp = common_setup.IssueExperiment(
revisions=revisions,
configs=CONFIGS,
suite=SUITE,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step(attributes=attributes)
exp()
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
import common_setup
CONFIGS = {
'astar_lmcount_lm_merged_rhw_hm': [
'--search',
'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true),mpd=true)'],
}
exp = common_setup.IssueExperiment(
search_revisions=["issue468-base", "issue468-v1"],
configs=CONFIGS,
suite=suites.suite_optimal_with_ipc11(),
)
exp.add_comparison_table_step()
exp()
from downward.configs import default_configs_optimal
from downward.reports.scatter import ScatterPlotReport
import common_setup
REVS = ["issue214-base", "issue214-v5"]
CONFIGS = {"blind": ["--search", "astar(blind())"]}
TEST_RUN = False
if TEST_RUN:
SUITE = "gripper:prob01.pddl"
PRIORITY = None # "None" means local experiment
else:
SUITE = list(sorted(set(suites.suite_all()) - set(suites.suite_optimal_with_ipc11())))
PRIORITY = 0 # number means maia experiment
exp = common_setup.MyExperiment(
grid_priority=PRIORITY,
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
parsers=['state_size_parser.py'],
)
exp.add_comparison_table_step(
attributes=common_setup.MyExperiment.DEFAULT_TABLE_ATTRIBUTES + ['bytes_per_state', 'variables', 'state_var_t_size']
)
def main(rev1, rev2):
REVS = [rev1, rev2]
LIMITS = {"search_time": 1800}
SUITE = suites.suite_optimal_with_ipc11()
B_CONFIGS = {
'rl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'],
'cggl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'],
'dfp-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'],
}
G_CONFIGS = {
'rl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'],
'cggl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'],
'dfp-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))'],
}
F_CONFIGS = {
'rl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'],
'cggl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'],
'dfp-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))'],
}
CONFIGS = dict(B_CONFIGS)
CONFIGS.update(G_CONFIGS)
CONFIGS.update(F_CONFIGS)
exp = common_setup.IssueExperiment(
search_revisions=REVS,
configs=CONFIGS,
suite=SUITE,
limits=LIMITS,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_search_parser('ms-parser.py')
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
actual_search_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step(attributes=attributes)
exp()
def main(revisions=[]):
suite = suites.suite_optimal_with_ipc11()
configs = {}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v1-eval',
filter_config=[
'issue604-base-rl-b50k',
'issue604-base-cggl-b50k',
'issue604-base-dfp-b50k',
'issue604-base-rl-ginf',
'issue604-base-cggl-ginf',
'issue604-base-dfp-ginf',
'issue604-base-rl-f50k',
'issue604-base-cggl-f50k',
'issue604-base-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-eval',
filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-rest-eval',
filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'),
('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'),
('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'),
('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'),
('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'),
('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'),
('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'),
('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'),
('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-base-v7-comparison.html'))
exp()
def main(revisions=None):
SUITE = suites.suite_optimal_with_ipc11()
B_CONFIGS = {
"dfp-b50k": [
"--search",
"astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))",
]
}
G_CONFIGS = {
"dfp-ginf": [
"--search",
"astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=label_reduction(before_shrinking=true,before_merging=false)))",
]
}
F_CONFIGS = {
"dfp-f50k": [
"--search",
"astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=label_reduction(before_shrinking=false,before_merging=true)))",
]
}
CONFIGS = dict(B_CONFIGS)
CONFIGS.update(G_CONFIGS)
CONFIGS.update(F_CONFIGS)
exp = common_setup.IssueExperiment(
revisions=revisions,
configs=CONFIGS,
suite=SUITE,
test_suite=["depot:pfile1"],
processes=4,
email="*****@*****.**",
)
exp.add_resource("ms_parser", "ms-parser.py", dest="ms-parser.py")
exp.add_command("ms-parser", ["ms_parser"])
# planner outcome attributes
perfect_heuristic = Attribute("perfect_heuristic", absolute=True, min_wins=False)
proved_unsolvability = Attribute("proved_unsolvability", absolute=True, min_wins=False)
actual_search_time = Attribute("actual_search_time", absolute=False, min_wins=True, functions=[gm])
initial_h_value = Attribute("initial_h_value", absolute=False, min_wins=False)
# m&s attributes
ms_construction_time = Attribute("ms_construction_time", absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute("ms_abstraction_constructed", absolute=True, min_wins=False)
ms_final_size = Attribute("ms_final_size", absolute=False, min_wins=True)
ms_out_of_memory = Attribute("ms_out_of_memory", absolute=True, min_wins=True)
ms_out_of_time = Attribute("ms_out_of_time", absolute=True, min_wins=True)
search_out_of_memory = Attribute("search_out_of_memory", absolute=True, min_wins=True)
search_out_of_time = Attribute("search_out_of_time", absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
initial_h_value,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step(attributes=attributes)
exp()
def main(revisions=[]):
suite = suites.suite_optimal_with_ipc11()
configs = {
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v1-eval',filter_config=[
'issue604-base-rl-b50k',
'issue604-base-cggl-b50k',
'issue604-base-dfp-b50k',
'issue604-base-rl-ginf',
'issue604-base-cggl-ginf',
'issue604-base-dfp-ginf',
'issue604-base-rl-f50k',
'issue604-base-cggl-f50k',
'issue604-base-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-eval',filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_fetcher('data/issue604-v7-rest-eval',filter_config=[
'issue604-v7-rl-b50k',
'issue604-v7-cggl-b50k',
'issue604-v7-dfp-b50k',
'issue604-v7-rl-ginf',
'issue604-v7-cggl-ginf',
'issue604-v7-dfp-ginf',
'issue604-v7-rl-f50k',
'issue604-v7-cggl-f50k',
'issue604-v7-dfp-f50k',
])
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-base-rl-b50k', 'issue604-v7-rl-b50k'),
('issue604-base-cggl-b50k', 'issue604-v7-cggl-b50k'),
('issue604-base-dfp-b50k', 'issue604-v7-dfp-b50k'),
('issue604-base-rl-ginf', 'issue604-v7-rl-ginf'),
('issue604-base-cggl-ginf', 'issue604-v7-cggl-ginf'),
('issue604-base-dfp-ginf', 'issue604-v7-dfp-ginf'),
('issue604-base-rl-f50k', 'issue604-v7-rl-f50k'),
('issue604-base-cggl-f50k', 'issue604-v7-cggl-f50k'),
('issue604-base-dfp-f50k', 'issue604-v7-dfp-f50k'),
],attributes=attributes),outfile=os.path.join(
exp.eval_dir, 'issue604-base-v7-comparison.html'))
exp()
def __init__(self, path, repo, opt_or_sat, rev, base_rev=None,
use_core_configs=True, use_ipc_configs=True, use_extended_configs=False,
**kwargs):
"""
See :py:class:`DownwardExperiment <downward.experiments.DownwardExperiment>`
for inherited parameters.
The experiment will be built at *path*.
*repo* must be the path to a Fast Downward repository. This repository
is used to search for problem files.
If *opt_or_sat* is 'opt', configurations for optimal planning will be
tested on all domains suited for optimal planning. If it is 'sat',
configurations for satisficing planning will be tested on the
satisficing suite.
*rev* determines the new revision to test.
If *base_rev* is None (default), the latest revision on the branch default
that is an ancestor of *rev* will be used.
*use_core_configs* determines if the most common configurations are tested
(default: True).
*use_ipc_configs* determines if the configurations used in the IPCs are tested
(default: True).
*use_extended_configs* determines if some less common configurations are tested
(default: False).
"""
base_rev = checkouts.get_common_ancestor(repo, rev)
combos = [(Translator(repo, rev=r),
Preprocessor(repo, rev=r),
Planner(repo, rev=r))
for r in (base_rev, rev)]
DownwardExperiment.__init__(self, path, repo, combinations=combos, **kwargs)
# ------ suites and configs ------------------------------------
if opt_or_sat == 'opt':
self.add_suite(suite_optimal_with_ipc11())
configs = default_configs_optimal(use_core_configs,
use_ipc_configs,
use_extended_configs)
elif opt_or_sat == 'sat':
self.add_suite(suite_satisficing_with_ipc11())
configs = default_configs_satisficing(use_core_configs,
use_ipc_configs,
use_extended_configs)
else:
logging.critical('Select to test either \'opt\' or \'sat\' configurations')
for nick, command in configs.items():
self.add_config(nick, command)
# ------ reports -----------------------------------------------
comparison = CompareRevisionsReport(base_rev,
rev,
attributes=COMPARED_ATTRIBUTES)
self.add_report(comparison,
name='report-compare-scores',
outfile='report-compare-scores.html')
for nick in configs.keys():
config_before = '%s-%s' % (base_rev, nick)
config_after = '%s-%s' % (rev, nick)
for attribute in SCATTER_PLOT_ATTRIBUTES:
name = 'scatter-%s-%s' % (attribute, nick)
self.add_report(
ScatterPlotReport(
filter_config=[config_before, config_after],
attributes=[attribute],
get_category=lambda run1, run2: run1['domain']),
outfile=name)
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('cggl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('rl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('cggl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('rl-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('cggl-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue604-v4-eval')
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-v3-rl-b50k', 'issue604-v5-rl-b50k'),
('issue604-v3-cggl-b50k', 'issue604-v5-cggl-b50k'),
('issue604-v3-dfp-b50k', 'issue604-v5-dfp-b50k'),
('issue604-v3-rl-ginf', 'issue604-v5-rl-ginf'),
('issue604-v3-cggl-ginf', 'issue604-v5-cggl-ginf'),
('issue604-v3-dfp-ginf', 'issue604-v5-dfp-ginf'),
('issue604-v3-rl-f50k', 'issue604-v5-rl-f50k'),
('issue604-v3-cggl-f50k', 'issue604-v5-cggl-f50k'),
('issue604-v3-dfp-f50k', 'issue604-v5-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-v3-v5-comparison.html'))
exp.add_report(CompareConfigsReport(compared_configs=[
('issue604-v4-rl-b50k', 'issue604-v5-rl-b50k'),
('issue604-v4-cggl-b50k', 'issue604-v5-cggl-b50k'),
('issue604-v4-dfp-b50k', 'issue604-v5-dfp-b50k'),
('issue604-v4-rl-ginf', 'issue604-v5-rl-ginf'),
('issue604-v4-cggl-ginf', 'issue604-v5-cggl-ginf'),
('issue604-v4-dfp-ginf', 'issue604-v5-dfp-ginf'),
('issue604-v4-rl-f50k', 'issue604-v5-rl-f50k'),
('issue604-v4-cggl-f50k', 'issue604-v5-cggl-f50k'),
('issue604-v4-dfp-f50k', 'issue604-v5-dfp-f50k'),
],
attributes=attributes),
outfile=os.path.join(exp.eval_dir,
'issue604-v4-v5-comparison.html'))
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('cggl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('rl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('cggl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('dfp-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
IssueConfig('rl-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('cggl-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue604-base-dfp-ginf", "issue604-v1-dfp-ginf"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue604_base_v1_memory_dfp.png')
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue604-base-rl-ginf", "issue604-v1-rl-ginf"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue604_base_v1_memory_rl.png')
exp()
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
import common_setup
REVS = ["issue546-base", "issue546-v1"]
LIMITS = {"search_time": 1800}
SUITE = suites.suite_optimal_with_ipc11()
CONFIGS = {
"seq_opt_fdss_1": ["--alias", "seq-opt-fdss-1"],
"seq_opt_fdss_2": ["--alias", "seq-opt-fdss-2"],
}
exp = common_setup.IssueExperiment(
search_revisions=REVS,
configs=CONFIGS,
suite=SUITE,
limits=LIMITS,
)
exp.add_comparison_table_step(
attributes=common_setup.IssueExperiment.PORTFOLIO_ATTRIBUTES)
exp()
def main(revisions=None):
SUITE = suites.suite_optimal_with_ipc11()
B_CONFIGS = {
'rl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'],
'cggl-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'],
'dfp-b50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'],
}
G_CONFIGS = {
'rl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'],
'cggl-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'],
'dfp-ginf': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'],
}
F_CONFIGS = {
'rl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'],
'cggl-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'],
'dfp-f50k': ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'],
}
CONFIGS = dict(B_CONFIGS)
CONFIGS.update(G_CONFIGS)
CONFIGS.update(F_CONFIGS)
exp = common_setup.IssueExperiment(
revisions=revisions,
configs=CONFIGS,
suite=SUITE,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue601-base-eval')
exp.add_report(CompareConfigsReport(compared_configs=[
('issue601-base-rl-b50k', 'issue601-v1-rl-b50k'),
('issue601-base-cggl-b50k', 'issue601-v1-cggl-b50k'),
('issue601-base-dfp-b50k', 'issue601-v1-dfp-b50k'),
('issue601-base-rl-ginf', 'issue601-v1-rl-ginf'),
('issue601-base-cggl-ginf', 'issue601-v1-cggl-ginf'),
('issue601-base-dfp-ginf', 'issue601-v1-dfp-ginf'),
('issue601-base-rl-f50k', 'issue601-v1-rl-f50k'),
('issue601-base-cggl-f50k', 'issue601-v1-cggl-f50k'),
('issue601-base-dfp-f50k', 'issue601-v1-dfp-f50k'),
],attributes=attributes))
exp()
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
import configs
import common_setup
REVS = ["issue436-base", "issue436-v2"]
LIMITS = {"search_time": 1800}
SUITE = suites.suite_optimal_with_ipc11()
configs_optimal_core = configs.configs_optimal_core()
CONFIGS = {}
for name in [
'astar_merge_and_shrink_greedy_bisim',
'astar_merge_and_shrink_dfp_bisim', 'astar_ipdb', 'astar_hmax',
'astar_blind', 'astar_lmcut', 'astar_merge_and_shrink_bisim',
'astar_lmcount_lm_merged_rhw_hm'
]:
CONFIGS[name] = configs_optimal_core[name]
exp = common_setup.IssueExperiment(
search_revisions=REVS,
configs=CONFIGS,
suite=SUITE,
limits=LIMITS,
)
exp.add_absolute_report_step()
exp.add_comparison_table_step()
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
import common_setup
CONFIGS = {
"astar_ipdb": ["--search", "astar(ipdb())"],
"astar_pdb": ["--search", "astar(pdb())"],
"astar_gapdb": ["--search", "astar(gapdb())"],
}
exp = common_setup.IssueExperiment(
search_revisions=["issue488-base", "issue488-v1"], configs=CONFIGS, suite=suites.suite_optimal_with_ipc11()
)
exp.add_comparison_table_step()
exp()
