def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())'],
driver_options=['--search-time-limit', '5m']),
}
exp = IssueExperiment(
benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks'),
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
attribute = "total_time"
config_nick = 'astar-blind'
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}".format(rev, config_nick) for rev in revisions],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(exp.name, attribute, config_nick)
)
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks')
suite = [
'airport',
'depot',
'driverlog',
'elevators-opt08-strips',
'elevators-opt11-strips',
'freecell',
'hiking-opt14-strips',
'pipesworld-tankage',
]
configs = {
IssueConfig(
'astar-blind-ssec',
['--search', 'astar(blind(), pruning=stubborn_sets_ec())']),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
exp()
def main(revisions=None):
benchmarks_dir=os.path.expanduser('~/projects/downward/benchmarks')
suite=suites.suite_optimal_strips()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-blind-sss', ['--search', 'astar(blind(), pruning=stubborn_sets_simple())']),
IssueConfig('astar-blind-ssec', ['--search', 'astar(blind(), pruning=stubborn_sets_ec())']),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(
RelativeScatterPlotReport(
attributes=[attribute],
filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta', absolute=False, min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/projects/downward/benchmarks')
suite = [
'assembly',
'miconic-fulladl',
'openstacks',
'openstacks-sat08-adl',
'optical-telegraphs',
'philosophers',
'psr-large',
'psr-middle',
'trucks',
]
configs = {
IssueConfig('lazy-greedy-ff', [
'--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)'
]),
IssueConfig('lama-first', [], driver_options=['--alias',
'lama-first']),
IssueConfig('eager_greedy_cg', [
'--heuristic', 'h=cg()', '--search', 'eager_greedy(h, preferred=h)'
]),
IssueConfig('eager_greedy_cea', [
'--heuristic', 'h=cea()', '--search',
'eager_greedy(h, preferred=h)'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
IssueConfig('astar-ipdb', ['--search', 'astar(ipdb())']),
IssueConfig('astar-cegar-original',
['--search', 'astar(cegar(subtasks=[original()]))']),
IssueConfig(
'astar-cegar-lm-goals',
['--search', 'astar(cegar(subtasks=[landmarks(),goals()]))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=[
"issue627-base-%s" % config.nick,
"issue627-v2-%s" % config.nick
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue627_base_v2_memory_%s.png' % config.nick)
exp.add_report(RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=[
"issue627-base-%s" % config.nick,
"issue627-v2-%s" % config.nick
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue627_base_v2_total_time_%s.png' %
config.nick)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
exp.add_report(
RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue499-base-astar-lmcut", "issue499-v1-astar-lmcut"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue499_base_v1_memory.png'
)
exp.add_report(
RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue499-base-astar-lmcut", "issue499-v1-astar-lmcut"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue499_base_v1_total_time.png'
)
exp.add_report(
RelativeScatterPlotReport(
attributes=["expansions_until_last_jump"],
filter_config=["issue499-base-astar-lmcut", "issue499-v1-astar-lmcut"],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue499_base_v1_expansions_until_last_jump.png'
)
exp()
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('astar-blind', ['--search', 'astar(blind())']),
IssueConfig('astar-lmcut', ['--search', 'astar(lmcut())']),
IssueConfig('astar-ipdb', ['--search', 'astar(ipdb())']),
IssueConfig('astar-seq_opt_bjolp', [
'--search',
'astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true), mpd=true)'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
nick = config.nick
exp.add_report(RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue416-base-%s" % nick,
"issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_memory_%s.png' % nick)
exp.add_report(RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue416-base-%s" % nick,
"issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_total_time_%s.png' % nick)
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_all()
configs = {
IssueConfig('blind', ['--search', 'astar(blind())'],
driver_options=['--search-time-limit', '60s']),
IssueConfig('lama-first', [],
driver_options=[
'--alias', 'lama-first', '--search-time-limit', '60s'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl', 'gripper:prob01.pddl'],
processes=4,
email='*****@*****.**',
)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.append('translator_*')
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
suite = suites.suite_satisficing_with_ipc11()
configs = {
IssueConfig('seq_sat_lama_2011', [], driver_options=['--alias', 'seq-sat-lama-2011']),
IssueConfig('lama_first', [], driver_options=['--alias', 'lama-first']),
IssueConfig('ehc_lm_zhu', ['--search', 'ehc(lmcount(lm_zg()))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_comparison_table_step()
for config in configs:
nick = config.nick
exp.add_report(
RelativeScatterPlotReport(
attributes=["memory"],
filter_config=["issue416-base-%s" % nick, "issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_memory_%s.png' % nick
)
exp.add_report(
RelativeScatterPlotReport(
attributes=["total_time"],
filter_config=["issue416-base-%s" % nick, "issue416-v1-%s" % nick],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile='issue416_base_v1_total_time_%s.png' % nick
)
exp()
extra_attributes = [
perfect_heuristic,
ms_construction_time,
ms_atomic_construction_time,
ms_abstraction_constructed,
ms_atomic_fts_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step(attributes=attributes)
exp.add_scatter_plot_step(attributes=[ms_atomic_construction_time])
for algo_nick in ['dfp-b50k']: # 'rl-b50k', 'sbmiasm-b50k', 'sccs-dfp-b50k']:
algo = "issue851-v2-{}".format(algo_nick)
exp.add_report(
GeneralScatterPlotReport(
x_algo=algo,
y_algo=algo,
x_attribute='ms_atomic_construction_time',
y_attribute='total_time',
filter_algorithm=[algo],
attributes=['total_time'],
get_category=lambda run1, run2: run1["domain"],
),
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
configs = {
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'
]),
IssueConfig('dfp-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=infinity,threshold_before_merge=1))'
]),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(),label_reduction=exact(before_shrinking=false,before_merging=true),max_states=50000))'
]),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
CONFIGS = [
IssueConfig("blind", ["--search", "astar(blind())"],
driver_options=["--search-time-limit", "60s"])
]
sys.path.append(BENCHMARKS_DIR)
import suites
SUITE = suites.suite_optimal_strips()
ENVIRONMENT = MaiaEnvironment(priority=0,
email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_command("parser", ["custom-parser.py"])
exp.add_comparison_table_step(
attributes=exp.DEFAULT_TABLE_ATTRIBUTES +
["successor_generator_time", "reopened_until_last_jump"])
exp.add_scatter_plot_step(attributes=["successor_generator_time"])
exp()
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
# TODO: remove this filter when re-running experiments
def check_atomic_fts_constructed(run):
ms_atomic_construction_time = run.get('ms_atomic_construction_time')
ms_atomic_fts_constructed = False
if ms_atomic_construction_time is not None:
ms_atomic_fts_constructed = True
run['ms_atomic_fts_constructed'] = ms_atomic_fts_constructed
return run
exp.add_comparison_table_step(attributes=attributes,filter=[check_atomic_fts_constructed])
exp.add_scatter_plot_step(attributes=[ms_atomic_construction_time])
for algo_nick in ['dfp-b50k', 'rl-b50k', 'sccs-dfp-b50k']:
algo = "issue851-v1-{}".format(algo_nick)
exp.add_report(
GeneralScatterPlotReport(
x_algo = algo,
y_algo = algo,
x_attribute='ms_atomic_construction_time',
y_attribute='total_time',
filter_algorithm=[algo],
attributes=['total_time'],
get_category=lambda run1, run2: run1["domain"],
),
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser(os.path.join(DIR, "parser.py"))
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
#exp.add_absolute_report_step()
exp.add_comparison_table_step(
attributes=IssueExperiment.DEFAULT_TABLE_ATTRIBUTES +
["search_start_time", "search_start_memory", "init_time"])
for attribute in [
"memory", "total_time", "init_time", "expansions_until_last_jump"
]:
for config in CONFIGS:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_algorithm=[
"{}-{}".format(rev, config.nick) for rev in REVISIONS
],
get_category=lambda run1, run2: run1.get("domain")),
outfile="{}-{}-{}-{}-{}.png".format(
exp.name, attribute, config.nick, *REVISIONS))
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
REVS = ["issue481-base", "issue481-v1"]
SUITE = suites.suite_optimal_with_ipc11()
CONFIGS = [
# Greedy (tests single and alternating open lists)
IssueConfig("astar_lmcut", [
"--search",
"astar(lmcut())"
]),
]
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
suite=SUITE,
email="*****@*****.**"
)
exp.add_comparison_table_step()
exp()
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser("axiom_time_parser.py")
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_comparison_table_step(attributes=exp.DEFAULT_TABLE_ATTRIBUTES +
["axiom_time_inner", "axiom_time_outer"])
for attribute in ["axiom_time_inner", "axiom_time_outer"]:
for config in CONFIGS:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_algorithm=[
"{}-{}".format(rev, config.nick) for rev in REVISIONS
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}-{}-{}.png".format(
exp.name, attribute, config.nick, *REVISIONS))
exp.run_steps()
def main(revisions=None):
suite=suites.suite_optimal_strips()
suite.extend(suites.suite_ipc14_opt_strips())
configs = {
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
ms_dfp_nogoalrelevantpair_counter = Attribute('ms_dfp_nogoalrelevantpair_counter', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
ms_dfp_nogoalrelevantpair_counter,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
#for attribute in ["memory", "total_time"]:
#for config in configs:
#exp.add_report(
#RelativeScatterPlotReport(
#attributes=[attribute],
#filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
#get_category=lambda run1, run2: run1.get("domain"),
#),
#outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
#)
exp()
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
# dummy configs with correct names so that comparison report works
configs = {
IssueConfig('rl-b50k', []),
IssueConfig('cggl-b50k', []),
IssueConfig('dfp-b50k', []),
IssueConfig('rl-ginf', []),
IssueConfig('cggl-ginf', []),
IssueConfig('dfp-ginf', []),
IssueConfig('rl-f50k', []),
IssueConfig('cggl-f50k', []),
IssueConfig('dfp-f50k', []),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue655-base-eval')
exp.add_fetcher('data/issue655-v1-eval')
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
def main(revisions=None):
benchmarks_dir=os.path.expanduser('~/repos/downward/benchmarks')
suite=suites.suite_optimal_strips()
configs = {
#IssueConfig('dfp-reg-otn-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-reg-nto-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-reg-rnd-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-inv-otn-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-inv-nto-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-inv-rnd-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-rnd-otn-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-rnd-nto-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
#IssueConfig('dfp-rnd-rnd-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=true),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-reg-otn-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-reg-nto-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-reg-rnd-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=regular,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-inv-otn-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-inv-nto-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-inv-rnd-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=inverse,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-rnd-otn-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=old_to_new,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-rnd-nto-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
IssueConfig('dfp-rnd-rnd-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp(atomic_ts_order=random,product_ts_order=random,atomic_before_product=false),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic', absolute=True, min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability', absolute=True, min_wins=False)
actual_search_time = Attribute('actual_search_time', absolute=False, min_wins=True, functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time', absolute=False, min_wins=True, functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed', absolute=True, min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory', absolute=True, min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory', absolute=True, min_wins=True)
search_out_of_time = Attribute('search_out_of_time', absolute=True, min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue644-v2-dfp-tiebreaking-eval',
filter_config=[
'issue644-v1-dfp-reg-otn-pba-b50k',
'issue644-v1-dfp-reg-nto-pba-b50k',
'issue644-v1-dfp-reg-rnd-pba-b50k',
'issue644-v1-dfp-inv-otn-pba-b50k',
'issue644-v1-dfp-inv-nto-pba-b50k',
'issue644-v1-dfp-inv-rnd-pba-b50k',
'issue644-v1-dfp-rnd-otn-pba-b50k',
'issue644-v1-dfp-rnd-nto-pba-b50k',
'issue644-v1-dfp-rnd-rnd-pba-b50k',
'issue644-v2-dfp-reg-otn-pba-b50k',
'issue644-v2-dfp-reg-nto-pba-b50k',
'issue644-v2-dfp-reg-rnd-pba-b50k',
'issue644-v2-dfp-inv-otn-pba-b50k',
'issue644-v2-dfp-inv-nto-pba-b50k',
'issue644-v2-dfp-inv-rnd-pba-b50k',
'issue644-v2-dfp-rnd-otn-pba-b50k',
'issue644-v2-dfp-rnd-nto-pba-b50k',
'issue644-v2-dfp-rnd-rnd-pba-b50k',
])
exp.add_comparison_table_step()
#if matplotlib:
#for attribute in ["memory", "total_time"]:
#for config in configs:
#exp.add_report(
#RelativeScatterPlotReport(
#attributes=[attribute],
#filter_config=["{}-{}".format(rev, config.nick) for rev in revisions],
#get_category=lambda run1, run2: run1.get("domain"),
#),
#outfile="{}-{}-{}.png".format(exp.name, attribute, config.nick)
#)
exp()
from common_setup import IssueConfig, IssueExperiment
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue714-base", "issue714-v1"]
CONFIGS = [
IssueConfig(alias, [], driver_options=["--alias", alias])
for alias in [
"seq-sat-fdss-1", "seq-sat-fdss-2", "seq-sat-fdss-2014",
"seq-sat-fd-autotune-1", "seq-sat-fd-autotune-2"]
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = MaiaEnvironment(
priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_absolute_report_step(attributes=IssueExperiment.PORTFOLIO_ATTRIBUTES)
exp.add_comparison_table_step(attributes=IssueExperiment.PORTFOLIO_ATTRIBUTES)
exp.run_steps()
"operatorcounting([pho_constraints(patterns=systematic(3))])",
"operatorcounting([pho_constraints(patterns=systematic(4))])",
]
]
sys.path.append(BENCHMARKS_DIR)
import suites
SUITE = suites.suite_optimal_strips()
ENVIRONMENT = MaiaEnvironment(
priority=0, email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_command("parser", ["custom-parser.py"])
exp.add_comparison_table_step(
attributes=exp.DEFAULT_TABLE_ATTRIBUTES +
["num_sga_patterns", "num_interesting_patterns"])
exp.add_scatter_plot_step(attributes=["total_time"])
exp()
IssueConfig("{nick}-{build}".format(**locals()), ["--search", search],
build_options=[build],
driver_options=["--build", build]) for nick, search in SEARCHES
for build in BUILDS
]
ATTRIBUTES = [
"coverage", "error", "expansions_until_last_jump", "memory",
"score_memory", "total_time", "score_total_time",
"int_hash_set_load_factor", "int_hash_set_resizes"
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_command('run-custom-parser', [os.path.join(DIR, 'custom-parser.py')])
exp.add_comparison_table_step(attributes=ATTRIBUTES)
for relative in [False, True]:
exp.add_scatter_plot_step(relative=relative,
attributes=["memory", "total_time"])
exp.run_steps()
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
exp.add_parser(exp.PLANNER_PARSER)
exp.add_parser(os.path.join(DIR, "parser.py"))
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
#exp.add_absolute_report_step()
exp.add_comparison_table_step(
attributes=IssueExperiment.DEFAULT_TABLE_ATTRIBUTES +
["simplify_before", "simplify_after", "simplify_time"])
for attribute in ["memory", "total_time"]:
for config in CONFIGS:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_algorithm=[
"{}-{}".format(rev, config.nick) for rev in REVISIONS
],
get_category=lambda run1, run2: run1.get("domain")),
outfile="{}-{}-{}-{}-{}.png".format(
exp.name, attribute, config.nick, *REVISIONS))
exp.run_steps()
SUITE = suites.suite_optimal()
CONFIGS = [
IssueConfig("seq-opt-bjolp", [], driver_options=["--alias", "seq-opt-bjolp"]),
IssueConfig("seq-opt-bjolp-ocp", [
"--landmarks", "lm=lm_merged([lm_rhw(),lm_hm(m=1)])",
"--heuristic", "hlm=lmcount(lm,admissible=true,optimal=true)",
"--search", "astar(hlm,mpd=true)"]),
]
ENVIRONMENT = MaiaEnvironment(
priority=0, email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVS,
configs=CONFIGS,
environment=ENVIRONMENT
)
exp.add_suite(BENCHMARKS, SUITE)
exp.add_comparison_table_step()
exp.add_comparison_table_step(attributes=["memory","total_time", "search_time", "landmarks_generation_time"])
exp.add_scatter_plot_step(relative=True, attributes=["memory","total_time", "search_time", "landmarks_generation_time"])
exp()
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
exp.add_suite(BENCHMARKS_DIR, SUITE)
#exp.add_absolute_report_step()
exp.add_comparison_table_step()
for attribute in ["memory", "total_time"]:
for config in CONFIGS:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_algorithm=[
"{}-{}".format(rev, config.nick) for rev in REVISIONS
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}-{}-{}.png".format(
exp.name, attribute, config.nick, *REVISIONS))
exp.run_steps()
def filter_tasks_with_equal_values(self, run):
values = self._tasks_to_values[self._get_task(run)]
return len(set(values)) != 1
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_parse_again_step()
exp.add_fetcher(name='fetch')
ATTRIBUTES = ["error", "run_dir", "translator_*", "translator_output_sas_hash"]
# exp.add_absolute_report_step(
# outfile=os.path.join(exp.eval_dir, "{EXPNAME}.html".format(**locals())),
# attributes=ATTRIBUTES)
exp.add_comparison_table_step(attributes=ATTRIBUTES)
same_value_filters = SameValueFilters("translator_output_sas_hash")
# exp.add_absolute_report_step(
# outfile=os.path.join(exp.eval_dir, "{EXPNAME}-filtered.html".format(**locals())),
# attributes=ATTRIBUTES,
# filter=[same_value_filters.store_values, same_value_filters.filter_tasks_with_equal_values])
exp.add_comparison_table_step(
suffix="filtered",
attributes=ATTRIBUTES,
filter=[
same_value_filters.store_values,
same_value_filters.filter_tasks_with_equal_values
])
exp.add_report(TranslatorDiffReport(
