import common_setup
from common_setup import IssueConfig, IssueExperiment, RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue693-v2", "issue693-v3"]
BUILDS = ["release32", "release64"]
SEARCHES = [
("ff_lazy", "lazy_greedy(ff())"),
("cg_eager", "eager_greedy(cg())"),
("lm_rhw", "lazy_greedy(lmcount(lm_rhw()))"),
]
CONFIGS = [
IssueConfig("{nick}-{build}".format(**locals()), ["--search", search],
build_options=[build],
driver_options=["--build", build]) for nick, search in SEARCHES
for build in BUILDS
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
def main(revisions=None):
benchmarks_dir = os.environ["DOWNWARD_BENCHMARKS_IPC2018"]
# optimal union satisficing
suite = [
'briefcaseworld', 'cavediving-14-adl', 'citycar-sat14-adl',
'fsc-blocks', 'fsc-grid-a1', 'fsc-grid-a2', 'fsc-grid-r', 'fsc-hall',
'fsc-visualmarker', 'gedp-ds2ndp', 'miconic-simpleadl', 't0-adder',
't0-coins', 't0-comm', 't0-grid-dispose', 't0-grid-push',
't0-grid-trash', 't0-sortnet', 't0-sortnet-alt', 't0-uts',
"agricola-sat18", "caldera-sat18", "caldera-split-sat18",
"data-network-sat18", "flashfill-sat18", "nurikabe-sat18",
"organic-synthesis-sat18", "organic-synthesis-split-sat18",
"settlers-sat18", "snake-sat18", "spider-sat18", "termes-sat18"
]
environment = OracleGridEngineEnvironment(queue='all.q')
BUILD_OPTIONS = ["release64"]
DRIVER_OPTIONS = [
"--build", "release64", "--overall-time-limit", "30m",
"--overall-memory-limit", "4096M"
]
configs = {
IssueConfig('rb-multiply-out-ce', [
'--heuristic',
'hrb=RB(dag=from_coloring, extract_plan=true, transform=multiply_out_conditional_effects())',
'--search', 'lazy_greedy([hrb],reopen_closed=false)'
],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
environment=environment,
)
exp.add_suite(benchmarks_dir, suite)
exp.add_parser(exp.LAB_STATIC_PROPERTIES_PARSER)
exp.add_parser(exp.LAB_DRIVER_PARSER)
exp.add_parser(exp.EXITCODE_PARSER)
exp.add_parser(exp.TRANSLATOR_PARSER)
exp.add_parser(exp.SINGLE_SEARCH_PARSER)
#exp.add_parser(exp.PLANNER_PARSER)
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
exp.add_step('build', exp.build)
exp.add_step('start', exp.start_runs)
exp.add_fetcher(name='fetch')
exp.add_absolute_report_step(attributes=attributes)
algorithm_nicks = ['translate-symm-stabgoal-stabinit']
OTHER_REV = ''
exp.add_fetcher('data/2018-06-22-lifted-stabinit-stabgoal-order-mk-eval',
filter_algorithm=[
'{}-{}'.format(OTHER_REV, x) for x in algorithm_nicks
])
exp.add_report(
ComparativeReport(
algorithm_pairs=[('{}-{}'.format(OTHER_REV,
x), '{}-{}'.format(REVISION, x))
for x in algorithm_nicks],
attributes=attributes,
),
outfile=os.path.join(exp.eval_dir, 'a' + exp.name + '-compare.html'),
)
exp.run_steps()
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue778-base", "issue778-v1"]
CONFIGS = [
IssueConfig('lmcut-sss-ratio-0.2', [
'--search',
'astar(lmcut(), pruning=stubborn_sets_simple(min_required_pruning_ratio=0.2))'
]),
IssueConfig('lmcut-ssec-ratio-0.2', [
'--search',
'astar(lmcut(), pruning=stubborn_sets_simple(min_required_pruning_ratio=0.2))'
]),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
cost_type=one,reopen_closed=false)"""
],
"lama-first-typed": [
"--evaluator",
"hlm=lmcount(lm_factory=lm_rhw(reasonable_orders=true),transform=adapt_costs(one),pref=false)",
"--evaluator", "hff=ff(transform=adapt_costs(one))", "--search",
"lazy(alt([single(hff), single(hff, pref_only=true),"
"single(hlm), single(hlm, pref_only=true), type_based([hff, g()])], boost=1000),"
"preferred=[hff,hlm], cost_type=one, reopen_closed=false, randomize_successors=true,"
"preferred_successors_first=false)"
],
}
CONFIGS = [
IssueConfig(
"-".join([config_nick, build]),
config,
build_options=[build],
driver_options=["--build", build, "--overall-time-limit", "30m"])
for rev in REVISIONS for build in BUILDS
for config_nick, config in CONFIG_DICT.items()
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
("blind", "astar(blind())"),
("cegar", "astar(cegar())"),
("divpot", "astar(diverse_potentials())"),
("ipdb", "astar(ipdb())"),
("lmcut", "astar(lmcut())"),
("mas",
"astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),"
" merge_strategy=merge_sccs(order_of_sccs=topological,"
" merge_selector=score_based_filtering(scoring_functions=[goal_relevance, dfp, total_order])),"
" label_reduction=exact(before_shrinking=true, before_merging=false),"
" max_states=50000, threshold_before_merge=1))"),
("seq", "astar(operatorcounting([state_equation_constraints()]))"),
]
CONFIGS = [
IssueConfig(
"-".join([search_nick, build]), ["--search", search],
build_options=[build],
driver_options=["--build", build, "--overall-time-limit", "5m"])
for rev in REVISIONS for build in BUILDS
for search_nick, search in SEARCHES
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
def main(revisions=None):
suite = suites.suite_optimal_with_ipc11()
configs = {
IssueConfig('rl-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
#IssueConfig('cggl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-b50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
#IssueConfig('rl-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('cggl-ginf', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))'
]),
#IssueConfig('dfp-ginf', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=infinity,threshold=1,greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
#IssueConfig('rl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
#IssueConfig('cggl-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=cg_goal_level),shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-f50k', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))'
]),
}
exp = IssueExperiment(
revisions=revisions,
configs=configs,
suite=suite,
test_suite=['depot:pfile1'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
ms_memory_delta = Attribute('ms_memory_delta',
absolute=False,
min_wins=True)
extra_attributes = [
search_out_of_memory,
search_out_of_time,
perfect_heuristic,
proved_unsolvability,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
ms_memory_delta,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_comparison_table_step()
exp()
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue695-base", "issue695-v2"]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
CONFIGS = [
IssueConfig("astar_blind", ["--search", "astar(blind())"]),
IssueConfig("astar_blind-64", ["--search", "astar(blind())"],
build_options=["release64"],
driver_options=["--build", "release64"]),
IssueConfig("seq-opt-bjolp", [],
driver_options=["--alias", "seq-opt-bjolp"]),
IssueConfig(
"seq-opt-bjolp-64", [],
build_options=["release64"],
driver_options=["--build", "release64", "--alias", "seq-opt-bjolp"]),
]
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue752-v1"]
CONFIGS = [
IssueConfig('astar-seq-cplex', [
"--search",
"astar(operatorcounting([state_equation_constraints()], lpsolver=cplex))"
],
build_options=["release64"],
driver_options=["--build", "release64"]),
IssueConfig('astar-seq-soplex', [
"--search",
"astar(operatorcounting([state_equation_constraints()], lpsolver=soplex))"
],
build_options=["release64"],
driver_options=["--build", "release64"]),
IssueConfig('astar-seq-pho-cplex', [
"--search",
"astar(operatorcounting([state_equation_constraints(), lmcut_constraints()], lpsolver=cplex))"
],
build_options=["release64"],
driver_options=["--build", "release64"]),
IssueConfig('astar-seq-pho-soplex', [
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue700-base", "issue700-v1"]
CONFIGS = [
IssueConfig("blind", ["--search", "astar(blind())"]),
IssueConfig("lmcut", ["--search", "astar(lmcut())"]),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, MaiaEnvironment
from lab.reports import Attribute, geometric_mean
from common_setup import IssueConfig, IssueExperiment, DEFAULT_OPTIMAL_SUITE, is_test_run
BENCHMARKS_DIR = os.path.expanduser('~/repos/downward/benchmarks')
REVISIONS = ["issue707-v4"]
CONFIGS = [
IssueConfig('rl-b50k-nopruneunreachable', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_precomputed(merge_tree=linear(variable_order=reverse_level)),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1,prune_unreachable_states=false))'
]),
IssueConfig('dfp-b50k-nopruneunreachable', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[goal_relevance,dfp,total_order])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1,prune_unreachable_states=false))'
]),
IssueConfig('sccs-dfp-b50k-nopruneunreachable', [
'--search',
'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_sccs(order_of_sccs=topological,merge_selector=score_based_filtering(scoring_functions=[goal_relevance,dfp,total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1,prune_unreachable_states=false))'
]),
IssueConfig('rl-ginf-nopruneunreachable', [
'--search',
'astar(merge_and_shrink(merge_strategy=merge_precomputed(merge_tree=linear(variable_order=reverse_level)),shrink_strategy=shrink_bisimulation(greedy=true),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=infinity,threshold_before_merge=1,prune_unreachable_states=false))'
]),
IssueConfig('dfp-ginf-nopruneunreachable', [
'--search',
"hlm=lmcount(lm_factory=lm_rhw(reasonable_orders=true), transform=adapt_costs(one), preferred_operators={pref})"
.format(**locals()), "--evaluator", "hff=ff(transform=adapt_costs(one))",
"--search", "lazy_greedy([hff,hlm],preferred=[hff,hlm],"
"cost_type=one,reopen_closed=false)"
]) for pref in ["none", "simple"]] + [("lama-first-pref-{pref}-ignore".format(
**locals()
), [
"--evaluator",
"hlm=lmcount(lm_factory=lm_rhw(reasonable_orders=true), transform=adapt_costs(one), preferred_operators={pref})"
.format(**locals()), "--evaluator", "hff=ff(transform=adapt_costs(one))",
"--search", "lazy_greedy([hff,hlm],preferred=[hff],"
"cost_type=one,reopen_closed=false)"
]) for pref in ["simple"]]
CONFIGS = [
IssueConfig(config_nick,
config,
build_options=[build],
driver_options=["--build", build]) for rev in REVISIONS
for build in BUILDS for config_nick, config in CONFIG_NICKS
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue794-base", "issue794-v3"]
CONFIGS = [
IssueConfig('blind', ['--search', 'astar(blind())']),
]
SUITE = [
'assembly',
'miconic-fulladl',
'openstacks',
'openstacks-sat08-adl',
'optical-telegraphs',
'philosophers',
'psr-large',
'psr-middle',
'trucks',
]
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue860-base", "issue860-v2"]
BUILDS = ["release"]
CONFIG_NICKS = [
("astar-blind", ["--search", "astar(blind())"]),
("astar-lmcut", ["--search", "astar(lmcut())"
]), # inconsistent heuristic to test re-opening
("bjolp", [
"--evaluator",
"lmc=lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]),admissible=true)",
"--search", "astar(lmc,lazy_evaluator=lmc)"
]), # test lazy evaluator
]
CONFIGS = [
IssueConfig(config_nick,
config,
build_options=[build],
driver_options=["--build", build]) for build in BUILDS
for config_nick, config in CONFIG_NICKS
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
def main(revisions=None):
benchmarks_dir = os.path.expanduser('~/repos/downward/benchmarks')
suite = suites.suite_optimal_strips()
# dummy configs with correct names so that comparison report works
configs = {
IssueConfig('rl-b50k', []),
IssueConfig('cggl-b50k', []),
IssueConfig('dfp-b50k', []),
IssueConfig('rl-ginf', []),
IssueConfig('cggl-ginf', []),
IssueConfig('dfp-ginf', []),
IssueConfig('rl-f50k', []),
IssueConfig('cggl-f50k', []),
IssueConfig('dfp-f50k', []),
}
exp = IssueExperiment(
benchmarks_dir=benchmarks_dir,
suite=suite,
revisions=revisions,
configs=configs,
test_suite=['depot:p01.pddl'],
processes=4,
email='*****@*****.**',
)
exp.add_resource('ms_parser', 'ms-parser.py', dest='ms-parser.py')
exp.add_command('ms-parser', ['ms_parser'])
# planner outcome attributes
perfect_heuristic = Attribute('perfect_heuristic',
absolute=True,
min_wins=False)
proved_unsolvability = Attribute('proved_unsolvability',
absolute=True,
min_wins=False)
actual_search_time = Attribute('actual_search_time',
absolute=False,
min_wins=True,
functions=[gm])
# m&s attributes
ms_construction_time = Attribute('ms_construction_time',
absolute=False,
min_wins=True,
functions=[gm])
ms_abstraction_constructed = Attribute('ms_abstraction_constructed',
absolute=True,
min_wins=False)
ms_final_size = Attribute('ms_final_size', absolute=False, min_wins=True)
ms_out_of_memory = Attribute('ms_out_of_memory',
absolute=True,
min_wins=True)
ms_out_of_time = Attribute('ms_out_of_time', absolute=True, min_wins=True)
search_out_of_memory = Attribute('search_out_of_memory',
absolute=True,
min_wins=True)
search_out_of_time = Attribute('search_out_of_time',
absolute=True,
min_wins=True)
extra_attributes = [
perfect_heuristic,
proved_unsolvability,
actual_search_time,
ms_construction_time,
ms_abstraction_constructed,
ms_final_size,
ms_out_of_memory,
ms_out_of_time,
search_out_of_memory,
search_out_of_time,
]
attributes = exp.DEFAULT_TABLE_ATTRIBUTES
attributes.extend(extra_attributes)
exp.add_fetcher('data/issue655-base-eval')
exp.add_fetcher('data/issue655-v1-eval')
exp.add_comparison_table_step()
if matplotlib:
for attribute in ["memory", "total_time"]:
for config in configs:
exp.add_report(RelativeScatterPlotReport(
attributes=[attribute],
filter_config=[
"{}-{}".format(rev, config.nick) for rev in revisions
],
get_category=lambda run1, run2: run1.get("domain"),
),
outfile="{}-{}-{}.png".format(
exp.name, attribute, config.nick))
exp()
#! /usr/bin/env python2
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
EXPNAME = common_setup.get_experiment_name()
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue939-base"]
CONFIGS = [IssueConfig("translate-only", [], driver_options=["--translate"])]
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**")
# This was generated by running "./suites.py all" in the benchmarks
# repository.
SUITE = [
'agricola-opt18-strips',
'agricola-sat18-strips',
'airport',
'airport-adl',
'assembly',
'barman-mco14-strips',
'barman-opt11-strips',
'barman-opt14-strips',
'barman-sat11-strips',
'barman-sat14-strips',
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
BUILD = "release64"
REVISIONS = ["issue884-base", "issue884-v1", "issue884-v2", "issue884-v3"]
DRIVER_OPTIONS = ["--build", BUILD]
CONFIGS = [
IssueConfig(
nick + "-" + max_transitions_nick,
config,
build_options=[BUILD],
driver_options=DRIVER_OPTIONS)
for max_transitions_nick, max_transitions in [("2M", 2000000)]
for nick, config in [
("cegar-original", ["--search", "astar(cegar(subtasks=[original()], max_transitions={max_transitions}))".format(**locals())]),
("cegar-landmarks-goals", ["--search", "astar(cegar(max_transitions={max_transitions}))".format(**locals())]),
]
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
import os
from lab.environments import LocalEnvironment, MaiaEnvironment
from lab.reports import Attribute, arithmetic_mean, geometric_mean
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
from csv_report import CSVReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue705-base", "issue705-v5", "issue705-v6"]
CONFIGS = [
IssueConfig(
'bounded-blind',
['--search', 'astar(blind(), bound=0)'],
)
]
SUITE = list(
sorted(
set(common_setup.DEFAULT_OPTIMAL_SUITE)
| set(common_setup.DEFAULT_SATISFICING_SUITE)))
ENVIRONMENT = MaiaEnvironment(priority=0,
email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
from lab.reports import Attribute, geometric_mean
from downward.reports.compare import ComparativeReport
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
SCRIPT_NAME = os.path.splitext(os.path.basename(__file__))[0]
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue908-base", "issue908-v2"]
CONFIGS = [
IssueConfig("cpdbs-hc", ['--search', 'astar(cpdbs(hillclimbing))']),
IssueConfig("cpdbs-sys2", ['--search', 'astar(cpdbs(systematic(2)))']),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
configs = [
IssueConfig(
"cegar-10K-original",
["--search", "astar(cegar(subtasks=[original()],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-10K-landmarks-goals",
["--search", "astar(cegar(subtasks=[landmarks(), goals()],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-900s-landmarks-goals",
["--search", "astar(cegar(subtasks=[landmarks(), goals()],max_states=infinity,max_time=900))"]),
]
exp = IssueExperiment(
revisions=["issue633-v1", "issue633-v2"],
configs=configs,
suite=suites.suite_optimal_with_ipc11(),
test_suite=["depot:pfile1"],
email="*****@*****.**",
)
exp.add_comparison_table_step()
exp()
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue750-base", "issue750-v1"]
BUILD_OPTIONS = ["release32nolp"]
DRIVER_OPTIONS = ["--build", "release32nolp", "--search-time-limit", "1m"]
CONFIGS = [
IssueConfig("blind", ["--search", "astar(blind())"],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS),
IssueConfig("lama-first", [],
build_options=BUILD_OPTIONS,
driver_options=DRIVER_OPTIONS + ["--alias", "lama-first"]),
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = [
"issue781-v3", "issue781-v4-wss", "issue781-v4-varmark",
"issue781-v4-opportunistic"
]
CONFIGS = [
IssueConfig("{heuristic}-{pruning}".format(**locals()), [
"--search",
"astar({heuristic}(), pruning=stubborn_sets_{pruning}())".format(
**locals())
]) for heuristic in ["blind", "lmcut"] for pruning in ["queue"]
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, MaiaEnvironment
from downward import suites
from common_setup import IssueConfig, IssueExperiment, is_test_run
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue701-base", "issue701-v1"]
CONFIGS = [
IssueConfig(alias, [], driver_options=["--alias", alias]) for alias in [
"seq-sat-fd-autotune-1", "seq-sat-fd-autotune-2", "seq-sat-lama-2011",
"seq-sat-fdss-2014"
]
]
SUITE = suites.suite_all()
ENVIRONMENT = MaiaEnvironment(priority=-10, email="*****@*****.**")
if is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue814-v8", "issue814-v9"]
if common_setup.is_test_run():
BUILDS = ["release32"]
else:
BUILDS = ["debug64", "release32", "release64"]
CONFIGS = [
IssueConfig(
build + "-{heuristic}".format(**locals()),
["--evaluator", "h={heuristic}()".format(**locals()), "--search", "lazy_greedy([h],preferred=[h])"],
build_options=[build],
driver_options=["--build", build, "--overall-time-limit", "5m"])
for build in BUILDS
for heuristic in ["add", "ff"]
]
SUITE = common_setup.DEFAULT_SATISFICING_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue637-v1-base", "issue637-v6"]
DRIVER_OPTIONS = ["--overall-time-limit", "30m"]
CONFIGS = [
IssueConfig("cegar-landmarks-goals", ["--search", "astar(cegar())"],
driver_options=DRIVER_OPTIONS),
IssueConfig("cegar-original",
["--search", "astar(cegar(subtasks=[original()]))"],
driver_options=DRIVER_OPTIONS),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
from lab.environments import LocalEnvironment, MaiaEnvironment
from common_setup import IssueConfig, IssueExperiment, is_test_run
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue535-base", "issue535-v6"]
CONFIGS = [
IssueConfig("ehc-ff-pref-{heuristic}-{preferred_usage}".format(**locals(
)), [
"--heuristic", "hff=ff()", "--search",
"ehc(hff, preferred={heuristic}, preferred_usage={preferred_usage})".
format(**locals())
]) for preferred_usage in ["RANK_PREFERRED_FIRST", "PRUNE_BY_PREFERRED"]
for heuristic in ["add()", "hff"]
]
SUITE = [
'airport', 'assembly', 'barman-sat11-strips', 'barman-sat14-strips',
'blocks', 'cavediving-14-adl', 'childsnack-sat14-strips',
'citycar-sat14-adl', 'depot', 'driverlog', 'elevators-sat08-strips',
'elevators-sat11-strips', 'floortile-sat11-strips',
'floortile-sat14-strips', 'freecell', 'ged-sat14-strips', 'grid',
'gripper', 'hiking-sat14-strips', 'logistics00', 'logistics98',
'maintenance-sat14-adl', 'miconic', 'miconic-fulladl', 'miconic-simpleadl',
'movie', 'mprime', 'mystery', 'nomystery-sat11-strips', 'openstacks',
'openstacks-sat08-adl', 'openstacks-sat08-strips',
'openstacks-sat11-strips', 'openstacks-sat14-strips', 'openstacks-strips',
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from lab.reports import Attribute
from common_setup import IssueConfig, IssueExperiment
REVS = ["issue648-base", "issue648-v1"]
SUITE=suites.suite_optimal_strips()
SUITE.extend(suites.suite_ipc14_opt_strips())
CONFIGS = [
# Test label reduction, shrink_bucket_based (via shrink_fh and shrink_random)
IssueConfig('dfp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_bisimulation(max_states=50000,threshold=1,greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false)))']),
IssueConfig('dfp-f50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_fh(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
IssueConfig('dfp-r50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_dfp,shrink_strategy=shrink_random(max_states=50000),label_reduction=exact(before_shrinking=false,before_merging=true)))']),
# Test sampling
IssueConfig('ipdb', ['--search', 'astar(ipdb)']),
# Test genetic pattern generation
IssueConfig('genetic', ['--search', 'astar(zopdbs(patterns=genetic))']),
# Test cegar
IssueConfig(
"cegar-10K-goals-randomorder",
["--search", "astar(cegar(subtasks=[goals(order=random)],max_states=10000,max_time=infinity))"]),
IssueConfig(
"cegar-10K-original-randomorder",
["--search", "astar(cegar(subtasks=[original],max_states=10000,max_time=infinity,pick=random))"]),
]
exp = IssueExperiment(
import os.path
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
from lab.reports import Attribute, geometric_mean
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISION_CACHE = os.path.expanduser('~/lab/revision-cache')
REVISIONS = ["issue524-base-v2", "issue524-v2"]
CONFIGS = [
IssueConfig('lm_hm', [
'--landmarks', 'l=lm_hm()', '--heuristic', 'h=lmcount(l)', '--search',
'eager_greedy([h])'
]),
] + [
IssueConfig('lm_rhw', [
'--landmarks', 'l=lm_rhw()', '--heuristic', 'h=lmcount(l)', '--search',
'eager_greedy([h])'
]),
] + [
IssueConfig('lm_zg', [
'--landmarks', 'l=lm_zg()', '--heuristic', 'h=lmcount(l)', '--search',
'eager_greedy([h])'
]),
] + [
IssueConfig('lm_exhaust', [
'--landmarks', 'l=lm_exhaust()', '--heuristic', 'h=lmcount(l)',
'--search', 'eager_greedy([h])'
BUILDS = ["release32"]
SEARCHES = [
("blind", "astar(blind())"), ("divpot", "astar(diverse_potentials())"),
("lmcut", "astar(lmcut())"), ("cegar", "astar(cegar())"),
("systematic2", "astar(cpdbs(systematic(2)))"),
("mas", "astar(merge_and_shrink("
"shrink_strategy=shrink_bisimulation(greedy=false),"
"merge_strategy=merge_stateless("
"merge_selector=score_based_filtering("
"scoring_functions=[goal_relevance,dfp,total_order])),"
"label_reduction=exact(before_shrinking=true,before_merging=false),"
"max_states=50000,threshold_before_merge=1))")
]
CONFIGS = [
IssueConfig("{nick}-{build}".format(**locals()), ["--search", search],
build_options=[build],
driver_options=["--build", build, "--search-time-limit", "5m"])
for nick, search in SEARCHES for build in BUILDS
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = MaiaEnvironment(priority=0, email="*****@*****.**")
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
ENVIRONMENT = LocalEnvironment(processes=1)
exp = IssueExperiment(
revisions=REVISIONS,
configs=CONFIGS,
environment=ENVIRONMENT,
)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from downward import suites
from common_setup import IssueConfig, IssueExperiment
REVS = ["issue592-base", "issue592-v3"]
SUITE = suites.suite_optimal_strips()
CONFIGS = [
IssueConfig("lm_zg", [
"--landmarks", "lm=lm_zg()", "--heuristic", "hlm=lmcount(lm)",
"--search", "astar(hlm)"
]),
IssueConfig("lm_exhaust", [
"--landmarks", "lm=lm_exhaust()", "--heuristic", "hlm=lmcount(lm)",
"--search", "astar(hlm)"
]),
IssueConfig("lm_hm", [
"--landmarks", "lm=lm_hm(2)", "--heuristic", "hlm=lmcount(lm)",
"--search", "astar(hlm)"
]),
IssueConfig("lm_hm_max", [
"--landmarks", "lm=lm_hm(2)", "--heuristic",
"h1=lmcount(lm,admissible=true)", "--heuristic",
"h2=lmcount(lm,admissible=false)", "--search", "astar(max([h1,h2]))"
]),
]
exp = IssueExperiment(revisions=REVS,
from lab.environments import LocalEnvironment, BaselSlurmEnvironment
from downward.reports.compare import ComparativeReport
import common_setup
from common_setup import IssueConfig, IssueExperiment
from relativescatter import RelativeScatterPlotReport
DIR = os.path.dirname(os.path.abspath(__file__))
SCRIPT_NAME = os.path.splitext(os.path.basename(__file__))[0]
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
REVISIONS = ["issue903-base", "issue903-v1"]
DRIVER_OPTIONS = ["--overall-time-limit", "5m"]
CONFIGS = [
IssueConfig(
"cegar-original-1M",
["--search", "astar(cegar(subtasks=[original()], max_transitions=1M, max_time=infinity))"],
driver_options=DRIVER_OPTIONS),
IssueConfig(
"cegar-lm-goals-1M",
["--search", "astar(cegar(subtasks=[landmarks(), goals()], max_transitions=1M, max_time=infinity))"],
driver_options=DRIVER_OPTIONS),
]
SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
ENVIRONMENT = BaselSlurmEnvironment(
partition="infai_1",
email="*****@*****.**",
export=["PATH", "DOWNWARD_BENCHMARKS"])
if common_setup.is_test_run():
SUITE = IssueExperiment.DEFAULT_TEST_SUITE
