def generate_experiments(configs): SUITE = [ "gripper:prob01.pddl", "blocks:probBLOCKS-5-0.pddl", "visitall-sat11-strips:problem12.pddl", "airport:p01-airport1-p1.pddl" ] ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**", export=EXPORTS) if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=2) exp = IssueExperiment( revisions=REVISIONS, configs=configs, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_absolute_report_step() exp.run_steps()
BUILDS = ["release32"] else: BUILDS = ["debug64", "release32", "release64"] CONFIGS = [ IssueConfig( build + "-{heuristic}".format(**locals()), [ "--evaluator", "h={heuristic}()".format(**locals()), "--search", "lazy_greedy([h],preferred=[h])" ], build_options=[build], driver_options=["--build", build, "--overall-time-limit", "30m"]) for build in BUILDS for heuristic in ["add", "ff"] ] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2", email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER)
from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue747-base", "issue747-v1"] CONFIGS = [ IssueConfig('lazy-greedy-blind', ['--search', 'lazy_greedy([blind()])']), IssueConfig('lama-first', [], driver_options=["--alias", "lama-first"]), IssueConfig('lwastar-ff', [ "--heuristic", "h=ff()", "--search", "lazy_wastar([h],preferred=[h],w=5)" ]) ] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**") if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_absolute_report_step() exp.add_comparison_table_step() for attribute in ["total_time"]:
from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue752-v2"] CONFIGS = [ IssueConfig("opcount-seq-lmcut-soplex", ["--search", "astar(operatorcounting([state_equation_constraints(), lmcut_constraints()], lpsolver=soplex))"]), IssueConfig("diverse-potentials-soplex", ["--search", "astar(diverse_potentials(lpsolver=soplex))"]), IssueConfig("optimal-lmcount-soplex", ["--search", "astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]), admissible=true, optimal=true, lpsolver=soplex))"]), IssueConfig("opcount-seq-lmcut-cplex", ["--search", "astar(operatorcounting([state_equation_constraints(), lmcut_constraints()], lpsolver=cplex))"]), IssueConfig("diverse-potentials-cplex", ["--search", "astar(diverse_potentials(lpsolver=cplex))"]), IssueConfig("optimal-lmcount-cplex", ["--search", "astar(lmcount(lm_merged([lm_rhw(),lm_hm(m=1)]), admissible=true, optimal=true, lpsolver=cplex))"]), ] SUITE = common_setup.DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2", email="*****@*****.**") if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=4) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER)
"--evaluator", "hlm=lmcount(lm_factory=lm_rhw(reasonable_orders=true), transform=adapt_costs(one), preferred_operators={pref})" .format(**locals()), "--search", "lazy_greedy([hlm],preferred=[hlm]," "cost_type=one,reopen_closed=false)" ]) for index, pref in enumerate(["none", "simple", "all"])] CONFIGS = [ IssueConfig(config_nick, config, build_options=[build], driver_options=["--build", build]) for rev in REVISIONS for build in BUILDS for config_nick, config in CONFIG_NICKS ] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2", email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.SINGLE_SEARCH_PARSER)
from lab.experiment import Experiment from lab.environments import BaselSlurmEnvironment import os ENV = BaselSlurmEnvironment(partition="infai_1", email="*****@*****.**", memory_per_cpu="6354M") DIR = os.path.dirname(os.path.abspath(__file__)) exp = Experiment(environment=ENV) exp.add_resource("solver", os.path.join(DIR, "hardInstanceSearch")) for i in [13, 16, 21, 48, 55, 59, 81, 98]: run = exp.add_run() run.add_resource("PDB_1", os.path.join(DIR, "STP(4,4)-0-0;11;12;13;14;15-8bpe-lex.pdb"), symlink=True) run.add_resource("PDB_2", os.path.join(DIR, "STP(4,4)-0-0;1;2;3;4;5;6;7-8bpe-lex.pdb"), symlink=True) run.add_resource("PDB_3", os.path.join(DIR, "STP(4,4)-0-0;8;9;12;13-8bpe-lex.pdb"), symlink=True) run.add_command("solve", ["{solver}", str(i)], 1800, 6354) run.set_property("id", [str(i)]) exp.add_step("build", exp.build) exp.add_step("start", exp.start_runs) exp.add_fetcher(name="fetch") exp.run_steps()
from lab.environments import LocalEnvironment, BaselSlurmEnvironment import common_setup from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue700-base", "issue700-v1"] CONFIGS = [ IssueConfig( "ehc_ff", ["--heuristic", "h=ff()", "--search", "ehc(h, preferred=[h])"]), ] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_absolute_report_step() exp.add_comparison_table_step()
import os from lab.environments import LocalEnvironment, BaselSlurmEnvironment from downward.reports.compare import ComparativeReport import common_setup from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue743-v2", "issue743-v3"] CONFIGS = [IssueConfig('ipdb-900s', ['--search', 'astar(ipdb(max_time=900))'])] SUITE = common_setup.DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**") if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_absolute_report_step() exp.add_comparison_table_step() exp.run_steps()
NODE = platform.node() RUNNING_ON_CLUSTER = NODE.endswith((".scicore.unibas.ch", ".cluster.bc2.ch")) DIR = os.path.abspath(os.path.dirname(__file__)) REPO = os.path.dirname(DIR) IMAGES_DIR = os.environ["SINGULARITY_IMAGES"] assert os.path.isdir(IMAGES_DIR), IMAGES_DIR BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] if RUNNING_ON_CLUSTER: SUITE = ["depot", "freecell", "gripper", "zenotravel"] ENVIRONMENT = BaselSlurmEnvironment( partition="infai_1", email="*****@*****.**", memory_per_cpu="3872M", export=["PATH"], setup=BaselSlurmEnvironment.DEFAULT_SETUP, # Until recently, we had to load the Singularity module here # by adding "module load Singularity/2.6.1 2> /dev/null". ) TIME_LIMIT = 1800 else: SUITE = ["depot:p01.pddl", "gripper:prob01.pddl", "mystery:prob07.pddl"] ENVIRONMENT = LocalEnvironment(processes=2) TIME_LIMIT = 5 ATTRIBUTES = [ "cost", "coverage", "error", "g_values_over_time",
from lab.environments import LocalEnvironment, BaselSlurmEnvironment import common_setup from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue724-base", "issue724-v2"] CONFIGS = [ IssueConfig('lama-first', [], driver_options=["--alias", "lama-first"]), IssueConfig("ehc-ff", ["--search", "ehc(ff())"]), ] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS", "PYTHONPATH"]) if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_absolute_report_step() exp.add_comparison_table_step() for attribute in ["total_time"]:
from common_setup import IssueConfig, IssueExperiment, DEFAULT_OPTIMAL_SUITE, is_test_run BENCHMARKS_DIR=os.path.expanduser('~/repos/downward/benchmarks') REVISIONS = ["issue668-v5-hack"] CONFIGS = [ IssueConfig('sbf-miasm-rl-nto-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), IssueConfig('sbf-miasm-l-nto-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), IssueConfig('sbf-miasm-rnd-nto-abp-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=true)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), IssueConfig('sbf-miasm-rl-nto-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), IssueConfig('sbf-miasm-l-nto-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=level,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), IssueConfig('sbf-miasm-rnd-nto-pba-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=random,product_ts_order=new_to_old,atomic_before_product=false)])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), IssueConfig('sbf-miasm-allrnd-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),single_random])),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))'], driver_options=['--search-memory-limit', '2048M']), ] SUITE = DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**", export=["PATH"], partition='infai_1') if is_test_run(): SUITE = ['depot:p01.pddl', 'parcprinter-opt11-strips:p01.pddl', 'mystery:prob07.pddl'] ENVIRONMENT = LocalEnvironment(processes=4) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_parser('lab_driver_parser', exp.LAB_DRIVER_PARSER) exp.add_parser('exitcode_parser', exp.EXITCODE_PARSER) exp.add_parser('translator_parser', exp.TRANSLATOR_PARSER) exp.add_parser('single_search_parser', exp.SINGLE_SEARCH_PARSER)
import os from lab.environments import LocalEnvironment, BaselSlurmEnvironment from lab.reports import Attribute, geometric_mean from downward.reports.absolute import AbsoluteReport from downward.reports.compare import ComparativeReport from common_setup import IssueConfig, IssueExperiment, DEFAULT_OPTIMAL_SUITE, is_test_run BENCHMARKS_DIR = os.path.expanduser('~/repos/downward/benchmarks') REVISIONS = [] CONFIGS = [] SUITE = DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**", export=["PATH"]) if is_test_run(): SUITE = [ 'depot:p01.pddl', 'parcprinter-opt11-strips:p01.pddl', 'mystery:prob07.pddl' ] ENVIRONMENT = LocalEnvironment(processes=4) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) # planner outcome attributes
] NODE = platform.node() RUNNING_ON_CLUSTER = NODE.endswith((".scicore.unibas.ch", ".cluster.bc2.ch")) DIR = os.path.abspath(os.path.dirname(__file__)) REPO = os.path.dirname(DIR) IMAGES_DIR = os.environ["SINGULARITY_IMAGES"] assert os.path.isdir(IMAGES_DIR), IMAGES_DIR BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] if RUNNING_ON_CLUSTER: SUITE = ["depot", "freecell", "gripper", "zenotravel"] ENVIRONMENT = BaselSlurmEnvironment( partition="infai_1", email="*****@*****.**", memory_per_cpu="3872M", export=["PATH"], setup=BaselSlurmEnvironment.DEFAULT_SETUP + "\nmodule load Singularity/2.6.1 2> /dev/null", ) TIME_LIMIT = 1800 else: SUITE = ["depot:p01.pddl", "gripper:prob01.pddl", "mystery:prob07.pddl"] ENVIRONMENT = LocalEnvironment(processes=2) TIME_LIMIT = 5 ATTRIBUTES = [ "cost", "coverage", "run_dir", "total_time", "singularity_runtime",
from lab.environments import LocalEnvironment, BaselSlurmEnvironment from lab import tools from downward.reports.compare import ComparativeReport from downward.reports import PlanningReport from downward.experiment import FastDownwardExperiment import common_setup from common_setup import IssueConfig, IssueExperiment DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue874-base", "issue874-v1"] CONFIGS = [IssueConfig("translate-only", [], driver_options=["--translate"])] ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2", email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) # This was generated by running "./suites.py all" in the benchmarks # repository. I don't know if this is there is a better way of doing # this. SUITE = [ 'agricola-opt18-strips', 'agricola-sat18-strips', 'airport', 'airport-adl', 'assembly', 'barman-mco14-strips', 'barman-opt11-strips', 'barman-opt14-strips', 'barman-sat11-strips',
REMOTE = NODE.endswith(".scicore.unibas.ch") or NODE.endswith( ".cluster.bc2.ch") BENCHMARKS_DIR = os.environ["HTG_BENCHMARKS"] RUN_SCRIPT_DIR = str(Path(os.getcwd()).parent.parent) if REMOTE: SUITE = [ 'genome-edit-distance', 'genome-edit-distance-split', 'genome-edit-distance-positional', 'pipesworld-tankage-nosplit', 'organic-synthesis-MIT', 'organic-synthesis-original', 'organic-synthesis-alkene' ] ENV = BaselSlurmEnvironment( partition='infai_2', memory_per_cpu="6G", extra_options='#SBATCH --cpus-per-task=3', setup="%s\n%s" % (BaselSlurmEnvironment.DEFAULT_SETUP, "source /infai/blaas/virtualenvs/grounder/bin/activate\n"), export=["PATH", "DOWNWARD_BENCHMARKS", "POWER_LIFTED_DIR"]) else: SUITE = ['organic-synthesis-alkene:p2.pddl'] ENV = LocalEnvironment(processes=4) TIME_LIMIT = 1800 MEMORY_LIMIT = 16384 ATTRIBUTES = ['atoms', 'grounding_time', 'parsing_time', 'total_time'] # Create a new experiment. exp = Experiment(environment=ENV)
#! /usr/bin/env python2 # -*- coding: utf-8 -*- import os from lab.environments import LocalEnvironment, BaselSlurmEnvironment import common_setup from common_setup import IssueConfig, IssueExperiment EXPNAME = common_setup.get_experiment_name() DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue939-base"] CONFIGS = [IssueConfig("translate-only", [], driver_options=["--translate"])] ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2", email="*****@*****.**") # This was generated by running "./suites.py all" in the benchmarks # repository. SUITE = [ 'agricola-opt18-strips', 'agricola-sat18-strips', 'airport', 'airport-adl', 'assembly', 'barman-mco14-strips', 'barman-opt11-strips', 'barman-opt14-strips', 'barman-sat11-strips', 'barman-sat14-strips', 'blocks',
"--search", "lazy_greedy([heur], preferred=[heur])"])) CONFIGS.append(IssueConfig(h+"-overapprox-axiom-rules", ["--evaluator", "heur=%s" % h, "--search", "lazy_greedy([heur], preferred=[heur])", "--translate-options", "--overapproximate-axioms"]),) # Add lama-first CONFIGS.append(IssueConfig("lama-normal-axiom-rules", [], driver_options=["--alias", "lama-first"])) CONFIGS.append(IssueConfig("lama-overapprox-axiom-rules", ["--translate-options", "--overapproximate-axioms"], driver_options=["--alias", "lama-first"]),) # Add A* with blind CONFIGS.append(IssueConfig("blind-normal-axiom-rules", ["--search", "astar(blind)"])) CONFIGS.append(IssueConfig("blind-overapprox-axiom-rules", ["--search", "astar(blind)", "--translate-options", "--overapproximate-axioms"]),) SUITE = ["airport-adl"] ENVIRONMENT = BaselSlurmEnvironment( partition="infai_1", export=["PATH", "DOWNWARD_BENCHMARKS"]) if common_setup.is_test_run(): SUITE = ["depot:p01.pddl", "gripper:prob01.pddl", "psr-middle:p01-s17-n2-l2-f30.pddl"] ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER)
'eager_greedy([h])' ]), ] + [ IssueConfig('lm_exhaust', [ '--landmarks', 'l=lm_exhaust()', '--heuristic', 'h=lmcount(l)', '--search', 'eager_greedy([h])' ]), ] + [ IssueConfig('lm_merged', [ '--landmarks', 'l1=lm_exhaust()', '--landmarks', 'l2=lm_rhw()', '--landmarks', 'l=lm_merged([l1, l2])', '--heuristic', 'h=lmcount(l)', '--search', 'eager_greedy([h])' ]), ] + [IssueConfig("lama-first", [], driver_options=["--alias", "lama-first"])] SUITE = common_setup.DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**") SUITE = common_setup.DEFAULT_OPTIMAL_SUITE if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=4) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, revision_cache=REVISION_CACHE, ) exp.add_suite(BENCHMARKS_DIR, SUITE)
]), # ('rl-b50k', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_precomputed(merge_tree=linear(variable_order=reverse_level)),shrink_strategy=shrink_bisimulation(greedy=false),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']), # ('sbmiasm-b50k', ['--search', 'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_stateless(merge_selector=score_based_filtering(scoring_functions=[sf_miasm(shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']), # ('sccs-dfp-b50k', ['--search', 'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_sccs(order_of_sccs=topological,merge_selector=score_based_filtering(scoring_functions=[goal_relevance,dfp,total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']), ] CONFIGS = [ IssueConfig(config_nick, config, build_options=[build], driver_options=["--build", build]) for build in BUILDS for config_nick, config in CONFIG_NICKS ] SUITE = common_setup.DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(partition="infai_2", email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=4) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_parser(exp.EXITCODE_PARSER) exp.add_parser(exp.TRANSLATOR_PARSER)
'openstacks-strips', 'organic-synthesis-opt18-strips', 'organic-synthesis-split-opt18-strips', 'parcprinter-08-strips', 'parcprinter-opt11-strips', 'parking-opt11-strips', 'parking-opt14-strips', 'pathways-noneg', 'pegsol-08-strips', 'pegsol-opt11-strips', 'petri-net-alignment-opt18-strips', 'pipesworld-notankage', 'pipesworld-tankage', 'psr-small', 'rovers', 'satellite', 'scanalyzer-08-strips', 'scanalyzer-opt11-strips', 'snake-opt18-strips', 'sokoban-opt08-strips', 'sokoban-opt11-strips', 'spider-opt18-strips', 'storage', 'termes-opt18-strips', 'tetris-opt14-strips', 'tidybot-opt11-strips', 'tidybot-opt14-strips', 'tpp', 'transport-opt08-strips', 'transport-opt11-strips', 'transport-opt14-strips', 'trucks-strips', 'visitall-opt11-strips', 'visitall-opt14-strips', 'woodworking-opt08-strips', 'woodworking-opt11-strips', 'zenotravel' ] ENV = BaselSlurmEnvironment(email="*****@*****.**") else: SUITE = [ 'depot:p01.pddl', 'depot:p02.pddl', 'gripper:prob01.pddl', 'gripper:prob02.pddl', 'gripper:prob03.pddl', 'mystery:prob01.pddl', 'mystery:prob03.pddl' ] ENV = LocalEnvironment(processes=6) # Use path to your Fast Downward repository. REPO = os.environ["DOWNWARD_REPO"] BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISION_CACHE = os.path.expanduser('~/lab/revision-cache') exp = CEGARExperiment(soft_limit=20 * 1024, hard_limit=50 * 1024, environment=ENV,
# -*- coding: utf-8 -*- import os from lab.environments import LocalEnvironment, BaselSlurmEnvironment import common_setup from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue695-v3-base", "issue695-v3"] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1", email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) CONFIGS = [ IssueConfig("lama-first", [], driver_options=[ "--alias", "lama-first", "--overall-time-limit", "1m", ]), ] if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1)
from downward.reports.compare import ComparativeReport import common_setup from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue776-base", "issue776-v1"] CONFIGS = [ IssueConfig('lama', [], driver_options=['--alias', 'seq-sat-lama-2011']), IssueConfig('lama-first', [], driver_options=['--alias', 'lama-first']), IssueConfig('bjolp', [], driver_options=['--alias', 'seq-opt-bjolp']), ] SUITE = common_setup.DEFAULT_OPTIMAL_SUITE ENVIRONMENT = BaselSlurmEnvironment(email="*****@*****.**") if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=4) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_comparison_table_step() exp.run_steps()
def is_running_on_cluster(): node = platform.node() return node.endswith(".scicore.unibas.ch") or node.endswith( ".cluster.bc2.ch") PARTITION = "infai_2" EMAIL = "*****@*****.**" REPO = "/infai/kellert/prost/prost-dev/" REV = "HEAD" TIME_PER_STEP = 1.0 if is_running_on_cluster(): ENV = BaselSlurmEnvironment(partition=PARTITION, email=EMAIL) SUITES = IPC_ALL NUM_RUNS = 30 else: ENV = LocalEnvironment(processes=4) SUITES = [ELEVATORS_2011[0], NAVIGATION_2011[0]] NUM_RUNS = 2 # Generate the experiment exp = ProstExperiment(suites=SUITES, environment=ENV, num_runs=NUM_RUNS, time_per_step=TIME_PER_STEP) # Add algorithm configurations to the experiment
for rev in ["base", "v1", "v2", "v3", "v4", "v5", "v6"] ] BUILDS = ["release32"] SEARCHES = [ ("astar-inf", ["--search", "astar(const(infinity))"]), ] CONFIGS = [ IssueConfig("{nick}-{build}".format(**locals()), config, build_options=[build], driver_options=["--build", build]) for nick, config in SEARCHES for build in BUILDS ] SUITE = set(common_setup.DEFAULT_OPTIMAL_SUITE + common_setup.DEFAULT_SATISFICING_SUITE) ENVIRONMENT = BaselSlurmEnvironment(priority=0, email="*****@*****.**") if common_setup.is_test_run(): SUITE = IssueExperiment.DEFAULT_TEST_SUITE ENVIRONMENT = LocalEnvironment(processes=1) exp = IssueExperiment( revisions=REVISIONS, configs=CONFIGS, environment=ENVIRONMENT, ) exp.add_suite(BENCHMARKS_DIR, SUITE) exp.add_resource('sg_parser', 'sg-parser.py', dest='sg-parser.py') exp.add_command('sg-parser', ['{sg_parser}'])
from downward.experiment import FastDownwardExperiment from downward.reports.absolute import AbsoluteReport from downward.reports.compare import ComparativeReport from downward.reports.scatter import ScatterPlotReport from downward.reports.taskwise import TaskwiseReport from lab import cached_revision, reports from lab.environments import BaselSlurmEnvironment, LocalEnvironment from lab.reports import Attribute from lab.reports.filter import FilterReport DIR = os.path.dirname(os.path.abspath(__file__)) NODE = platform.node() REMOTE = NODE.endswith(".scicore.unibas.ch") or NODE.endswith( ".cluster.bc2.ch") if REMOTE: ENV = BaselSlurmEnvironment("*****@*****.**") else: ENV = LocalEnvironment(processes=2) REPO = os.environ["DOWNWARD_REPO"] BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REV_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") VCS = cached_revision.get_version_control_system(REPO) REV = "default" if VCS == cached_revision.MERCURIAL else "master" class QualityFilters(object): """Compute the IPC quality score. The IPC score is computed over the list of runs for each task. Since filters only work on individual runs, we can't compute the score with a single filter, but it is possible by using two filters:
# -*- coding: utf-8 -*- import os from lab.environments import LocalEnvironment, BaselSlurmEnvironment import common_setup from common_setup import IssueConfig, IssueExperiment from relativescatter import RelativeScatterPlotReport DIR = os.path.dirname(os.path.abspath(__file__)) BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] REVISIONS = ["issue526-base", "issue526-v1"] SUITE = common_setup.DEFAULT_SATISFICING_SUITE ENVIRONMENT = BaselSlurmEnvironment(partition="infai_1", email="*****@*****.**", export=["PATH", "DOWNWARD_BENCHMARKS"]) CONFIGS = [ IssueConfig("ehc_ff", ["--heuristic", "h=ff()", "--search", "ehc(h, preferred=[h])"], driver_options=["--overall-time-limit", "5m"]), IssueConfig( "lama-first-lazy", [], driver_options=["--alias", "lama-first", "--overall-time-limit", "5m"]), IssueConfig("lama-first-eager", [ "--evaluator", """hlm=lama_synergy(lm_rhw(reasonable_orders=true,lm_cost_type=one), transform=adapt_costs(one))""", "--evaluator", "hff=ff_synergy(hlm)", "--search",