def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True)) exp.add_step(Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp()
def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True)) exp.add_step(Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') exp()
def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: dirty_paths = [ path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir] if os.path.exists(path) ] if dirty_paths: logging.critical( 'The last run found a regression. Please inspect what ' 'went wrong and then delete the following directories ' 'manually: %s' % dirty_paths) exp.add_step( Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') # We abort if there is a regression and keep the directories. exp.add_step( Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step( Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir)) exp()
def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: dirty_paths = [path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir] if os.path.exists(path)] if dirty_paths: logging.critical( 'The last run found a regression. Please inspect what ' 'went wrong and then delete the following directories ' 'manually: %s' % dirty_paths) exp.add_step(Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') # We abort if there is a regression and keep the directories. exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir)) exp()
from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment # TODO: Use add_revision() once it's available. from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from check import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, 'f5110717a963') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', [ '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])' ]), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']),
from lab.fetcher import Fetcher from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from check import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, '5c061af448cd') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', [ '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])' ]), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']),
from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from regression_test import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, '1e20b69c5f43') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']), ('merge-and-shrink-bisim', ['--search', 'astar(merge_and_shrink(' 'merge_strategy=merge_dfp,' 'shrink_strategy=shrink_bisimulation('
from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from regression_test import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, 'eb9f8c86918f') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']), ('merge-and-shrink-bisim', ['--search', 'astar(merge_and_shrink(' 'merge_strategy=merge_dfp,' 'shrink_strategy=shrink_bisimulation('
from lab.fetcher import Fetcher from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from regression_test import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, 'b82ff7dc99e6') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', [ '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])' ]), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']),
from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from check import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, '5c061af448cd') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']), ('merge-and-shrink-bisim', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),' 'shrink_strategy=shrink_bisimulation(max_states=50000,greedy=false,' 'group_by_h=true)))']),
from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment # TODO: Use add_revision() once it's available. from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from check import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, 'f5110717a963') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']), ('merge-and-shrink-bisim', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),' 'shrink_strategy=shrink_bisimulation(max_states=50000,greedy=false,' 'group_by_h=true)))']),
from lab.fetcher import Fetcher from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from regression_test import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, '1e20b69c5f43') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', [ '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])' ]), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']),
from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from regression_test import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, 'b82ff7dc99e6') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']), ('merge-and-shrink-bisim', ['--search', 'astar(merge_and_shrink(' 'merge_strategy=merge_dfp,' 'shrink_strategy=shrink_bisimulation('
from lab.steps import Step from lab.experiment import ARGPARSER from downward.experiments import DownwardExperiment from downward.checkouts import Translator, Preprocessor, Planner from downward import checkouts from downward.reports.absolute import AbsoluteReport from check import Check, RegressionCheckReport DIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(DIR, '../../')) EXPERIMENTS_DIR = os.path.expanduser('~/experiments') BASELINE = checkouts.get_global_rev(REPO, '8852bb518934') if not BASELINE: logging.critical('Baseline not set or not found in repo.') CONFIGS = {} CONFIGS['nightly'] = [ ('lmcut', ['--search', 'astar(lmcut())']), ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']), ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']), ('lama-2011', ['ipc', 'seq-sat-lama-2011']), ('blind', ['--search', 'astar(blind())']), ('merge-and-shrink-bisim', ['--search', 'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),' 'shrink_strategy=shrink_bisimulation(max_states=50000,greedy=false,' 'group_by_h=true)))']),