Example #1
0
def main():
    args = parse_custom_args()

    if not args.revision:
        rev = 'WORK'
        name = 'current'
    elif args.revision.lower() == 'baseline':
        rev = BASELINE
        name = 'baseline'
    else:
        rev = checkouts.get_global_rev(REPO, args.revision)
        name = rev

    combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]

    exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
    exp.add_suite(SUITES[args.test])
    for nick, config in CONFIGS[args.test]:
        exp.add_config(nick, config)
    exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')

    # Only compare results if we are not running the baseline experiment.
    if rev != BASELINE:
        exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True))
        exp.add_step(Step('fetch-baseline-results', Fetcher(),
                          get_exp_dir('baseline', args.test) + '-eval',
                          exp.eval_dir))
        exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
        exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
                       name='regression-check')
        exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
        exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
        exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))

    exp()
def main():
    args = parse_custom_args()

    if not args.revision:
        rev = 'WORK'
        name = 'current'
    elif args.revision.lower() == 'baseline':
        rev = BASELINE
        name = 'baseline'
    else:
        rev = checkouts.get_global_rev(REPO, args.revision)
        name = rev

    combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]

    exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
    exp.add_suite(SUITES[args.test])
    for nick, config in CONFIGS[args.test]:
        exp.add_config(nick, config)
    exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')

    # Only compare results if we are not running the baseline experiment.
    if rev != BASELINE:
        exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True))
        exp.add_step(Step('fetch-baseline-results', Fetcher(),
                          get_exp_dir('baseline', args.test) + '-eval',
                          exp.eval_dir))
        exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
        exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
        exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
        exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))
        exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
                       name='regression-check')

    exp()
Example #3
0
def main():
    args = parse_custom_args()

    if not args.revision:
        rev = 'WORK'
        name = 'current'
    elif args.revision.lower() == 'baseline':
        rev = BASELINE
        name = 'baseline'
    else:
        rev = checkouts.get_global_rev(REPO, args.revision)
        name = rev

    combo = [(Translator(REPO,
                         rev=rev), Preprocessor(REPO,
                                                rev=rev), Planner(REPO,
                                                                  rev=rev))]

    exp = DownwardExperiment(path=get_exp_dir(name, args.test),
                             repo=REPO,
                             combinations=combo)
    exp.add_suite(SUITES[args.test])
    for nick, config in CONFIGS[args.test]:
        exp.add_config(nick, config)
    exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
                   name='report')

    # Only compare results if we are not running the baseline experiment.
    if rev != BASELINE:
        dirty_paths = [
            path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir]
            if os.path.exists(path)
        ]
        if dirty_paths:
            logging.critical(
                'The last run found a regression. Please inspect what '
                'went wrong and then delete the following directories '
                'manually: %s' % dirty_paths)
        exp.add_step(
            Step('fetch-baseline-results', Fetcher(),
                 get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir))
        exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
                       name='comparison')
        exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
                       name='regression-check')
        # We abort if there is a regression and keep the directories.
        exp.add_step(
            Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
        exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
        exp.add_step(
            Step('rm-preprocessed-tasks', shutil.rmtree,
                 exp.preprocessed_tasks_dir))
        exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir))

    exp()
Example #4
0
def main():
    args = parse_custom_args()

    if not args.revision:
        rev = 'WORK'
        name = 'current'
    elif args.revision.lower() == 'baseline':
        rev = BASELINE
        name = 'baseline'
    else:
        rev = checkouts.get_global_rev(REPO, args.revision)
        name = rev

    combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]

    exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
    exp.add_suite(SUITES[args.test])
    for nick, config in CONFIGS[args.test]:
        exp.add_config(nick, config)
    exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')

    # Only compare results if we are not running the baseline experiment.
    if rev != BASELINE:
        dirty_paths = [path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir]
                       if os.path.exists(path)]
        if dirty_paths:
            logging.critical(
                'The last run found a regression. Please inspect what '
                'went wrong and then delete the following directories '
                'manually: %s' % dirty_paths)
        exp.add_step(Step('fetch-baseline-results', Fetcher(),
                          get_exp_dir('baseline', args.test) + '-eval',
                          exp.eval_dir))
        exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
        exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
                       name='regression-check')
        # We abort if there is a regression and keep the directories.
        exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
        exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
        exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))
        exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir))

    exp()
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
# TODO: Use add_revision() once it's available.
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from check import Check, RegressionCheckReport

DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, 'f5110717a963')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff',
     ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea',
     ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', [
        '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search',
        'lazy_greedy([hff, hcea], preferred=[hff, hcea])'
    ]),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
Example #6
0
from lab.fetcher import Fetcher
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from check import Check, RegressionCheckReport

DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, '5c061af448cd')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff',
     ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea',
     ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', [
        '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search',
        'lazy_greedy([hff, hcea], preferred=[hff, hcea])'
    ]),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
Example #7
0
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from regression_test import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, '1e20b69c5f43')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink('
             'merge_strategy=merge_dfp,'
             'shrink_strategy=shrink_bisimulation('
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from regression_test import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, 'eb9f8c86918f')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink('
             'merge_strategy=merge_dfp,'
             'shrink_strategy=shrink_bisimulation('
Example #9
0
from lab.fetcher import Fetcher
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from regression_test import Check, RegressionCheckReport

DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, 'b82ff7dc99e6')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff',
     ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea',
     ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', [
        '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search',
        'lazy_greedy([hff, hcea], preferred=[hff, hcea])'
    ]),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
Example #10
0
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from check import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, '5c061af448cd')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),'
            'shrink_strategy=shrink_bisimulation(max_states=50000,greedy=false,'
            'group_by_h=true)))']),
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from regression_test import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, 'eb9f8c86918f')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink('
             'merge_strategy=merge_dfp,'
             'shrink_strategy=shrink_bisimulation('
Example #12
0
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
# TODO: Use add_revision() once it's available.
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from check import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, 'f5110717a963')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),'
            'shrink_strategy=shrink_bisimulation(max_states=50000,greedy=false,'
            'group_by_h=true)))']),
Example #13
0
from lab.fetcher import Fetcher
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from regression_test import Check, RegressionCheckReport

DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, '1e20b69c5f43')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff',
     ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea',
     ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', [
        '--heuristic', 'hff=ff()', '--heuristic', 'hcea=cea()', '--search',
        'lazy_greedy([hff, hcea], preferred=[hff, hcea])'
    ]),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from regression_test import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, 'b82ff7dc99e6')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink('
             'merge_strategy=merge_dfp,'
             'shrink_strategy=shrink_bisimulation('
from lab.steps import Step
from lab.experiment import ARGPARSER

from downward.experiments import DownwardExperiment
from downward.checkouts import Translator, Preprocessor, Planner
from downward import checkouts
from downward.reports.absolute import AbsoluteReport

from check import Check, RegressionCheckReport


DIR = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(DIR, '../../'))
EXPERIMENTS_DIR = os.path.expanduser('~/experiments')

BASELINE = checkouts.get_global_rev(REPO, '8852bb518934')
if not BASELINE:
    logging.critical('Baseline not set or not found in repo.')
CONFIGS = {}
CONFIGS['nightly'] = [
    ('lmcut', ['--search', 'astar(lmcut())']),
    ('lazy-greedy-ff', ['--heuristic', 'h=ff()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-cea', ['--heuristic', 'h=cea()', '--search', 'lazy_greedy(h, preferred=h)']),
    ('lazy-greedy-ff-cea', ['--heuristic', 'hff=ff()', '--heuristic',  'hcea=cea()',
                            '--search', 'lazy_greedy([hff, hcea], preferred=[hff, hcea])']),
    ('lama-2011', ['ipc', 'seq-sat-lama-2011']),
    ('blind', ['--search', 'astar(blind())']),
    ('merge-and-shrink-bisim', ['--search',
        'astar(merge_and_shrink(merge_strategy=merge_linear(variable_order=reverse_level),'
            'shrink_strategy=shrink_bisimulation(max_states=50000,greedy=false,'
            'group_by_h=true)))']),