def main():
args = parse_custom_args()
if not args.revision:
rev = 'WORK'
name = 'current'
elif args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = checkouts.get_global_rev(REPO, args.revision)
name = rev
combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]
exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
exp.add_suite(SUITES[args.test])
for nick, config in CONFIGS[args.test]:
exp.add_config(nick, config)
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True))
exp.add_step(Step('fetch-baseline-results', Fetcher(),
get_exp_dir('baseline', args.test) + '-eval',
exp.eval_dir))
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))
exp()
def main():
args = parse_custom_args()
if not args.revision:
rev = 'WORK'
name = 'current'
elif args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = checkouts.get_global_rev(REPO, args.revision)
name = rev
combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]
exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
exp.add_suite(SUITES[args.test])
for nick, config in CONFIGS[args.test]:
exp.add_config(nick, config)
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True))
exp.add_step(Step('fetch-baseline-results', Fetcher(),
get_exp_dir('baseline', args.test) + '-eval',
exp.eval_dir))
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
exp()
def main():
args = parse_custom_args()
if not args.revision:
rev = 'WORK'
name = 'current'
elif args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = checkouts.get_global_rev(REPO, args.revision)
name = rev
combo = [(Translator(REPO,
rev=rev), Preprocessor(REPO,
rev=rev), Planner(REPO,
rev=rev))]
exp = DownwardExperiment(path=get_exp_dir(name, args.test),
repo=REPO,
combinations=combo)
exp.add_suite(SUITES[args.test])
for nick, config in CONFIGS[args.test]:
exp.add_config(nick, config)
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [
path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir]
if os.path.exists(path)
]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_step(
Step('fetch-baseline-results', Fetcher(),
get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir))
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step(
Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
exp.add_step(
Step('rm-preprocessed-tasks', shutil.rmtree,
exp.preprocessed_tasks_dir))
exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir))
exp()
def main():
args = parse_custom_args()
if not args.revision:
rev = 'WORK'
name = 'current'
elif args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = checkouts.get_global_rev(REPO, args.revision)
name = rev
combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]
exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
exp.add_suite(SUITES[args.test])
for nick, config in CONFIGS[args.test]:
exp.add_config(nick, config)
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir]
if os.path.exists(path)]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_step(Step('fetch-baseline-results', Fetcher(),
get_exp_dir('baseline', args.test) + '-eval',
exp.eval_dir))
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))
exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir))
exp()
def add_config(self, nick, config, timeout=None):
DownwardExperiment.add_config(self, nick, config, timeout=timeout)
self._config_nicks.append(nick)
def add_config(self, nick, config, timeout=None):
DownwardExperiment.add_config(self, nick, config, timeout=timeout)
self._config_nicks.append(nick)
