def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True)) exp.add_step(Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp()
def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: exp.steps.insert(0, Step('rm-eval-dir', shutil.rmtree, exp.eval_dir, ignore_errors=True)) exp.add_step(Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') exp()
def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: dirty_paths = [ path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir] if os.path.exists(path) ] if dirty_paths: logging.critical( 'The last run found a regression. Please inspect what ' 'went wrong and then delete the following directories ' 'manually: %s' % dirty_paths) exp.add_step( Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') # We abort if there is a regression and keep the directories. exp.add_step( Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step( Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir)) exp()
def main(): args = parse_custom_args() if not args.revision: rev = 'WORK' name = 'current' elif args.revision.lower() == 'baseline': rev = BASELINE name = 'baseline' else: rev = checkouts.get_global_rev(REPO, args.revision) name = rev combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))] exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo) exp.add_suite(SUITES[args.test]) for nick, config in CONFIGS[args.test]: exp.add_config(nick, config) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report') # Only compare results if we are not running the baseline experiment. if rev != BASELINE: dirty_paths = [path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir] if os.path.exists(path)] if dirty_paths: logging.critical( 'The last run found a regression. Please inspect what ' 'went wrong and then delete the following directories ' 'manually: %s' % dirty_paths) exp.add_step(Step('fetch-baseline-results', Fetcher(), get_exp_dir('baseline', args.test) + '-eval', exp.eval_dir)) exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison') exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS), name='regression-check') # We abort if there is a regression and keep the directories. exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path)) exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path)) exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir)) exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir)) exp()