def testRegistryAdditions(self): class MyRunner(Runner): def run(): pass def staging_required(): return False class MyMetric(Metric): pass register_metric(MyMetric) register_runner(MyRunner) experiment = get_experiment_with_batch_and_single_trial() experiment.runner = MyRunner() experiment.add_tracking_metric(MyMetric(name="my_metric")) with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json") as f: save_experiment( experiment, f.name, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ) loaded_experiment = load_experiment( f.name, decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ) self.assertEqual(loaded_experiment, experiment) os.remove(f.name)
def testSaveAndLoad(self): with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json") as f: save_experiment(self.experiment, f.name) loaded_experiment = load_experiment(f.name) self.assertEqual(loaded_experiment, self.experiment) os.remove(f.name)
def testSaveValidation(self): with self.assertRaises(ValueError): save_experiment( self.experiment.trials[0], "test.json", encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, )
def testSaveAndLoad(self): with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json") as f: save_experiment( self.experiment, f.name, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ) loaded_experiment = load_experiment( f.name, decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ) self.assertEqual(loaded_experiment, self.experiment) os.remove(f.name)
def testSaveValidation(self): with self.assertRaises(ValueError): save_experiment(self.experiment.trials[0], "test.json")
def matbench_fold(fold): t0 = time() train_inputs, train_outputs = task.get_train_and_val_data(fold) train_val_df = pd.DataFrame({ "formula": train_inputs.values, "target": train_outputs.values }) if dummy: train_val_df = train_val_df[:25] optimization_config = OptimizationConfig(objective=Objective( metric=CrabNetMetric(name=metric, train_val_df=train_val_df, n_splits=n_splits), minimize=True, ), ) # TODO: use status_quo (Arm) as default CrabNet parameters exp = Experiment( name="nested_crabnet_mae_saas", search_space=search_space, optimization_config=optimization_config, runner=SyntheticRunner(), ) sobol = Models.SOBOL(exp.search_space) print("evaluating SOBOL points") for _ in range(n_sobol): print(_) trial = exp.new_trial(generator_run=sobol.gen(1)) trial.run() trial.mark_completed() data = exp.fetch_data() j = -1 new_value = np.nan best_so_far = np.nan for j in range(n_saas): saas = Models.FULLYBAYESIAN( experiment=exp, data=exp.fetch_data(), num_samples= num_samples, # Increasing this may result in better model fits warmup_steps= warmup_steps, # Increasing this may result in better model fits gp_kernel= "rbf", # "rbf" is the default in the paper, but we also support "matern" torch_device=tkwargs["device"], torch_dtype=tkwargs["dtype"], verbose=False, # Set to True to print stats from MCMC disable_progbar= True, # Set to False to print a progress bar from MCMC ) generator_run = saas.gen(1) best_arm, _ = generator_run.best_arm_predictions trial = exp.new_trial(generator_run=generator_run) trial.run() trial.mark_completed() data = Data.from_multiple_data([data, trial.fetch_data()]) new_value = trial.fetch_data().df["mean"].min() best_so_far = data.df["mean"].min() tf = time() print( f"iter{j}, BestInIter:{new_value:.3f}, BestSoFar:{best_so_far:.3f} elapsed time: {tf - t0}", ) exp.fetch_data() best_parameters = best_arm.parameters experiment_fpath = join(experiment_dir, "experiment" + str(fold) + ".json") save_experiment(exp, experiment_fpath) test_pred, default_mae, test_mae, best_parameterization = get_test_results( task, fold, best_parameters, train_val_df) print(f"default_mae: {default_mae}") print(f"test_mae: {test_mae}") # maes.append(test_mae) # [0.32241879861870626, ...] # task.record(fold, test_pred, params=best_parameterization) return test_pred, best_parameterization