def testRegistryAdditions(self):
class MyRunner(Runner):
def run():
pass
def staging_required():
return False
class MyMetric(Metric):
pass
register_metric(MyMetric)
register_runner(MyRunner)
experiment = get_experiment_with_batch_and_single_trial()
experiment.runner = MyRunner()
experiment.add_tracking_metric(MyMetric(name="my_metric"))
with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json") as f:
save_experiment(
experiment,
f.name,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
loaded_experiment = load_experiment(
f.name,
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(loaded_experiment, experiment)
os.remove(f.name)
def testSaveAndLoad(self):
with tempfile.NamedTemporaryFile(mode="w+",
delete=False,
suffix=".json") as f:
save_experiment(self.experiment, f.name)
loaded_experiment = load_experiment(f.name)
self.assertEqual(loaded_experiment, self.experiment)
os.remove(f.name)
def testSaveValidation(self):
with self.assertRaises(ValueError):
save_experiment(
self.experiment.trials[0],
"test.json",
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
def testSaveAndLoad(self):
with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json") as f:
save_experiment(
self.experiment,
f.name,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
loaded_experiment = load_experiment(
f.name,
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(loaded_experiment, self.experiment)
os.remove(f.name)
def testSaveValidation(self):
with self.assertRaises(ValueError):
save_experiment(self.experiment.trials[0], "test.json")
def matbench_fold(fold):
t0 = time()
train_inputs, train_outputs = task.get_train_and_val_data(fold)
train_val_df = pd.DataFrame({
"formula": train_inputs.values,
"target": train_outputs.values
})
if dummy:
train_val_df = train_val_df[:25]
optimization_config = OptimizationConfig(objective=Objective(
metric=CrabNetMetric(name=metric,
train_val_df=train_val_df,
n_splits=n_splits),
minimize=True,
), )
# TODO: use status_quo (Arm) as default CrabNet parameters
exp = Experiment(
name="nested_crabnet_mae_saas",
search_space=search_space,
optimization_config=optimization_config,
runner=SyntheticRunner(),
)
sobol = Models.SOBOL(exp.search_space)
print("evaluating SOBOL points")
for _ in range(n_sobol):
print(_)
trial = exp.new_trial(generator_run=sobol.gen(1))
trial.run()
trial.mark_completed()
data = exp.fetch_data()
j = -1
new_value = np.nan
best_so_far = np.nan
for j in range(n_saas):
saas = Models.FULLYBAYESIAN(
experiment=exp,
data=exp.fetch_data(),
num_samples=
num_samples, # Increasing this may result in better model fits
warmup_steps=
warmup_steps, # Increasing this may result in better model fits
gp_kernel=
"rbf", # "rbf" is the default in the paper, but we also support "matern"
torch_device=tkwargs["device"],
torch_dtype=tkwargs["dtype"],
verbose=False, # Set to True to print stats from MCMC
disable_progbar=
True, # Set to False to print a progress bar from MCMC
)
generator_run = saas.gen(1)
best_arm, _ = generator_run.best_arm_predictions
trial = exp.new_trial(generator_run=generator_run)
trial.run()
trial.mark_completed()
data = Data.from_multiple_data([data, trial.fetch_data()])
new_value = trial.fetch_data().df["mean"].min()
best_so_far = data.df["mean"].min()
tf = time()
print(
f"iter{j}, BestInIter:{new_value:.3f}, BestSoFar:{best_so_far:.3f} elapsed time: {tf - t0}",
)
exp.fetch_data()
best_parameters = best_arm.parameters
experiment_fpath = join(experiment_dir, "experiment" + str(fold) + ".json")
save_experiment(exp, experiment_fpath)
test_pred, default_mae, test_mae, best_parameterization = get_test_results(
task, fold, best_parameters, train_val_df)
print(f"default_mae: {default_mae}")
print(f"test_mae: {test_mae}")
# maes.append(test_mae) # [0.32241879861870626, ...]
# task.record(fold, test_pred, params=best_parameterization)
return test_pred, best_parameterization
