def search_Mahalanobis_hyperparams(model,
sample_mean,
precision,
layer_indexs,
num_classes,
ind_dataloader_val_for_train,
ood_dataloader_val_for_train,
ind_dataloader_val_for_test,
ood_dataloader_val_for_test,
std):
# magnitude_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
magnitude_list = [0.02, 0.015, 0.01, 0.009, 0.008, 0.007, 0.006, 0.005, 0.004, 0.003, 0.002, 0.0014, 0.001, 0.0005]
best_magnitude = None
best_tnr = 0
for m in tqdm(magnitude_list, desc="magnitude"):
ind_features_val_for_train = get_Mahalanobis_score_ensemble(model, ind_dataloader_val_for_train, layer_indexs, num_classes, sample_mean, precision, m, std)
ood_features_val_for_train = get_Mahalanobis_score_ensemble(model, ood_dataloader_val_for_train, layer_indexs, num_classes, sample_mean, precision, m, std)
ind_features_val_for_test = get_Mahalanobis_score_ensemble(model, ind_dataloader_val_for_test, layer_indexs, num_classes, sample_mean, precision, m, std)
ood_features_val_for_test = get_Mahalanobis_score_ensemble(model, ood_dataloader_val_for_test, layer_indexs, num_classes, sample_mean, precision, m, std)
lr = train_lr(ind_features_val_for_train, ood_features_val_for_train)
metrics = get_metrics(lr, ind_features_val_for_test, ood_features_val_for_test, acc_type="best")
print("m:{}, metrics:{}".format(m, metrics))
if metrics['TNR@tpr=0.95'] > best_tnr:
best_tnr = metrics['TNR@tpr=0.95']
best_magnitude = m
return best_magnitude
def search_FSS_hyperparams(model,
fss,
layer_indexs,
ind_dataloader_val_for_train,
ood_dataloader_val_for_train,
ind_dataloader_val_for_test,
ood_dataloader_val_for_test,
std):
# magnitude_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
magnitude_list = [0.2, 0.18, 0.16, 0.14, 0.12, 0.1, 0.08, 0.06, 0.04, 0.02, 0.01, 0.008, 0.006, 0.004, 0.002, 0.001, 0.0008, 0.0006, 0.0004, 0.0002, 0.0001, 0.0]
best_magnitude = None
best_tnr = 0
for m in tqdm(magnitude_list, desc="magnitude"):
ind_features_val_for_train = get_FSS_score_ensem_process(model, ind_dataloader_val_for_train, fss, layer_indexs, m, std)
ood_features_val_for_train = get_FSS_score_ensem_process(model, ood_dataloader_val_for_train, fss, layer_indexs, m, std)
ind_features_val_for_test = get_FSS_score_ensem_process(model, ind_dataloader_val_for_test, fss, layer_indexs, m, std)
ood_features_val_for_test = get_FSS_score_ensem_process(model, ood_dataloader_val_for_test, fss, layer_indexs, m, std)
lr = train_lr(ind_features_val_for_train, ood_features_val_for_train)
metrics = get_metrics(lr, ind_features_val_for_test, ood_features_val_for_test, acc_type="best")
print("m:{}, metrics:{}".format(m, metrics))
if metrics['TNR@tpr=0.95'] > best_tnr:
best_tnr = metrics['TNR@tpr=0.95']
best_magnitude = m
return best_magnitude
def search_ODIN_hyperparams(model, ind_dataloader_val_for_train,
ood_dataloader_val_for_train,
ind_dataloader_val_for_test,
ood_dataloader_val_for_test, std):
# magnitude_list = [0, 0.0005, 0.001, 0.0012, 0.0014, 0.0018, 0.002, 0.0024, 0.005, 0.008, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0]
magnitude_list = [
0.2, 0.18, 0.16, 0.14, 0.12, 0.1, 0.08, 0.06, 0.04, 0.02, 0.01, 0.008,
0.006, 0.004, 0.002, 0.001, 0.0008, 0.0006, 0.0004, 0.0002, 0.0001, 0.0
]
# magnitude_list = [0.009, 0.01, 0.011, 0.012, 0.013, 0.014, 0.015]
# magnitude_list = [0,]
# magnitude_list = [0, 0.01,]
temperature_list = [1, 10, 100, 1000]
# temperature_list = [1000,]
# temperature_list = [1,]
best_magnitude = None
best_temperature = None
best_tnr = 0
for m in tqdm(magnitude_list, desc="magnitude"):
for t in tqdm(temperature_list, desc="temperature"):
print("get_ODIN_score for ind_scores_train")
ind_scores_train = get_ODIN_score(model,
ind_dataloader_val_for_train, m,
t, std)
print("get_ODIN_score for ood_scores_train")
ood_scores_train = get_ODIN_score(model,
ood_dataloader_val_for_train, m,
t, std)
ind_features_val_for_train = ind_scores_train.reshape(-1, 1)
ood_features_val_for_train = ood_scores_train.reshape(-1, 1)
print("train lr")
lr = train_lr(ind_features_val_for_train,
ood_features_val_for_train)
print("get_ODIN_score for ind_scores_test")
ind_scores_test = get_ODIN_score(model,
ind_dataloader_val_for_test, m, t,
std)
print("get_ODIN_score for ood_scores_test")
ood_scores_test = get_ODIN_score(model,
ood_dataloader_val_for_test, m, t,
std)
ind_features_val_for_test = ind_scores_test.reshape(-1, 1)
ood_features_val_for_test = ood_scores_test.reshape(-1, 1)
metrics = get_metrics(lr, ind_features_val_for_test,
ood_features_val_for_test)
print("t:{}, m:{}, metrics:{}".format(t, m, metrics))
if metrics['TNR@tpr=0.95'] > best_tnr:
best_tnr = metrics['TNR@tpr=0.95']
best_magnitude = m
best_temperature = t
return best_temperature, best_magnitude
best_temperature,
std=std)
ind_ensemble_val.append(ind_scores_val_for_train)
ood_ensemble_val.append(ood_scores_val_for_train)
ind_scores_test = get_ODIN_score(model,
ind_dataloader_test,
best_magnitude,
best_temperature,
std=std)
ood_scores_test = get_ODIN_score(model,
ood_dataloader_test,
best_magnitude,
best_temperature,
std=std)
ind_ensemble_test.append(ind_scores_test)
ood_ensemble_test.append(ood_scores_test)
take_mean_and_reshape = lambda x: np.array(x).mean(axis=0).reshape(-1, 1)
ind_val, ood_val, ind_test, ood_test = map(
take_mean_and_reshape,
[ind_ensemble_val, ood_ensemble_val, ind_ensemble_test, ood_ensemble_test])
# ----- Train OoD detector using validation data -----
from lib.metric import get_metrics, train_lr
lr = train_lr(ind_val, ood_val)
# ----- Calculating metrics using test data -----
metrics = get_metrics(lr, ind_test, ood_test, acc_type="best")
print("metrics: ", metrics)
std=std)
ind_features_test = get_Mahalanobis_score_ensemble(model,
ind_dataloader_test,
layer_indexs,
num_classes,
sample_mean,
precision,
best_magnitude,
std=std)
ood_features_test = get_Mahalanobis_score_ensemble(
model,
ood_dataloader_test,
layer_indexs,
num_classes,
sample_mean,
precision,
best_magnitude,
std=std)[:len(ind_features_test)]
# ----- Training OoD detector using validation data -----
lr = train_lr(ind_features_val_for_train, ood_features_val_for_train)
# ----- Calculating metrics using test data -----
metrics = get_metrics(lr,
ind_features_test,
ood_features_test,
acc_type="best")
print("best params: ", best_magnitude)
print("metrics: ", metrics)