def fit_with_annealing(learn:Learner, num_epoch:int, lr:float=1e-3, annealing_start:float=0.7,
callbacks:list=None)->None:
n = len(learn.data.train_dl)
anneal_start = int(n*num_epoch*annealing_start)
phase0 = TrainingPhase(anneal_start).schedule_hp('lr', lr)
phase1 = TrainingPhase(n*num_epoch - anneal_start).schedule_hp('lr', lr, anneal=annealing_cos)
phases = [phase0, phase1]
sched = GeneralScheduler(learn, phases)
learn.callbacks.append(sched)
learn.fit(num_epoch, callbacks=callbacks)
def train_classifier(model, config, x_train, y_train, x_val, y_val, train_tfms=None):
loss_func = torch.nn.CrossEntropyLoss()
if train_tfms is None:
train_tfms = []
train_ds = ImageArrayDS(x_train, y_train, train_tfms)
val_ds = ImageArrayDS(x_val, y_val)
data = ImageDataBunch.create(train_ds, val_ds, bs=config['batch_size'])
callbacks = [partial(EarlyStoppingCallback, min_delta=1e-3, patience=config['patience'])]
learner = Learner(data, model, metrics=accuracy, loss_func=loss_func, callback_fns=callbacks)
learner.fit(config['epochs'], config['start_lr'], wd=config['weight_decay'])
return learner
def train(self, graph, max_epoch=100, min_delta=0, patience=0):
model_num = self._model_num
self._model_num = self._model_num + 1
learn = Learner(self.data, graph.generate_model(), loss_func=self.loss_func, metrics=self.metrics,
callback_fns=[partial(ValueTrackingCallback,
value_holder=self.accuracy,
monitor=self.monitor,
min_delta=min_delta,
patience=patience)])
progress_disabled_ctx(learn)
learn.fit(max_epoch)
print(f'Saving model {model_num}...', end='')
graph.save(os.path.join(self.path, str(model_num)))
print(' Done!')
print(f'Model number: {model_num}\nBest accuracy: {self.accuracy.value}')
return model_num, self.accuracy.value.item()
batch_size=batch_size,
num_workers=2,
pin_memory=True))
model = Unet(1, 3, n=4)
learner = Learner(databunch, model, loss_func=torch.nn.MSELoss())
test_data = list(Dataset(0, 16))
test_x = torch.stack([a[0] for a in test_data]).cuda()
test_y = [np.array(a[1]) for a in test_data]
epoch = -1
while not should_stop():
epoch += 1
print('Epoch:', epoch)
learner.fit(1)
prediction = model(test_x)
image_out = None
for i in range(len(prediction)):
image_x = np.concatenate(3 * [255 * np.swapaxes(test_x[i], 0, 2)],
axis=-1)
image_pred = 255 * np.swapaxes(np.asarray(prediction[i]), 0, 2)
image_y = 255 * np.swapaxes(test_y[i], 0, 2)
current_out = np.concatenate((image_x, image_pred, image_y), axis=0)
if image_out is None:
image_out = current_out
else:
image_out = np.concatenate((image_out, current_out), axis=1)
nn.Dropout(p=0.5),
nn.Linear(512, num_classes),
)
def forward(self, x):
x = self.l1(x)
x = self.l2(x)
x = self.l3(x)
x = self.l4(x)
x = x.view(x.size(0), -1)
x = self.classifier(x)
return F.log_softmax(x, dim=1)
if __name__ == "__main__":
datasetdir = os.path.join(os.path.dirname(__file__), './kuzu_mnist')
datasetdir = os.path.abspath(datasetdir)
# Load dataset
databunch = get_databunch(datasetdir)
print('Dataset loaded')
# Create VGG model
learn = Learner(databunch, VGG(), metrics=accuracy)
# Train
learn.fit(1)
# Save
learn.save('vgg_model_with_norm')
def main():
data, x_train, y_train, x_val, y_val = load_data()
loss_func = torch.nn.CrossEntropyLoss()
models = {
'cnn': AnotherConv(),
'resnet': resnet_masked(pretrained=True),
'resnet_multiple': resnet_linear(pretrained=True)
}
estimation_samples = 5_000
ndcgs, estimator_type, model_types = [], [], []
accuracies = []
for i in range(config['model_runs']):
print('==models run==', i + 1)
for name, model in models.items():
callbacks = [
partial(EarlyStoppingCallback,
patience=3,
min_delta=1e-2,
monitor='valid_loss')
]
learner = Learner(data,
model,
loss_func=loss_func,
metrics=[accuracy],
callback_fns=callbacks)
learner.fit(100, lr, wd=weight_decay)
inferencer = Inferencer(model)
masks = build_masks(DEFAULT_MASKS)
for j in range(config['repeat_runs']):
idxs = np.random.choice(len(x_val),
estimation_samples,
replace=False)
x_current = x_val[idxs]
y_current = y_val[idxs]
# masks
current_ll = ll(inferencer, x_current, y_current)
for mask_name, mask in masks.items():
print(mask_name)
estimator = build_estimator(
'bald_masked',
inferencer,
nn_runs=config['nn_runs'],
dropout_mask=mask,
dropout_rate=config['dropout_uq'],
num_classes=config['num_classes'])
uq = estimator.estimate(x_current)
estimator.reset()
ndcgs.append(uq_ndcg(-current_ll, uq))
estimator_type.append(mask_name)
estimator.reset()
model_types.append(name)
accuracies.append(learner.recorder.metrics[-1][0].item())
#
try:
plt.figure(figsize=(12, 8))
plt.title(f"NDCG on different train samples")
df = pd.DataFrame({
'ndcg': ndcgs,
'estimator_type': estimator_type,
'model': model_types
})
sns.boxplot(data=df, x='estimator_type', y='ndcg', hue='model')
plt.show()
plt.figure(figsize=(12, 8))
plt.title('Accuracies')
df = pd.DataFrame({'accuracy': accuracies, 'model': model_types})
sns.boxplot(data=df, y='accuracy', x='model')
plt.show()
except Exception as e:
print(e)
import ipdb
ipdb.set_trace()
def benchmark_uncertainty(config):
results = []
plt.figure(figsize=(10, 8))
for i in range(config['repeats']):
x_set, y_set, x_val, y_val, train_tfms = config['prepare_dataset'](
config)
if len(x_set) > config['train_size']:
_, x_train, _, y_train = train_test_split(
x_set, y_set, test_size=config['train_size'], stratify=y_set)
else:
x_train, y_train = x_set, y_set
train_ds = ImageArrayDS(x_train, y_train, train_tfms)
val_ds = ImageArrayDS(x_val, y_val)
data = ImageDataBunch.create(train_ds, val_ds, bs=config['batch_size'])
loss_func = torch.nn.CrossEntropyLoss()
np.set_printoptions(threshold=sys.maxsize, suppress=True)
model = build_model(config['model_type'])
callbacks = [
partial(EarlyStoppingCallback,
min_delta=1e-3,
patience=config['patience'])
]
learner = Learner(data,
model,
metrics=accuracy,
loss_func=loss_func,
callback_fns=callbacks)
learner.fit(config['epochs'],
config['start_lr'],
wd=config['weight_decay'])
images = torch.FloatTensor(x_val).cuda()
probabilities = F.softmax(model(images), dim=1).detach().cpu().numpy()
predictions = np.argmax(probabilities, axis=-1)
for name in config['estimators']:
ue = calc_ue(model, images, probabilities, name, config['nn_runs'])
mistake = 1 - (predictions == y_val).astype(np.int)
roc_auc = roc_auc_score(mistake, ue)
print(name, roc_auc)
results.append((name, roc_auc))
if i == config['repeats'] - 1:
fpr, tpr, thresholds = roc_curve(mistake, ue, pos_label=1)
plt.plot(fpr, tpr, label=name, alpha=0.8)
plt.xlabel('FPR')
plt.ylabel('TPR')
dir = Path(ROOT_DIR) / 'experiments' / 'data' / 'ood'
plt.title(f"{config['name']} uncertainty ROC")
plt.legend()
file = f"var_{label}_roc_{config['name']}_{config['train_size']}_{config['nn_runs']}"
plt.savefig(dir / file)
# plt.show()
df = pd.DataFrame(results, columns=['Estimator type', 'ROC-AUC score'])
df = df.replace('mc_dropout', 'MC dropout')
df = df.replace('decorrelating_sc', 'decorrelation')
df = df[df['Estimator type'] != 'k_dpp_noisereg']
print(df)
fig, ax = plt.subplots(figsize=(8, 6))
plt.subplots_adjust(left=0.2)
with sns.axes_style('whitegrid'):
sns.boxplot(data=df,
x='ROC-AUC score',
y='Estimator type',
orient='h',
ax=ax)
ax.yaxis.grid(True)
ax.xaxis.grid(True)
plt.title(f'{config["name"]} wrong prediction ROC-AUC')
file = f"var_{label}_boxplot_{config['name']}_{config['train_size']}_{config['nn_runs']}"
plt.savefig(dir / file)
df.to_csv(dir / file + '.csv')
