def fit_with_annealing(learn:Learner, num_epoch:int, lr:float=1e-3, annealing_start:float=0.7, callbacks:list=None)->None: n = len(learn.data.train_dl) anneal_start = int(n*num_epoch*annealing_start) phase0 = TrainingPhase(anneal_start).schedule_hp('lr', lr) phase1 = TrainingPhase(n*num_epoch - anneal_start).schedule_hp('lr', lr, anneal=annealing_cos) phases = [phase0, phase1] sched = GeneralScheduler(learn, phases) learn.callbacks.append(sched) learn.fit(num_epoch, callbacks=callbacks)
def train_classifier(model, config, x_train, y_train, x_val, y_val, train_tfms=None): loss_func = torch.nn.CrossEntropyLoss() if train_tfms is None: train_tfms = [] train_ds = ImageArrayDS(x_train, y_train, train_tfms) val_ds = ImageArrayDS(x_val, y_val) data = ImageDataBunch.create(train_ds, val_ds, bs=config['batch_size']) callbacks = [partial(EarlyStoppingCallback, min_delta=1e-3, patience=config['patience'])] learner = Learner(data, model, metrics=accuracy, loss_func=loss_func, callback_fns=callbacks) learner.fit(config['epochs'], config['start_lr'], wd=config['weight_decay']) return learner
def train(self, graph, max_epoch=100, min_delta=0, patience=0): model_num = self._model_num self._model_num = self._model_num + 1 learn = Learner(self.data, graph.generate_model(), loss_func=self.loss_func, metrics=self.metrics, callback_fns=[partial(ValueTrackingCallback, value_holder=self.accuracy, monitor=self.monitor, min_delta=min_delta, patience=patience)]) progress_disabled_ctx(learn) learn.fit(max_epoch) print(f'Saving model {model_num}...', end='') graph.save(os.path.join(self.path, str(model_num))) print(' Done!') print(f'Model number: {model_num}\nBest accuracy: {self.accuracy.value}') return model_num, self.accuracy.value.item()
batch_size=batch_size, num_workers=2, pin_memory=True)) model = Unet(1, 3, n=4) learner = Learner(databunch, model, loss_func=torch.nn.MSELoss()) test_data = list(Dataset(0, 16)) test_x = torch.stack([a[0] for a in test_data]).cuda() test_y = [np.array(a[1]) for a in test_data] epoch = -1 while not should_stop(): epoch += 1 print('Epoch:', epoch) learner.fit(1) prediction = model(test_x) image_out = None for i in range(len(prediction)): image_x = np.concatenate(3 * [255 * np.swapaxes(test_x[i], 0, 2)], axis=-1) image_pred = 255 * np.swapaxes(np.asarray(prediction[i]), 0, 2) image_y = 255 * np.swapaxes(test_y[i], 0, 2) current_out = np.concatenate((image_x, image_pred, image_y), axis=0) if image_out is None: image_out = current_out else: image_out = np.concatenate((image_out, current_out), axis=1)
nn.Dropout(p=0.5), nn.Linear(512, num_classes), ) def forward(self, x): x = self.l1(x) x = self.l2(x) x = self.l3(x) x = self.l4(x) x = x.view(x.size(0), -1) x = self.classifier(x) return F.log_softmax(x, dim=1) if __name__ == "__main__": datasetdir = os.path.join(os.path.dirname(__file__), './kuzu_mnist') datasetdir = os.path.abspath(datasetdir) # Load dataset databunch = get_databunch(datasetdir) print('Dataset loaded') # Create VGG model learn = Learner(databunch, VGG(), metrics=accuracy) # Train learn.fit(1) # Save learn.save('vgg_model_with_norm')
def main(): data, x_train, y_train, x_val, y_val = load_data() loss_func = torch.nn.CrossEntropyLoss() models = { 'cnn': AnotherConv(), 'resnet': resnet_masked(pretrained=True), 'resnet_multiple': resnet_linear(pretrained=True) } estimation_samples = 5_000 ndcgs, estimator_type, model_types = [], [], [] accuracies = [] for i in range(config['model_runs']): print('==models run==', i + 1) for name, model in models.items(): callbacks = [ partial(EarlyStoppingCallback, patience=3, min_delta=1e-2, monitor='valid_loss') ] learner = Learner(data, model, loss_func=loss_func, metrics=[accuracy], callback_fns=callbacks) learner.fit(100, lr, wd=weight_decay) inferencer = Inferencer(model) masks = build_masks(DEFAULT_MASKS) for j in range(config['repeat_runs']): idxs = np.random.choice(len(x_val), estimation_samples, replace=False) x_current = x_val[idxs] y_current = y_val[idxs] # masks current_ll = ll(inferencer, x_current, y_current) for mask_name, mask in masks.items(): print(mask_name) estimator = build_estimator( 'bald_masked', inferencer, nn_runs=config['nn_runs'], dropout_mask=mask, dropout_rate=config['dropout_uq'], num_classes=config['num_classes']) uq = estimator.estimate(x_current) estimator.reset() ndcgs.append(uq_ndcg(-current_ll, uq)) estimator_type.append(mask_name) estimator.reset() model_types.append(name) accuracies.append(learner.recorder.metrics[-1][0].item()) # try: plt.figure(figsize=(12, 8)) plt.title(f"NDCG on different train samples") df = pd.DataFrame({ 'ndcg': ndcgs, 'estimator_type': estimator_type, 'model': model_types }) sns.boxplot(data=df, x='estimator_type', y='ndcg', hue='model') plt.show() plt.figure(figsize=(12, 8)) plt.title('Accuracies') df = pd.DataFrame({'accuracy': accuracies, 'model': model_types}) sns.boxplot(data=df, y='accuracy', x='model') plt.show() except Exception as e: print(e) import ipdb ipdb.set_trace()
def benchmark_uncertainty(config): results = [] plt.figure(figsize=(10, 8)) for i in range(config['repeats']): x_set, y_set, x_val, y_val, train_tfms = config['prepare_dataset']( config) if len(x_set) > config['train_size']: _, x_train, _, y_train = train_test_split( x_set, y_set, test_size=config['train_size'], stratify=y_set) else: x_train, y_train = x_set, y_set train_ds = ImageArrayDS(x_train, y_train, train_tfms) val_ds = ImageArrayDS(x_val, y_val) data = ImageDataBunch.create(train_ds, val_ds, bs=config['batch_size']) loss_func = torch.nn.CrossEntropyLoss() np.set_printoptions(threshold=sys.maxsize, suppress=True) model = build_model(config['model_type']) callbacks = [ partial(EarlyStoppingCallback, min_delta=1e-3, patience=config['patience']) ] learner = Learner(data, model, metrics=accuracy, loss_func=loss_func, callback_fns=callbacks) learner.fit(config['epochs'], config['start_lr'], wd=config['weight_decay']) images = torch.FloatTensor(x_val).cuda() probabilities = F.softmax(model(images), dim=1).detach().cpu().numpy() predictions = np.argmax(probabilities, axis=-1) for name in config['estimators']: ue = calc_ue(model, images, probabilities, name, config['nn_runs']) mistake = 1 - (predictions == y_val).astype(np.int) roc_auc = roc_auc_score(mistake, ue) print(name, roc_auc) results.append((name, roc_auc)) if i == config['repeats'] - 1: fpr, tpr, thresholds = roc_curve(mistake, ue, pos_label=1) plt.plot(fpr, tpr, label=name, alpha=0.8) plt.xlabel('FPR') plt.ylabel('TPR') dir = Path(ROOT_DIR) / 'experiments' / 'data' / 'ood' plt.title(f"{config['name']} uncertainty ROC") plt.legend() file = f"var_{label}_roc_{config['name']}_{config['train_size']}_{config['nn_runs']}" plt.savefig(dir / file) # plt.show() df = pd.DataFrame(results, columns=['Estimator type', 'ROC-AUC score']) df = df.replace('mc_dropout', 'MC dropout') df = df.replace('decorrelating_sc', 'decorrelation') df = df[df['Estimator type'] != 'k_dpp_noisereg'] print(df) fig, ax = plt.subplots(figsize=(8, 6)) plt.subplots_adjust(left=0.2) with sns.axes_style('whitegrid'): sns.boxplot(data=df, x='ROC-AUC score', y='Estimator type', orient='h', ax=ax) ax.yaxis.grid(True) ax.xaxis.grid(True) plt.title(f'{config["name"]} wrong prediction ROC-AUC') file = f"var_{label}_boxplot_{config['name']}_{config['train_size']}_{config['nn_runs']}" plt.savefig(dir / file) df.to_csv(dir / file + '.csv')