def predict_dataset( ds_type: str = 'test', cpu=True, sample_size=config.BEST_MODEL_PARAMS['sample_size'], with_oversampling=config.BEST_MODEL_PARAMS['with_oversampling'], with_focal_loss=config.BEST_MODEL_PARAMS['with_focal_loss'], with_weighted_loss=config.BEST_MODEL_PARAMS['with_weighted_loss'], confusion_matrix_filename='test_confusion_matrix'): learn = load_saved_learner(sample_size=sample_size, with_oversampling=with_oversampling, with_focal_loss=with_focal_loss, with_weighted_loss=with_weighted_loss, cpu=cpu) classes = {c: i for i, c in enumerate(learn.data.classes)} # create a DF with filepath and class label (int) data = pd.read_csv(config.PROCESSED_DATA_DIR / 'labels_full.csv') data = data[data['ds_type'] == ds_type][['name', 'label']] data['label_int'] = data.label.apply(lambda x: classes[x]) print(f'Running predictions on {data.shape[0]} data samples') data['pred_probability'] = pd.Series(dtype=object) for k, i in enumerate(data.index): pred = learn.predict( open_image(config.PROCESSED_DATA_DIR / data.loc[i, 'name'], convert_mode='L')) data.loc[i, 'y_pred'] = pred[0].data.item() data.at[i, 'pred_probability'] = pred[2].numpy() if k % 200 == 0 and k > 0: print(f'{k} images done..') data.to_csv(config.DATA_DIR / 'predictions.csv', index=False) print(f'Building classification interpretation..') interp = ClassificationInterpretation( learn=learn, losses=np.zeros(data.shape[0]), preds=tensor(data['pred_probability'].to_list()), y_true=tensor(data.label_int.to_list())) mat = interp.confusion_matrix() # sum diagonal / all data_size *100 accuracy = np.trace(mat) / mat.sum() * 100 print(mat) print(f'Accuracy: {accuracy}') interp.plot_confusion_matrix(return_fig=True).savefig( f'test_{confusion_matrix_filename}', dpi=200) joblib.dump(mat, f'test_{confusion_matrix_filename}.pkl') return mat, accuracy
def get_confusion(learner: str) -> str: from fastai.vision import ClassificationInterpretation import dill with open(learner, 'rb') as fp: learner = dill.load(fp) interpreter = ClassificationInterpretation.from_learner(learner) return str(interpreter.confusion_matrix())
def run_mnist(input_path, output_path, batch_size, epochs, learning_rate, model=Mnist_NN()): path = Path(input_path) ## Defining transformation ds_tfms = get_transforms( do_flip=False, flip_vert=False, max_rotate=15, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, ) ## Creating Databunch data = (ImageItemList.from_folder(path, convert_mode="L").split_by_folder( train="training", valid="testing").label_from_folder().transform( tfms=ds_tfms, size=28).databunch(bs=batch_size)) ## Defining the learner mlp_learner = Learner(data=data, model=model, loss_func=nn.CrossEntropyLoss(), metrics=accuracy) # Training the model mlp_learner.fit_one_cycle(epochs, learning_rate) val_acc = int( np.round(mlp_learner.recorder.metrics[-1][0].numpy().tolist(), 3) * 1000) ## Saving the model mlp_learner.save("mlp_mnist_stg_1_" + str(val_acc)) ## Evaluation print("Evaluating Network..") interp = ClassificationInterpretation.from_learner(mlp_learner) print(classification_report(interp.y_true, interp.pred_class)) ## Plotting train and validation loss mlp_learner.recorder.plot_losses() plt.savefig(output_path + "/loss.png") mlp_learner.recorder.plot_metrics() plt.savefig(output_path + "/metric.png")
def evaluate_model(model, output_path, plot=True): """ Function to evaluate model performance. Generates Classification report, loss and metric plot. """ interp = ClassificationInterpretation.from_learner(model) print(classification_report(interp.y_true, interp.pred_class)) if plot: # Plotting loss progression with each epoch model.recorder.plot_losses() plt.savefig(output_path + "/loss.png") # Plotting metric progression with each epoch model.recorder.plot_metrics() plt.savefig(output_path + "/metric.png")
def on_train_end(self, **kwargs): "Load the best model." if self.save_model: # Adapted from fast.ai "SaveModelCallback" if self.model_path.is_file(): with self.model_path.open('rb') as model_file: self.learn.load(model_file, purge=False) print('Loaded best saved model from {}'.format( self.model_path)) if self.confusion_matrix: interpret = ClassificationInterpretation.from_learner(self.learn) conf_plt = interpret.plot_confusion_matrix(dpi=self.conf_dpi, return_fig=True, title='Confusion Matrix') self.interpret = interpret wandb.log({"Confusion Matrix": wandb.Image(conf_plt)})
def run_shallownet(input_path, output_path, batch_size, epochs, learning_rate): path = Path(input_path) # Creating Databunch data = ( ImageItemList.from_folder(path) .split_by_folder(train="train", valid="test") .label_from_folder() .transform(tfms=None, size=32) .databunch(bs=batch_size) ) # Defining the learner sn_learner = Learner( data=data, model=ShallowNet(n_class=data.c, size=32, in_channels=3), loss_func=nn.CrossEntropyLoss(), metrics=accuracy, ) # Training the model sn_learner.fit_one_cycle(epochs, learning_rate) val_acc = int( np.round(sn_learner.recorder.metrics[-1][0].numpy().tolist(), 3) * 1000 ) # Saving the model sn_learner.save("sn_cifar10_stg_1_" + str(val_acc)) # Evaluation print("Evaluating Network..") interp = ClassificationInterpretation.from_learner(sn_learner) print(classification_report(interp.y_true, interp.pred_class)) # Plotting train and validation loss sn_learner.recorder.plot_losses() plt.savefig(output_path + "/loss.png") sn_learner.recorder.plot_metrics() plt.savefig(output_path + "/metric.png")
def train(self, epochs=10, firstrun=False, min_lr=None, interpret=False): """Trains the model and saves the best model Use One cycle scheduler for learning rate. The model with least test loss is saved as recentbest.pth in model_dir. Parameters ---------- epochs : int, optional Number of epochs, by default 10 firstrun : bool, optional If the frozen layers exist are unfreezed, by default False min_lr : double, optional The minimum learning rate for differential LRs, by default None interpret : bool, optional Show top losses after training, by default False """ print("Training..") self.learn.fit_one_cycle( epochs, max_lr=slice(min_lr, 1e-3), callbacks=[ callbacks.SaveModelCallback(self.learn, name="recentbest"), callbacks.ReduceLROnPlateauCallback(self.learn, patience=1), ], ) self.learn.recorder.plot_losses() if interpret: self.interpretation = ClassificationInterpretation.from_learner(self.learn) print(self.interpretation.most_confused(min_val=2)) if firstrun: self.learn.save("firstrun") self.learn.unfreeze() self.learn.fit_one_cycle(1) self.findlr()
from fastai.vision import models, get_transforms, cnn_learner, ImageDataBunch, imagenet_stats, ClassificationInterpretation from fastai.metrics import accuracy PATH = 'DATA PATH' tfms = get_transforms(flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.1) data = ImageDataBunch.from_folder(PATH, ds_tfms=tfms, bs=64, size=224, num_workers=4).normalize(imagenet_stats) model = cnn_learner(data, models.resnet34, metrics=accuracy, pretrained=True) model.fit_one_cycle(5) model.save('foodnotfoodv1') model.unfreeze() model.lr_find() model.recorder.plot() model.fit_one_cycle(2, max_lr=slice(1e-5, 1e-4)) interp = ClassificationInterpretation.from_learner(model) interp.plot_confusion_matrix() model.export()
def model_evaluation(learner): '''evaluate the trained model''' learn = learner.load('unfrozen-model') interp = ClassificationInterpretation.from_learner(learn) print(interp.plot_confusion_matrix()) return None