def predict_dataset(
ds_type: str = 'test',
cpu=True,
sample_size=config.BEST_MODEL_PARAMS['sample_size'],
with_oversampling=config.BEST_MODEL_PARAMS['with_oversampling'],
with_focal_loss=config.BEST_MODEL_PARAMS['with_focal_loss'],
with_weighted_loss=config.BEST_MODEL_PARAMS['with_weighted_loss'],
confusion_matrix_filename='test_confusion_matrix'):
learn = load_saved_learner(sample_size=sample_size,
with_oversampling=with_oversampling,
with_focal_loss=with_focal_loss,
with_weighted_loss=with_weighted_loss,
cpu=cpu)
classes = {c: i for i, c in enumerate(learn.data.classes)}
# create a DF with filepath and class label (int)
data = pd.read_csv(config.PROCESSED_DATA_DIR / 'labels_full.csv')
data = data[data['ds_type'] == ds_type][['name', 'label']]
data['label_int'] = data.label.apply(lambda x: classes[x])
print(f'Running predictions on {data.shape[0]} data samples')
data['pred_probability'] = pd.Series(dtype=object)
for k, i in enumerate(data.index):
pred = learn.predict(
open_image(config.PROCESSED_DATA_DIR / data.loc[i, 'name'],
convert_mode='L'))
data.loc[i, 'y_pred'] = pred[0].data.item()
data.at[i, 'pred_probability'] = pred[2].numpy()
if k % 200 == 0 and k > 0:
print(f'{k} images done..')
data.to_csv(config.DATA_DIR / 'predictions.csv', index=False)
print(f'Building classification interpretation..')
interp = ClassificationInterpretation(
learn=learn,
losses=np.zeros(data.shape[0]),
preds=tensor(data['pred_probability'].to_list()),
y_true=tensor(data.label_int.to_list()))
mat = interp.confusion_matrix()
# sum diagonal / all data_size *100
accuracy = np.trace(mat) / mat.sum() * 100
print(mat)
print(f'Accuracy: {accuracy}')
interp.plot_confusion_matrix(return_fig=True).savefig(
f'test_{confusion_matrix_filename}', dpi=200)
joblib.dump(mat, f'test_{confusion_matrix_filename}.pkl')
return mat, accuracy
def get_confusion(learner: str) -> str:
from fastai.vision import ClassificationInterpretation
import dill
with open(learner, 'rb') as fp:
learner = dill.load(fp)
interpreter = ClassificationInterpretation.from_learner(learner)
return str(interpreter.confusion_matrix())
def run_mnist(input_path,
output_path,
batch_size,
epochs,
learning_rate,
model=Mnist_NN()):
path = Path(input_path)
## Defining transformation
ds_tfms = get_transforms(
do_flip=False,
flip_vert=False,
max_rotate=15,
max_zoom=1.1,
max_lighting=0.2,
max_warp=0.2,
)
## Creating Databunch
data = (ImageItemList.from_folder(path, convert_mode="L").split_by_folder(
train="training", valid="testing").label_from_folder().transform(
tfms=ds_tfms, size=28).databunch(bs=batch_size))
## Defining the learner
mlp_learner = Learner(data=data,
model=model,
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy)
# Training the model
mlp_learner.fit_one_cycle(epochs, learning_rate)
val_acc = int(
np.round(mlp_learner.recorder.metrics[-1][0].numpy().tolist(), 3) *
1000)
## Saving the model
mlp_learner.save("mlp_mnist_stg_1_" + str(val_acc))
## Evaluation
print("Evaluating Network..")
interp = ClassificationInterpretation.from_learner(mlp_learner)
print(classification_report(interp.y_true, interp.pred_class))
## Plotting train and validation loss
mlp_learner.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
mlp_learner.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def evaluate_model(model, output_path, plot=True):
"""
Function to evaluate model performance.
Generates Classification report, loss and metric plot.
"""
interp = ClassificationInterpretation.from_learner(model)
print(classification_report(interp.y_true, interp.pred_class))
if plot:
# Plotting loss progression with each epoch
model.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
# Plotting metric progression with each epoch
model.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def on_train_end(self, **kwargs):
"Load the best model."
if self.save_model:
# Adapted from fast.ai "SaveModelCallback"
if self.model_path.is_file():
with self.model_path.open('rb') as model_file:
self.learn.load(model_file, purge=False)
print('Loaded best saved model from {}'.format(
self.model_path))
if self.confusion_matrix:
interpret = ClassificationInterpretation.from_learner(self.learn)
conf_plt = interpret.plot_confusion_matrix(dpi=self.conf_dpi, return_fig=True, title='Confusion Matrix')
self.interpret = interpret
wandb.log({"Confusion Matrix": wandb.Image(conf_plt)})
def run_shallownet(input_path, output_path, batch_size, epochs, learning_rate):
path = Path(input_path)
# Creating Databunch
data = (
ImageItemList.from_folder(path)
.split_by_folder(train="train", valid="test")
.label_from_folder()
.transform(tfms=None, size=32)
.databunch(bs=batch_size)
)
# Defining the learner
sn_learner = Learner(
data=data,
model=ShallowNet(n_class=data.c, size=32, in_channels=3),
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy,
)
# Training the model
sn_learner.fit_one_cycle(epochs, learning_rate)
val_acc = int(
np.round(sn_learner.recorder.metrics[-1][0].numpy().tolist(), 3) * 1000
)
# Saving the model
sn_learner.save("sn_cifar10_stg_1_" + str(val_acc))
# Evaluation
print("Evaluating Network..")
interp = ClassificationInterpretation.from_learner(sn_learner)
print(classification_report(interp.y_true, interp.pred_class))
# Plotting train and validation loss
sn_learner.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
sn_learner.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def train(self, epochs=10, firstrun=False, min_lr=None, interpret=False):
"""Trains the model and saves the best model
Use One cycle scheduler for learning rate. The model with least test
loss is saved as recentbest.pth in model_dir.
Parameters
----------
epochs : int, optional
Number of epochs, by default 10
firstrun : bool, optional
If the frozen layers exist are unfreezed, by default False
min_lr : double, optional
The minimum learning rate for differential LRs, by default None
interpret : bool, optional
Show top losses after training, by default False
"""
print("Training..")
self.learn.fit_one_cycle(
epochs,
max_lr=slice(min_lr, 1e-3),
callbacks=[
callbacks.SaveModelCallback(self.learn, name="recentbest"),
callbacks.ReduceLROnPlateauCallback(self.learn, patience=1),
],
)
self.learn.recorder.plot_losses()
if interpret:
self.interpretation = ClassificationInterpretation.from_learner(self.learn)
print(self.interpretation.most_confused(min_val=2))
if firstrun:
self.learn.save("firstrun")
self.learn.unfreeze()
self.learn.fit_one_cycle(1)
self.findlr()
from fastai.vision import models, get_transforms, cnn_learner, ImageDataBunch, imagenet_stats, ClassificationInterpretation
from fastai.metrics import accuracy
PATH = 'DATA PATH'
tfms = get_transforms(flip_vert=True,
max_lighting=0.1,
max_zoom=1.05,
max_warp=0.1)
data = ImageDataBunch.from_folder(PATH,
ds_tfms=tfms,
bs=64,
size=224,
num_workers=4).normalize(imagenet_stats)
model = cnn_learner(data, models.resnet34, metrics=accuracy, pretrained=True)
model.fit_one_cycle(5)
model.save('foodnotfoodv1')
model.unfreeze()
model.lr_find()
model.recorder.plot()
model.fit_one_cycle(2, max_lr=slice(1e-5, 1e-4))
interp = ClassificationInterpretation.from_learner(model)
interp.plot_confusion_matrix()
model.export()
def model_evaluation(learner):
'''evaluate the trained model'''
learn = learner.load('unfrozen-model')
interp = ClassificationInterpretation.from_learner(learn)
print(interp.plot_confusion_matrix())
return None