def train_model(clf: CnnClassifier, results_file: TextIO) -> TrainedModel:
'''
Trains a CNN classifier.
Computes the accuracy on the validation set.
Returns both the trained classifier and accuracy.
'''
results_file.write('epoch,train_loss,train_loss_sd,val_accuracy\n')
n_epochs = 100
for i in range(n_epochs):
print(f"epoch {i+1}")
# reset list of per-epoch training loss
loss_train = []
for batch in train_batches:
# train classifier
loss_train.append(clf.train(batch.data, batch.label))
# output as requested
loss_train = np.array(loss_train)
print(f" train loss: {np.mean(loss_train):5.3f} +- "
f"{np.std(loss_train):5.3f}")
# calculate validation accuracy
accuracy = Accuracy()
for batch in val_batches:
# predict and update accuracy
prediction = clf.predict(batch.data)
accuracy.update(prediction, batch.label)
print(f" val acc: {accuracy}")
results_file.write(f"{i},{np.mean(loss_train)},")
results_file.write(f"{np.std(loss_train)},{accuracy.accuracy()}\n")
return TrainedModel(clf, accuracy)
def grid_search(lr_options, momentum_options, nesterov):
"""Make a Grid Search with the given options
Return the best model(TrainedModel) and a Pandas DataFrae with all validation accuracies
"""
models = []
best_model = TrainedModel(None, Accuracy())
# create a dataframe with all validation accuracies
result_df = pd.DataFrame(index=lr_options, columns=momentum_options)
result_df.index.name = 'Learning Rate'
result_df.columns.name = 'Momentum'
# loop over all combinations
for lr in lr_options:
for momentum in momentum_options:
nesterov = True
print('Training with parameters: lr={}, momentum={}, nesterov={} '.format(lr, momentum, nesterov), end='\r')
model = train_model(lr, momentum)
models.append(model)
result_df.at[model.model.lr, model.model.momentum] = model.accuracy.accuracy()
if model.accuracy.__gt__(best_model.accuracy):
best_model = model
return (best_model, result_df)
def train_model(lr: float, momentum: float) -> TrainedModel:
'''
Trains a linear classifier with a given learning rate (lr) and momentum.
Computes the accuracy on the validation set.
Returns both the trained classifier and accuracy.
'''
# TODO implement step 3
clf = LinearClassifier(input_dim=3072,
num_classes=2,
lr=lr,
momentum=momentum,
nesterov=True)
n_epochs = 10
for i in range(n_epochs):
train_batches = iter(bg_training)
i = 1
for batch in train_batches:
if (i < len(bg_training)):
clf.train(data=batch.data, labels=batch.label)
i += 1
else:
break
accuracy = Accuracy()
val_batches = iter(bg_validation)
j = 1
for batch in val_batches:
if (j < len(bg_validation)):
predictions = clf.predict(data=batch.data)
target = batch.label
accuracy.update(predictions, target)
j += 1
else:
break
return TrainedModel(clf, accuracy)
def train_model(lr: float, momentum: float) -> TrainedModel:
'''
Trains a linear classifier with a given learning rate (lr) and momentum.
Computes the accuracy on the validation set.
Returns both the trained classifier and accuracy.
'''
clf = LinearClassifier(input_dim=3072, num_classes=train_data.num_classes(), lr=lr, momentum=momentum, nesterov=False)
n_epochs = 10
for i in range(n_epochs):
for batch in train_batches:
clf.train(batch.data,batch.label)
accuracy = Accuracy()
for batch in val_batches:
prediction = clf.predict(batch.data)
accuracy.update(prediction, batch.label)
return TrainedModel(clf, accuracy)
def train_model(lr: float, momentum: float) -> TrainedModel:
'''
Trains a linear classifier with a given learning rate (lr) and momentum.
Computes the accuracy on the validation set.
Returns both the trained classifier and accuracy.
'''
# Step 3: train linear classifier, 10 epochs
clf = LinearClassifier(3072, train_data.num_classes(), lr, momentum, True)
n_epochs = 10
for i in range(n_epochs):
for batch in train_batches:
# train classifier
clf.train(batch.data, batch.label)
accuracy = Accuracy()
for batch in val_batches:
# predict and update accuracy
prediction = clf.predict(batch.data)
accuracy.update(prediction, batch.label)
return TrainedModel(clf, accuracy)
def train_model(lr: float, momentum: float) -> TrainedModel:
'''
Trains a linear classifier with a given learning rate (lr) and momentum.
Computes the accuracy on the validation set.
Returns both the trained classifier and accuracy.
'''
# TODO implement step 3
clf = LinearClassifier(...)
n_epochs = 10
for i in range(n_epochs):
for batch in train_batches:
# train classifier
accuracy = Accuracy()
for batch in val_batches:
# predict and update accuracy
return TrainedModel(clf, accuracy)
not_improved_since = 0
best_accuracy = 0
stop_epoch = 0
for epoch in range(100):
losses = []
for batch in train_batches:
loss = clf.train(batch.data, batch.label)
losses.append(loss)
losses = np.array(losses)
mean = round(np.mean(losses), 3)
std = round(np.std(losses), 3)
print("epoch {}".format(epoch))
print(" train loss: {} +- {}".format(mean, std))
accuracy = Accuracy()
for batch in val_batches:
predictions = clf.predict(batch.data)
accuracy.update(predictions, batch.label)
acc = round(accuracy.accuracy(), 3)
# Early stopping
if acc > best_accuracy:
stop_epoch = epoch
torch.save(net.state_dict(), MODEL_PATH)
not_improved_since = 0
else:
not_improved_since += 1
if not_improved_since > 5: # if not improved since 5 epochs stop training
break
print(" val acc: accuracy: {}".format(acc))
print("Best model on epoch {}".format(stop_epoch))
NUM_CHANNELS = 3
NUM_CLASSES = 2
pets_training = PetsDataset(dir, Subset.TRAINING)
pets_validation = PetsDataset(dir, Subset.VALIDATION)
pets_test = PetsDataset(dir, Subset.TEST)
batchGenerator_training = BatchGenerator(pets_training, len(pets_training), False,
op=chain([type_cast(dtype=np.float32), vectorize()]))
batchGenerator_validation = BatchGenerator(pets_validation, len(pets_validation), False,
op=chain([type_cast(dtype=np.float32), vectorize()]))
batchGenerator_test = BatchGenerator(pets_test, len(pets_test), False,
op=chain([type_cast(dtype=np.float32), vectorize()]))
best_accuracy = Accuracy()
best_k = -1
results = {}
knn = None
for k in range(1, 100, 40): # grid search example
knn = KnnClassifier(k, IMAGE_HEIGHT*IMAGE_WIDTH*NUM_CHANNELS, NUM_CLASSES)
accuracy = Accuracy()
# train and compute validation accuracy ...
for batch in batchGenerator_training:
knn.train(batch.data, batch.labels)
predictions = None
validation_batch = None
for batch in batchGenerator_validation:
output = self.linear_layer5(output)
output = self.batch_norm_layer5(output)
output = self.linear_layer6(output)
output = self.batch_norm_layer6(output)
output = self.linear_layer7(output)
output = self.batch_norm_layer7(output)
output = self.linear_layer8(output)
output = self.batch_norm_layer8(output)
output = self.linear_layer9(output)
return output
net = CatDogNet()
clf = CnnClassifier(net, (BATCH_SIZE, NUM_CHANNELS, IMAGE_HEIGHT, IMAGE_WIDTH), NUM_CLASSES, lr, wd)
loss_list = []
accuracy = Accuracy()
since = time.time()
for epoch in range(0, EPOCHS):
print("Epoche: ", epoch + 1)
for batch in batchGenerator_training:
loss = clf.train(batch.data, batch.label)
loss_list.append(loss)
loss = np.array(loss_list)
loss_mean = np.mean(loss)
loss_deviation = np.std(loss)
print("Train loss: ", loss_mean, "-+", loss_deviation)
accuracy.reset()
for batch in batchGenerator_validation:
n_epochs = 10
for i in range(n_epochs):
for batch in train_batches:
# train classifier
clf.train(batch.data, batch.label)
accuracy = Accuracy()
for batch in val_batches:
# predict and update accuracy
prediction = clf.predict(batch.data)
accuracy.update(prediction, batch.label)
return TrainedModel(clf, accuracy)
# Step 4: random search for good parameter values
best_model = TrainedModel(None, Accuracy()) # accuracy 0
best_lr = -1
best_momentum = -1
NUM_ATTEMPTS = 1000
# output file: CSV of lr, momentum, accuracy
# this is then used for plotting
f = open('results.csv', 'w')
for i in range(NUM_ATTEMPTS):
# try random lr and momentum in the range of 0 to 1
lr = random.random()
momentum = random.random()
# train model
model = train_model(lr, momentum)
# output hyperparameters & validation accuracy
f.write(f"{lr},{momentum},{model.accuracy.accuracy()}\n")
# did we improve the accuracy?
z_size.append(model.accuracy.accuracy())
#print(x_pos)
z_pos = np.zeros(num_bars)
x_size = np.repeat(0.02, num_bars)
y_size = np.repeat(0.02, num_bars)
ax.bar3d(x_pos, y_pos, z_pos, x_size, y_size, z_size, color='red')
ax.set_xlabel('learning rate')
ax.set_ylabel('momentum')
ax.set_zlabel('accuracy')
plt.show()
#test accuracy:
accuracy = Accuracy()
test_batches = iter(bg_test)
j = 1
for batch in test_batches:
if (j < len(bg_test)):
predictions = best_model.model.predict(data=batch.data)
target = batch.label
accuracy.update(predictions, target)
j += 1
else:
break
print(f"Accuracy on test set of best model: {accuracy}")
import cv2
from dlvc.datasets.pets import PetsDataset
from dlvc.models.linear import LinearClassifier
from dlvc.batches import BatchGenerator
from dlvc.test import Accuracy
from dlvc.dataset import Subset
import dlvc.ops as ops
np.random.seed(0)
pets_train = PetsDataset("../cifar-10-batches-py/", Subset.TRAINING)
pets_val = PetsDataset("../cifar-10-batches-py/", Subset.VALIDATION)
random_accuracy = Accuracy()
validation_accuracy = Accuracy()
train_accuracy = Accuracy()
print('Number of Classes = {}'.format(pets_train.num_classes()))
print('Number of Images = {}'.format(pets_train.__len__()))
print('First 10 Classes >>> {}'.format(pets_train.labels[:10]))
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1 / 127.5),
])
train_batches = BatchGenerator(pets_train, 100, False, op)
def train(lr, wd, operation):
print("Training a network with:")
print("Weight Decay = {}".format(wd))
print("Augmentation = {}".format(operation))
print("Learning Rate = {}".format(lr))
device = torch.device("cuda" if CUDA else "cpu")
img_shape = train_data.image_shape()
num_classes = train_data.num_classes()
net = Net(img_shape, num_classes).to(device)
clf = CnnClassifier(net, (0, *img_shape), num_classes, lr, wd)
op = operations[operation]
train_batches = BatchGenerator(train_data, 128, False, op)
val_batches = BatchGenerator(val_data, 128, False, op)
not_improved_since = 0
best_accuracy = 0
best_loss = 0
stop_epoch = 0
for epoch in range(NR_EPOCHS):
print("Epoch {}/{}".format(epoch, NR_EPOCHS), end="\r")
losses = []
for batch in train_batches:
loss = clf.train(batch.data, batch.label)
losses.append(loss)
losses = np.array(losses)
mean = round(np.mean(losses), 3)
std = round(np.std(losses), 3)
accuracy = Accuracy()
for batch in val_batches:
predictions = clf.predict(batch.data)
accuracy.update(predictions, batch.label)
acc = round(accuracy.accuracy(), 3)
# Early stopping
if acc > best_accuracy:
stop_epoch = epoch
not_improved_since = 0
best_accuracy = acc
best_loss = mean
else:
not_improved_since += 1
if not_improved_since > EARLY_STOPPING: # if not improved since 5 epochs stop training
break
print()
print("Best val accuracy after epoch {}".format(stop_epoch + 1))
print("Validation Accuracy: {}".format(best_accuracy))
print("Train Loss: {}".format(best_loss))
with open(RESULTS_FILE, "a") as file:
file.write("Trained a network with:\n")
file.write("Weight Decay = {}\n".format(wd))
file.write("Augmentation = {}\n".format(operation))
file.write("Learning Rate = {}\n".format(lr))
file.write("---\n")
file.write("Best val accuracy after epoch {}\n".format(stop_epoch + 1))
file.write("Validation Accuracy: {}\n".format(best_accuracy))
file.write("Train Loss: {}\n".format(best_loss))
file.write("\n#################################\n")
print('Training with parameters: lr={}, momentum={}, nesterov={} '.format(lr, momentum, nesterov), end='\r')
model = train_model(lr, momentum)
models.append(model)
result_df.at[model.model.lr, model.model.momentum] = model.accuracy.accuracy()
if model.accuracy.__gt__(best_model.accuracy):
best_model = model
return (best_model, result_df)
lr_options = [0.5, 0.2, 0.1, 0.01, 0.001]
momentum_options = [0.9, 0.5, 0.1, 0.01, 0.001]
start_time = time.time()
best_model, result_df = grid_search(lr_options=lr_options, momentum_options=momentum_options, nesterov=True)
end_time = time.time()
print('Wall time: {}s'.format(round(end_time - start_time, 2)))
print(result_df)
print('\nBest Model:')
print('Parameters: lr={}, momentum={}'.format(best_model.model.lr, best_model.model.momentum))
test_accuracy = Accuracy()
for batch in test_batches:
prediction = best_model.model.predict(batch.data)
test_accuracy.update(prediction, batch.label)
print('Test Accuracy = {}'.format(test_accuracy.accuracy()))
training_batch = load_dataset(Subset.TRAINING, augment=True)
validation_batch = load_dataset(Subset.VALIDATION)
model = get_pretrained_model()
enable_grad(model, False)
if torch.cuda.is_available():
model = model.cuda()
learning_rate = 0.001
weight_decay = 0.001
cnn_cl = cnn.CnnClassifier(model, (3, 224, 224), num_classes=2, lr=learning_rate, wd=weight_decay, adam=False)
loss_list = []
measure = Accuracy()
accuracies = []
mean_loss_list = []
best_accuracy = 0
for epoch in range(1, 301):
if epoch == 100:
enable_grad(model, True) # enable fine-tuning of pre-trained layers at epoch 50
predictions = np.zeros((1, 2))
loss_list = []
labels = []
for training_data in training_batch:
loss = cnn_cl.train(training_data.data, training_data.label)
loss_list.append(loss)
op = ops.chain([ops.vectorize(), ops.type_cast(np.float32)])
return batches.BatchGenerator(dataset, len(dataset), True, op)
if __name__ == "__main__":
training_batch = load_dataset(Subset.TRAINING)
validation_batch = load_dataset(Subset.VALIDATION)
test_batch = load_dataset(Subset.TEST)
training_data = next(iter(training_batch))
validation_data = next(iter(validation_batch))
test_data = next(iter(test_batch))
measure = Accuracy()
accuracies = []
ks = [1, 5, 20, 50, 100]
print('Starting grid search with k={}'.format(ks))
print()
for k in ks:
print('Train classifier with k={}'.format(k))
knn_cl = knn.KnnClassifier(k=k,
input_dim=INPUT_DIM,
num_classes=NUM_CLASSES)
# train the classifier
print('\tRun training... ', end='', flush=True)
knn_cl.train(training_data.data, training_data.label)
])
batchGenerator_training = BatchGenerator(pets_training,
BATCH_SIZE,
shuffle=True,
op=op_chain)
batchGenerator_validation = BatchGenerator(pets_validation,
BATCH_SIZE,
shuffle=False,
op=op_chain)
clf = CnnClassifier(net, (BATCH_SIZE, NUM_CHANNELS, IMAGE_HEIGHT, IMAGE_WIDTH),
NUM_CLASSES, lr, wd)
loss_list = []
best_accuracy = 0.0
accuracy = Accuracy()
epochs_since_best_accuracy = 0
for epoch in range(0, EPOCHS):
print("Epoche: ", epoch + 1)
for batch in batchGenerator_training:
loss = clf.train(batch.data, batch.label)
loss_list.append(loss)
loss = np.array(loss_list)
loss_mean = np.mean(loss)
loss_deviation = np.std(loss)
print("Train loss: ", loss_mean, "-+", loss_deviation)
accuracy.reset()
for batch in batchGenerator_validation:
def grid_search_optimizer(train: Batch, validation: Batch, test: Batch,
s_scope: tuple, input_dim: int, num_classes: int):
best_accuracy_valid = Accuracy()
best_k = 0
accuracy_results = []
k_values = []
for k in range(s_scope[0], s_scope[1], s_scope[2]):
knn_classifier = KnnClassifier(k, input_dim, num_classes)
accuracy = Accuracy()
knn_classifier.train(train.data, train.label)
predictions = knn_classifier.predict(validation.data)
accuracy.update(predictions, validation.label)
current_accuracy = accuracy.accuracy()
accuracy_results.append(current_accuracy)
k_values.append(k)
print("I am working on k = " + str(k) + ' and ' + str(current_accuracy))
if accuracy > best_accuracy_valid:
best_accuracy_valid = accuracy
best_k = k
best_knn_classifier = KnnClassifier(best_k, input_dim, num_classes)
accuracy = Accuracy()
best_knn_classifier.train(train.data, train.label)
predictions = best_knn_classifier.predict(test.data)
accuracy.update(predictions, test.label
)
best_accuracy_test = accuracy.accuracy()
return best_k, best_accuracy_valid, best_accuracy_test, accuracy_results, k_values