def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
#load data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
#hyperparameters
eta = FLAGS.learning_rate
eps = 1e-6 # convergence criterion
max_steps = FLAGS.max_steps
b_size = FLAGS.batch_size
#test_data
x_test = cifar10["test"].images
y_test = cifar10["test"].labels
#get usefull dimensions
n_channels = np.size(x_test, 1)
n_classes = np.size(y_test, 1)
n_batches = np.size(x_test, 0) // b_size
#load whole train data ############################################################
x_train = cifar10["train"].images
x_train = torch.tensor(x_train, requires_grad=False).type(dtype).to(device)
n_train_batches = np.size(x_train, 0) // b_size
#initialize the ConvNet model
model = ConvNet(n_channels, n_classes)
get_loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=eta)
model.to(device)
train_loss = []
test_loss = []
train_acc = []
test_acc = []
for step in range(max_steps):
#get batch
x, y = cifar10['train'].next_batch(b_size)
x = torch.tensor(x).type(dtype).to(device)
y = torch.tensor(y).type(dtype).to(device)
#forward pass
pred = model.forward(x)
#get training loss
current_loss = get_loss(pred, y.argmax(dim=1))
optimizer.zero_grad()
#get training loss gradient
current_loss.backward()
#get training accuracy
current_train_acc = accuracy(pred, y)
optimizer.step()
#free memory up
pred.detach()
x.detach()
y.detach()
#select evaluation step
if (step % FLAGS.eval_freq) == 0:
# c_train_loss = current_loss.data.item()
# train_loss.append(c_train_loss)
# train_acc.append(current_train_acc)
c_train_loss = 0
current_train_acc = 0
c_test_loss = 0
current_test_acc = 0
#loop through train set in batches ######################################################
for test_batch in range(n_train_batches):
#load test data
x_train, y_train = cifar10['train'].next_batch(b_size)
x_train = torch.tensor(
x_train, requires_grad=False).type(dtype).to(device)
y_train = torch.tensor(
y_train, requires_grad=False).type(dtype).to(device)
#get test batch results
train_pred = model.forward(x_train)
current_train_loss = get_loss(train_pred,
y_train.argmax(dim=1))
c_train_loss += current_train_loss.data.item()
current_train_acc += accuracy(train_pred, y_train)
#free memory up
train_pred.detach()
x_train.detach()
y_train.detach()
#loop through test set in batches
for test_batch in range(n_batches):
#load test data
x_test, y_test = cifar10['test'].next_batch(b_size)
x_test = torch.tensor(
x_test, requires_grad=False).type(dtype).to(device)
y_test = torch.tensor(
y_test, requires_grad=False).type(dtype).to(device)
#get test batch results
test_pred = model.forward(x_test)
current_test_loss = get_loss(test_pred, y_test.argmax(dim=1))
c_test_loss += current_test_loss.data.item()
current_test_acc += accuracy(test_pred, y_test)
#free memory up
test_pred.detach()
x_test.detach()
y_test.detach()
#get full training set results #########################################################
c_train_loss = c_train_loss / n_train_batches
current_train_acc = current_train_acc / n_train_batches
train_loss.append(c_train_loss)
train_acc.append(current_train_acc)
#get full test set results
c_test_loss = c_test_loss / n_batches
current_test_acc = current_test_acc / n_batches
test_loss.append(c_test_loss)
test_acc.append(current_test_acc)
print('\nStep ', step, '\n------------\nTraining Loss = ',
round(c_train_loss, 4),
', Train Accuracy = ', current_train_acc, '\nTest Loss = ',
round(c_test_loss, 4), ', Test Accuracy = ',
round(current_test_acc, 4))
if step > 0 and abs(test_loss[(int(step / FLAGS.eval_freq))] -
test_loss[int(step / FLAGS.eval_freq) -
1]) < eps:
break
plot_graphs(train_loss,
'Training Loss',
'orange',
test_loss,
'Test Loss',
'blue',
title='Adams optimization',
ylabel='Loss',
xlabel='Steps')
plot_graphs(train_acc,
'Training Accuracy',
'darkorange',
test_acc,
'Test Accuracy',
'darkred',
title='Adams optimization',
ylabel='Accuracy',
xlabel='Steps')
#save results:
np.save('train_loss', train_loss)
np.save('train_acc', train_acc)
np.save('test_loss', test_loss)
np.save('test_acc', test_acc)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# Load data
cifar10 = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir,
one_hot=False,
validation_size=5000)
train_set = cifar10['train']
test_set = cifar10['test']
val_set = cifar10['validation']
# Initialize model
n_channels = len(train_set.images[0].shape)
n_classes = train_set.labels.max() + 1
# set device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = ConvNet(n_channels, n_classes)
model = model.to(device)
model = nn.DataParallel(model)
cross_entropy = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
total_loss = 0
losses = []
val_acc = []
train_acc = []
for i in range(FLAGS.max_steps + 1):
# prepare batch
x, y = train_set.next_batch(FLAGS.batch_size)
x, y = torch.tensor(x), torch.tensor(y, dtype=torch.long)
x, y = x.to(device), y.to(device)
# forward pass
out = model(x)
loss = cross_entropy(out, y)
total_loss += loss.item()
# keep track of training accuracy
train_acc.append(accuracy(out, y))
# backward pass
model.zero_grad()
loss.backward()
optimizer.step()
if i % FLAGS.eval_freq == 0 and i != 0:
with torch.no_grad():
val_inputs = test_set.images
val_inputs = torch.tensor(val_inputs)
val_inputs = val_inputs.to(device)
pred = model(val_inputs)
targ = torch.tensor(test_set.labels)
targ = targ.to(device)
acc = accuracy(pred, targ)
losses.append(total_loss)
val_acc.append(acc)
print()
print("- - - - - - - - - -")
print('- STEPS:\t\t\t', i)
print('- TRAIN ACC: \t\t\t', np.array(train_acc).mean())
print('- VALIDATION ACC:\t\t', acc)
print("- - - - - - - - - -")
train_acc = []
total_loss = 0
print("Loss over time: \t", losses)
print("Val acc over time: \t", val_acc)
with open('cnn_data.dill', 'wb') as f:
dill.dump({'train_loss': losses, 'val_acc': val_acc}, f)
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
print(train.images.shape)
# transform = transforms.Compose(
# [transforms.Resize((224, 224)),
# transforms.ToTensor(),
# transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
if FLAGS.model == 'ALEX':
train.images = torch.Tensor([resize(img) for img in train.images])
test.images = torch.Tensor([resize(img) for img in test.images])
print('doneee')
model = models.alexnet(pretrained=True)
torch.save(model.state_dict(), 'alexnet.txt')
# model = torch.load('alexnet.txt')
for param in model.features.parameters():
param.requires_grad = False
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.classifier.parameters(),
lr=FLAGS.learning_rate)
else:
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(),
lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(2): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
if FLAGS.model == 'ADAM': x = resizer(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
if FLAGS.model == 'ADAM': batch_x = resizer(batch_x)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).float().to(device)
y = torch.from_numpy(y).float().to(device)
n_classes = y.shape[1]
n_channels = x.shape[1]
CNN = ConvNet(n_channels, n_classes)
CNN.to(device)
if OPTIMIZER_DEFAULT == 'SGD':
optimizer = optim.SGD(CNN.parameters())
elif OPTIMIZER_DEFAULT == 'ADAM':
optimizer = optim.Adam(CNN.parameters())
else:
print('Try SGD or ADAM...')
loss = nn.CrossEntropyLoss()
l_list = list()
t_list = list()
train_acc = list()
test_acc = list()
iterations = list()
print('\nTraining...')
for i in range(FLAGS.max_steps):
optimizer.zero_grad()
s_pred = CNN(x)
f_loss = loss(s_pred, y.argmax(dim=1))
f_loss.backward()
optimizer.step()
if i % FLAGS.eval_freq == 0:
iterations.append(i + 1)
l_list.append(round(f_loss.item(), 3))
train_acc.append(accuracy(s_pred, y))
test_size = cifar10['test'].labels.shape[0]
iter_num = 10
tmp_size = int(test_size / iter_num)
tmp_correct = 0
tmp_loss = 0
for j in range(iter_num):
t_x, t_y = cifar10['test'].next_batch(tmp_size)
t_x = torch.from_numpy(t_x).float().to(device)
t_y = torch.from_numpy(t_y).float().to(device)
t_pred = CNN(t_x)
tmp_loss += loss(t_pred, t_y.argmax(dim=1)).item()
tmp_correct += accuracy(t_pred, t_y) * tmp_size
test_acc.append(tmp_correct / test_size)
t_list.append(round(tmp_loss / iter_num, 3))
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).float().to(device)
y = torch.from_numpy(y).float().to(device)
print('Done!\n')
print('Training Losses:', l_list)
print('Test Losses:', t_list)
print('Training Accuracies:', train_acc)
print('Test Accuracies:', test_acc)
print('Best Test Accuracy:', max(test_acc))
fig, axs = plt.subplots(1, 2, figsize=(10, 5))
axs[0].plot(iterations, train_acc, iterations, test_acc)
axs[0].set_xlabel('Iteration')
axs[0].set_ylabel('Accuracy')
axs[0].legend(('train', 'test'))
axs[1].plot(iterations, l_list, iterations, t_list)
axs[1].set_xlabel('Iteration')
axs[1].set_ylabel('Loss')
axs[1].legend(('train', 'test'))
fig.tight_layout()
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
cifar10 = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train_data = cifar10['train']
n_channels = train_data.images.shape[0]
n_classes = train_data.labels.shape[1]
net = ConvNet(n_channels, n_classes)
net.to(device)
params = net.parameters()
optimizer = torch.optim.Adam(params, lr=FLAGS.learning_rate)
criterion = torch.nn.CrossEntropyLoss()
rloss = 0
train_acc_plot = []
test_acc_plot = []
loss_train = []
loss_test = []
print(f'[DEBUG] start training.... Max steps {FLAGS.max_steps}')
for i in range(0, FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x, y = torch.from_numpy(x).float().to(device), torch.from_numpy(
y).float().to(device)
out = net.forward(x)
loss = criterion(out, y.argmax(1))
optimizer.zero_grad()
loss.backward()
optimizer.step()
rloss += loss.item()
if i % FLAGS.eval_freq == 0:
train_accuracy = accuracy(out, y)
with torch.no_grad():
test_accuracys, test_losses = [], []
for j in range(0, FLAGS.max_steps):
test_x, test_y = cifar10['test'].next_batch(
FLAGS.batch_size)
test_x, test_y = torch.from_numpy(test_x).float().to(
device), torch.from_numpy(test_y).float().to(device)
test_out = net.forward(test_x)
test_loss = criterion(test_out, test_y.argmax(1))
test_accuracy = accuracy(test_out, test_y)
if device == 'cpu':
test_losses.append(test_loss)
else:
test_losses.append(test_loss.cpu().data.numpy())
test_accuracys.append(test_accuracy)
t_acc = np.array(test_accuracys).mean()
t_loss = np.array(test_losses).mean()
train_acc_plot.append(train_accuracy)
test_acc_plot.append(t_acc)
loss_train.append(rloss / (i + 1))
loss_test.append(t_loss)
print(
f"iter {i}, train_loss_avg {rloss/(i + 1)}, test_loss_avg {t_loss}, train_acc {train_accuracy}, test_acc_avg {t_acc}"
)
print('[DEBUG] Done training')
if FLAGS.plot:
print('[DEBUG] Start plotting...')
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
ax1.plot(np.arange(len(train_acc_plot)),
train_acc_plot,
label='training')
ax1.plot(np.arange(len(test_acc_plot)), test_acc_plot, label='testing')
ax1.set_title('Training evaluation with batch size ' +
str(FLAGS.batch_size) + '\n learning rate ' +
str(FLAGS.learning_rate) + '\n best accuracy ' +
str(max(test_acc_plot)))
ax1.set_ylabel('Accuracy')
ax1.legend()
ax2.plot(np.arange(len(loss_train)), loss_train, label='Train Loss')
ax2.plot(np.arange(len(loss_test)), loss_test, label='Test Loss')
ax2.set_title('Loss evaluation')
ax2.set_ylabel('Loss')
ax2.legend()
plt.xlabel('Iteration')
plt.savefig('convnet.png')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
# select which device to train the model on
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# compute the input size of the MLP
input_size, n_classes = 3 * 32 * 32, 10
# if no pretrained model is passed to the commandline the convnet model will be initialized
if FLAGS.model == "custom":
model = ConvNet(3, n_classes).to(device)
else:
# check if the requested pretrained is available
assert FLAGS.model in pretrained_dict, "Model not available in pre_trained dict, given: {}, available: {}".format(
FLAGS.model, pretrained_dict.keys())
model = pretrained_dict[FLAGS.model](pretrained=True)
# image transforms needed to be alligned with the type of input the pretrained model is used to
normalize = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
resize = T.Resize(256)
centerCrop = T.CenterCrop(224)
# break out the last layer and add a linear layer that has 10 outputs instead of 1000
layers = []
first_linear = False
for child in model.children():
if list(child.children()) == []:
if isinstance(child, nn.Linear) and not first_linear:
layers.append(nn.Flatten())
first_linear = True
layers.append(child)
else:
for grandchild in child.children():
if isinstance(grandchild, nn.Linear) and not first_linear:
layers.append(nn.Flatten())
first_linear = True
layers.append(grandchild)
model = nn.Sequential(
*layers[:-1],
nn.Linear(in_features=layers[-1].in_features, out_features=10))
model.to(device)
# freeze the layers that are pretrained
for child in list(model.children())[:-1]:
for param in child.parameters():
param.requires_grad = False
# define the dataset, loss function and optimizer
dataset = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_fn = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=FLAGS.learning_rate)
for step in range(FLAGS.max_steps):
X_train, y_train = dataset['train'].next_batch(FLAGS.batch_size)
optimizer.zero_grad()
# from to normalize the data for pretrained model https://discuss.pytorch.org/t/how-to-efficiently-normalize-a-batch-of-tensor-to-0-1/65122
if FLAGS.model != "custom":
X_train, y_train = torch.tensor(X_train).float(), torch.tensor(
y_train).float().to(device)
X_train -= X_train.min(1, keepdim=True)[0]
X_train /= X_train.max(1, keepdim=True)[0]
X_train = torch.tensor([
normalize(T.ToTensor()(centerCrop(resize(
T.ToPILImage()(x))))).numpy() for x in X_train
]).to(device)
else:
# move to correct device and shape for MLP
X_train, y_train = torch.tensor(X_train).float().to(
device), torch.tensor(y_train).float().to(device)
predictions = model(X_train)
train_loss = loss_fn(predictions, y_train.argmax(1).long())
train_loss.backward()
optimizer.step()
# add the loss and accuracy to the lists for plotting
train_overall_loss.append(train_loss.cpu().detach().sum())
train_overall_accuracy.append(
accuracy(predictions.cpu().detach(),
y_train.cpu().detach()))
train_x_axis.append(step)
# test the model when eval freq is reached or if it is the last step
if not step % FLAGS.eval_freq or step + 1 == FLAGS.max_steps:
model.eval()
test_accuracies, test_losses_list = [], []
# test batchwise since it doesnot fit my gpu
for X_test, y_test in cifar_test_generator(dataset):
if FLAGS.model != "custom":
X_test, y_test = torch.tensor(X_test).float(
), torch.tensor(y_test).float().to(device)
X_test -= X_test.min(1, keepdim=True)[0]
X_test /= X_test.max(1, keepdim=True)[0]
X_test = torch.tensor([
normalize(T.ToTensor()(centerCrop(
resize(T.ToPILImage()(x))))).numpy()
for x in X_test
]).to(device)
else:
# move to correct device and shape for MLP
X_test, y_test = torch.tensor(X_test).float().to(
device), torch.tensor(y_test).float().to(device)
predictions = model(X_test)
test_loss = loss_fn(predictions, y_test.argmax(1).long())
test_accuracy = accuracy(predictions, y_test)
# add the values to compute the average loss and accuracy for the entire testset
test_accuracies.append(test_accuracy.cpu().detach())
test_losses_list.append(test_loss.cpu().detach().sum())
print(
"[{:5}/{:5}] Train loss {:.5f} Test loss {:.5f} Test accuracy {:.5f}"
.format(step, FLAGS.max_steps, train_loss, test_loss,
sum(test_accuracies) / len(test_accuracies)))
test_overall_accuracy.append(
sum(test_accuracies) / len(test_accuracies))
test_overall_loss.append(
sum(test_losses_list) / len(test_losses_list))
test_x_axis.append(step)
model.train()
# freeze the pretrained layers
if FLAGS.model != "custom":
for child in list(model.children())[:-1]:
for param in child.parameters():
param.requires_grad = False
plt.plot(train_x_axis, train_overall_loss, label="Avg Train loss")
plt.plot(test_x_axis, test_overall_loss, label="Avg Test loss")
plt.legend()
plt.savefig("convnet_loss_curve")
plt.show()
plt.plot(train_x_axis,
train_overall_accuracy,
label="Train batch accuracy")
plt.plot(test_x_axis, test_overall_accuracy, label="Test set accuracy")
plt.legend()
plt.savefig("convnet_accuracy_curve")
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
# DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
########################
# PUT YOUR CODE HERE #
#######################
eval_batch = 5
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
x_train, y_train = cifar10['train'].next_batch(FLAGS.batch_size)
x_train = torch.from_numpy(x_train).to(device)
y_train = torch.from_numpy(y_train).to(device)
crossent_softmax = nn.CrossEntropyLoss()
conv = ConvNet(x_train.shape[1], y_train.shape[1])
optimizer = torch.optim.Adam(conv.parameters(), lr=FLAGS.learning_rate)
conv.to(device)
train_accs = []
train_losses = []
eval_accs = []
eval_losses = []
for i in np.arange(FLAGS.max_steps):
print('\nStep: {}\n'.format(i))
print('Training: ')
optimizer.zero_grad()
logits = conv(x_train)
train_loss = crossent_softmax(logits, y_train.argmax(dim=-1))
train_acc = accuracy(logits, y_train)
print('loss: {:.4f}, acc: {:.4f}\n'.format(train_loss, train_acc))
train_loss.backward()
optimizer.step()
x_train, y_train = cifar10['train'].next_batch(FLAGS.batch_size)
x_train = torch.from_numpy(x_train).to(device)
y_train = torch.from_numpy(y_train).to(device)
if i % FLAGS.eval_freq == 0:
with torch.no_grad():
print('Evaluation: ')
# x_eval, y_eval = cifar10['test'].images, cifar10['test'].labels
x_eval, y_eval = cifar10['test'].next_batch(FLAGS.batch_size *
eval_batch)
x_eval = torch.from_numpy(x_eval).to(device)
y_eval = torch.from_numpy(y_eval).to(device)
logits = conv(x_eval)
eval_loss = crossent_softmax(logits, y_eval.argmax(dim=-1))
eval_acc = accuracy(logits, y_eval)
train_losses.append(train_loss)
train_accs.append(train_acc)
eval_losses.append(eval_loss)
eval_accs.append(eval_acc)
print('loss: {:.4f}, acc: {:.4f}'.format(eval_loss, eval_acc))
print('Evaluation: ')
# x_eval, y_eval = cifar10['test'].images, cifar10['test'].labels
x_eval, y_eval = cifar10['test'].next_batch(FLAGS.batch_size * eval_batch)
x_eval = torch.from_numpy(x_eval).to(device)
y_eval = torch.from_numpy(y_eval).to(device)
logits = conv(x_eval)
eval_loss = crossent_softmax(logits, y_eval.argmax(dim=-1))
eval_acc = accuracy(logits, y_eval)
train_losses.append(train_loss)
train_accs.append(train_acc)
eval_losses.append(eval_loss)
eval_accs.append(eval_acc)
print('loss: {:.4f}, acc: {:.4f}'.format(eval_loss, eval_acc))
print('Finished training.')
plt.figure(figsize=(10, 5))
plt.plot(np.arange(len(train_losses)), train_losses, label='training loss')
plt.plot(np.arange(len(eval_losses)), eval_losses, label='evaluation loss')
plt.legend()
plt.xlabel('Iterations [x{}]'.format(FLAGS.eval_freq))
plt.savefig('results/conv_loss.png', bbox_inches='tight')
plt.figure(figsize=(10, 5))
plt.plot(np.arange(len(train_accs)), train_accs, label='training accuracy')
plt.plot(np.arange(len(eval_accs)), eval_accs, label='evaluation accuracy')
plt.legend()
plt.xlabel('Iterations [x{}]'.format(FLAGS.eval_freq))
plt.savefig('results/conv_acc.png', bbox_inches='tight')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
print()
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train = cifar10['train']
input_channels = 3
output_class = 10
net = ConvNet(input_channels, output_class)
net.to(device)
if OPTIMIZER_DEFAULT == 'ADAM':
optimizer = torch.optim.Adam(net.parameters(),lr=FLAGS.learning_rate)
CrossEntropyLoss = nn.CrossEntropyLoss()
loss_iter = []
loss_mean = []
acc_iter = []
loss_sum = 0.0
for iter_n in np.arange(0,FLAGS.max_steps):
x, labels = train.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
labels = np.argwhere(labels>0)
labels = torch.from_numpy(labels[:,1]).to(device)
optimizer.zero_grad()
net.train()
predictions = net.forward(x)
loss = CrossEntropyLoss(predictions,labels.long())
loss_iter.append(loss.item())
loss_sum += loss.item()
loss_mean.append(np.mean(loss_iter[:-50:-1]))
loss.backward()
optimizer.step()
print("Iter: {}, training Loss: {}".format(iter_n, "%.3f"%loss.item()))
if (iter_n+1) % int(FLAGS.eval_freq) == 0:
print("....Testing.....\n")
test = cifar10['test']
acc = []
with torch.no_grad():
for _ in np.arange(0,(test.num_examples//FLAGS.batch_size)):
x, t = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
t = torch.tensor(t).to(device)
net.eval()
y = net.forward(x)
acc.append(accuracy(y,t))
acc_iter.append(np.mean(acc))
print("Testing accuracy: {}".format("%.3f"%np.mean(acc)))
print()
plt.plot(loss_iter,'r-',alpha=0.1, label="Batch loss")
plt.plot(loss_mean,'b-', label="Average loss")
plt.legend()
plt.xlabel("Iterations")
plt.ylabel("Loss")
plt.title("Training Loss")
plt.grid(True)
plt.show()
plt.close()
plt.plot(acc_iter,'g-')
plt.xlabel("Iterations")
plt.ylabel("Accuracy")
plt.grid(True)
plt.title("Test Accuracy")
plt.show()
plt.close()
print()
print("COMPLETED")
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
lr = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
optim = FLAGS.optimizer
#fetch data
cifar10 = cifar10_utils.get_cifar10(data_dir)
n_classes = 10
n_channels = 3
eval_rounds = int(np.ceil(cifar10['test']._num_examples / batch_size))
model = ConvNet(n_channels, n_classes)
ce = torch.nn.CrossEntropyLoss()
pars = model.parameters()
# optimizer
optim_pars = {'params': pars, 'lr': lr, 'weight_decay': FLAGS.weight_decay}
if optim == 'adadelta':
optimizer = torch.optim.Adadelta(**optim_pars)
elif optim == 'adagrad':
optimizer = torch.optim.Adagrad(**optim_pars)
elif optim == 'rmsprop':
optimizer = torch.optim.RMSprop(**optim_pars)
elif optim == 'adam':
optimizer = torch.optim.Adam(**optim_pars)
else: # SGD
optimizer = torch.optim.SGD(**optim_pars)
model.to(device)
eval_i = 0
cols = ['train_acc', 'test_acc', 'train_loss', 'test_loss', 'secs']
# train
results = []
name = f'convnet-pytorch-{optim}'
with SummaryWriter(name) as w:
for step in tqdm(range(FLAGS.max_steps)):
optimizer.zero_grad()
X, y = cifar10['train'].next_batch(batch_size)
X = torch.tensor(X).type(dtype).to(device)
train_predictions = model.forward(X)
X.detach()
y = torch.tensor(y).type(dtype).to(device)
train_acc = accuracy(train_predictions, y)
idx_train = torch.argmax(y, dim=-1).long()
y.detach()
train_loss = ce(train_predictions, idx_train)
train_predictions.detach()
# stop if loss has converged!
check = 10
if len(results) >= 2 * check:
threshold = 1e-6
losses = [result['test_loss'] for result in results]
current = np.mean(losses[-check:])
prev = np.mean(losses[-2 * check:-check])
if (prev - current) < threshold:
break
# # at each epoch, we divide the learning rate by this if the dev accuracy decreases
# if dev_acc > prev_acc:
# lr /= learning_decay
# prev_acc = dev_acc
train_loss.backward()
optimizer.step()
# evaluate
if step % FLAGS.eval_freq == 0:
time = int(step / FLAGS.eval_freq)
start = timer()
test_accs = []
test_losses = []
for t in range(eval_rounds):
X, y = cifar10['test'].next_batch(batch_size)
X = torch.tensor(
X, requires_grad=False).type(dtype).to(device)
y = torch.tensor(
y, requires_grad=False).type(dtype).to(device)
test_predictions = model.forward(X)
X.detach()
test_accs.append(accuracy(test_predictions, y))
test_losses.append(
ce(test_predictions, y.argmax(dim=1)).item())
test_predictions.detach()
y.detach()
end = timer()
secs = end - start
test_acc = np.mean(test_accs)
test_loss = np.mean(test_losses)
vals = [train_acc, test_acc, train_loss, test_loss, secs]
stats = dict(
zip(cols, [
np.asscalar(i.detach().cpu().numpy().take(0))
if isinstance(i, torch.Tensor) else np.asscalar(i)
if isinstance(i, (np.ndarray, np.generic)) else i
for i in vals
]))
print(
yaml.dump({
k: round(i, 3) if isinstance(i, float) else i
for k, i in stats.items()
}))
w.add_scalars('metrics', stats, time)
results.append(stats)
df = pd.DataFrame(results, columns=cols)
meta = {
'framework': 'pytorch',
'algo': 'convnet',
'optimizer': optim,
'batch_size': FLAGS.batch_size,
'learning_rate': FLAGS.learning_rate,
'dnn_hidden_units': '',
'weight_decay': FLAGS.weight_decay,
'max_steps': FLAGS.max_steps,
}
for k, v in meta.items():
df[k] = v
output_file = 'results/results.csv' # f'{name}.csv'
if os.path.isfile(output_file):
df.to_csv(f'{name}.csv', header=False, mode='a')
else:
df.to_csv(f'{name}.csv', header=True, mode='w')
torch.save(model.state_dict(), f'{name}.pth')
print('done!')
return test_loss
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on
the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
#all external parameters in a readable format.
lr = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
#fetch data
data = cifar10_utils.get_cifar10(data_dir)
n_classes = 10
n_channels = 3
#number of iterations to train the data in the whole dataset:
n_iter = 1 # int(np.ceil(data["train"]._num_examples/batch_size))
#number of evaluations
num_evals = int(np.ceil(data['test']._num_examples / batch_size))
#load model
cnn_model = ConvNet(n_channels, n_classes)
#Loss function
loss_XE = torch.nn.CrossEntropyLoss()
#keep track of how loss and accuracy evolves over time.
loss_train = np.zeros(max_steps + 1) #loss on training data
acc_train = np.zeros(max_steps + 1) #accuracy on training data
loss_eval = np.zeros(max_steps + 1) #loss on test data
acc_eval = np.zeros(max_steps + 1) #accuracy on test data
#Optimizer
optmizer = optim.Adam(cnn_model.parameters(), lr=lr)
#let's put some gpus to work!
cnn_model.to(device)
#index to keep track of the evaluations.
eval_i = 0
#Train shit
for s in range(max_steps):
for n in range(n_iter):
#fetch next batch of data
X, y = data['train'].next_batch(batch_size)
#use torch tensor + gpu
X = torch.from_numpy(X).type(dtype).to(device)
y = torch.from_numpy(y).type(dtype).to(device)
#reset gradient to zero before gradient descent.
optmizer.zero_grad()
#calculate loss
probs = cnn_model(X) #automatically calls .forward()
loss = loss_XE(probs, y.argmax(dim=1))
#backward propagation
loss.backward()
optmizer.step()
#stores the loss and accuracy of the trainind data for later analysis.
loss_train[eval_i] += loss.item() / num_evals #
acc_train[eval_i] += accuracy(probs, y) / num_evals
probs.detach()
if (s % eval_freq == 0) | (s == (max_steps - 1)):
#calculate accuracy for the whole data set
for t in range(num_evals):
#fetch all the data
X, y = data['test'].next_batch(batch_size)
#use torch tensor + gpu, no gradient needed.
X = torch.tensor(X, requires_grad=False).type(dtype).to(device)
y = torch.tensor(y, requires_grad=False).type(dtype).to(device)
#actually calculates loss and accuracy for the batch
probs = cnn_model.forward(X)
loss_eval[eval_i] += loss_XE(
probs,
y.argmax(dim=1)).item() # detach().data.cpu().item()
acc_eval[eval_i] += accuracy(probs, y)
probs.detach()
#frees memory
X.detach()
y.detach()
#average the losses and accuracies across test batches
loss_eval[eval_i] /= num_evals
acc_eval[eval_i] /= num_evals
#print performance
print(f"step {s} out of {max_steps}")
print(
f" loss: {loss_eval[eval_i]}, accuracy: {acc_eval[eval_i]}")
print(
f" loss: {loss_train[eval_i]}, accuracy: {acc_train[eval_i]}"
)
#save the results
# np.save("loss_eval", loss_eval)
# np.save("accuracy_eval", acc_eval)
#increments eval counter
eval_i += 1
#Save intermediary results for later analysis
print("saving results in folder...")
np.save("loss_train", loss_train)
np.save("accuracy_train", acc_train)
np.save("loss_eval", loss_eval)
np.save("accuracy_eval", acc_eval)
print("savign model")
torch.save(cnn_model.state_dict(), cnn_model.__class__.__name__ + ".pt")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# Get everything ready
data = cifar10_utils.get_cifar10()
n_classes = data['train'].labels.shape[1] # 10
n_channels = data['train'].images.shape[1] # 3
cnn = ConvNet(n_channels, n_classes)
loss_module = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr = FLAGS.learning_rate)
test_accuracies = []
train_losses = []
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
cnn.to(device)
# Iterate over the batches
for iteration in range(0, FLAGS.max_steps):
optimizer.zero_grad()
# Evaluate on whole test set
if (iteration%FLAGS.eval_freq == 0):
print("Iteration {}...".format(iteration))
epochs = data['test'].epochs_completed
batch_accuracies = []
while (epochs-data['test'].epochs_completed) == 0:
test_batch, test_batch_labels = data['test'].next_batch(FLAGS.batch_size)
test_probabilities = cnn.forward(torch.from_numpy(test_batch).to(device))
acc = accuracy(test_probabilities, torch.from_numpy(test_batch_labels).to(device))
batch_accuracies.append(acc.item())
test_accuracy = np.mean(batch_accuracies)
print("Test accuracy:", test_accuracy)
test_accuracies.append(test_accuracy)
# Train on batch
train_batch, train_batch_labels = data['train'].next_batch(FLAGS.batch_size)
train_probabilities = cnn.forward(torch.from_numpy(train_batch).to(device))
loss = loss_module(train_probabilities, torch.argmax(torch.from_numpy(train_batch_labels), dim=1).long().to(device))
if (iteration%FLAGS.eval_freq == 0):
train_losses.append(loss.item())
loss.backward()
optimizer.step()
# Plot results
x = range(0, len(test_accuracies)*FLAGS.eval_freq, FLAGS.eval_freq)
fig, ax = plt.subplots()
ax.plot(x, train_losses)
ax.set(xlabel='batches', ylabel='loss',
title='Loss training set after batches trained')
ax.grid()
fig.savefig("figures/cnn_loss_{0}_{1}_{2}.png".format(FLAGS.learning_rate, FLAGS.max_steps, FLAGS.batch_size))
# plt.show()
x = range(0, len(test_accuracies)*FLAGS.eval_freq, FLAGS.eval_freq)
fig, ax = plt.subplots()
ax.plot(x, test_accuracies)
ax.set(xlabel='batches', ylabel='accuracy',
title='Accuracy test set after batches trained')
ax.grid()
fig.savefig("figures/cnn_results_{0}_{1}_{2}.png".format(FLAGS.learning_rate, FLAGS.max_steps, FLAGS.batch_size))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# get device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# loop through data
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py',
validation_size=2000)
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
# get channels
n_channels = np.size(x, 1)
# create model
net = ConvNet(n_channels, 10)
net.to(device)
# get loss function and optimizer
crossEntropy = nn.CrossEntropyLoss()
# keep track of loss and accuracy
loss_list = []
loss_val_list = []
accuracy_train_list = []
accuracy_val_list = []
# set optimizer
optimizer = torch.optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
# loop for the amount of steps
for i in range(FLAGS.max_steps):
# create torch compatible input
x = Variable(torch.from_numpy(x), requires_grad=True)
x = x.to(device)
# perform forward pass
out = net(x)
# convert one hot to indices and create torch compatible input
label_index = np.argmax(y, axis=1)
label_index = torch.LongTensor(label_index)
label_index = label_index.to(device)
# apply cross entropy
loss = crossEntropy(out, label_index)
# show progress and run network on validation set
if i % FLAGS.eval_freq == 0:
# convert output to numpy array, differs when using cuda compared to cpu
if torch.cuda.is_available():
train_out = out.cpu()
out_numpy = train_out.data[:].numpy()
else:
out_numpy = out.data[:].numpy()
# calculate accuracy
accuracy_train = accuracy(out_numpy, y)
# don't track the gradients
with torch.no_grad():
# load validation data
x_val, y_val = cifar10['validation'].next_batch(
FLAGS.batch_size)
# create torch compatible input
x_val = Variable(torch.from_numpy(x_val), requires_grad=False)
x_val = x_val.to(device)
# run on validation set
val_out = net.forward(x_val)
# convert one hot to indices and create torch compatible input
y_val_index = np.argmax(y_val, axis=1)
y_val_index = torch.LongTensor(y_val_index)
y_val_index = y_val_index.to(device)
# apply cross entropy
loss_val = crossEntropy.forward(val_out, y_val_index)
# convert output to numpy array, differs when using cuda compared to cpu
if torch.cuda.is_available():
val_out = val_out.cpu()
val_out_numpy = val_out.data[:].numpy()
loss_val = loss_val.cpu()
loss_val = loss_val.data.numpy()
loss_train = loss.cpu()
loss_train = loss_train.data.numpy()
else:
val_out_numpy = val_out.data[:].numpy()
loss_val = loss_val.data.numpy()
loss_train = loss.data.numpy()
accuracy_val = accuracy(val_out_numpy, y_val)
# save variables
accuracy_train_list.append(accuracy_train)
accuracy_val_list.append(accuracy_val)
loss_list.append(loss_train)
loss_val_list.append(loss_val)
# print progress
print(
"##############################################################"
)
print("Epoch ", i)
print(
"---------------------------------------------------------------"
)
print("The ACCURACY on the TRAIN set is currently: ",
accuracy_train)
print(
"---------------------------------------------------------------"
)
print("The ACCURACY on the VALIDATION set is currently:",
accuracy_val)
print(
"---------------------------------------------------------------"
)
print("The LOSS on the TRAIN set is currently:", loss_train)
print(
"---------------------------------------------------------------"
)
print("The LOSS on the VALIDATION set is currently:", loss_val)
print(
"---------------------------------------------------------------"
)
print(
"###############################################################"
)
print("\n")
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
# insert new databatch for next loop
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
# run test data through network without tracking the gradients
with torch.no_grad():
# test
x, y = cifar10['test'].images, cifar10['test'].labels
# convert variable to torch compatible input
x = Variable(torch.from_numpy(x), requires_grad=False)
x = x.to(device)
# get output
out = net(x)
# convert output to numpy array, differs when using cuda compared to cpu
if torch.cuda.is_available():
out = out.cpu()
out_numpy = out.data[:].numpy()
else:
out_numpy = out.data[:].numpy()
# calculate accuracy
test_accuracy = accuracy(out_numpy, y)
print("The accuracy on the test set is:")
print(test_accuracy)
# save test, training and validation accuracies and losses to make a plot afterwards
lists = [
accuracy_train_list, accuracy_val_list, loss_list, loss_val_list,
test_accuracy
]
pickle.dump(lists, open("lists.p", "wb"))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# print(FLAGS.batch_size)
# print(FLAGS.eval_freq)
# print(FLAGS.learning_rate)
# print(FLAGS.max_steps)
cifar10 = cifar10_utils.get_cifar10()
if torch.cuda.is_available():
# print(torch.device('cpu'), torch.device("cuda"))
device = torch.device("cuda")
else:
device = torch.device("cpu")
network = ConvNet(3, 10)
network.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(network.parameters(), lr=FLAGS.learning_rate)
plotting_accuracy = []
plotting_loss = []
plotting_accuracy_test = []
plotting_loss_test = []
for i in range(1, FLAGS.max_steps - 1):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
x = x.to(device)
y = y.to(device)
out = network.forward(x)
loss = criterion(out, y.argmax(dim=1))
# print("Batch: {} Loss {}".format(i, loss))
acc = accuracy(out, y)
# print("Accuracy: {}".format(acc))
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (i % FLAGS.eval_freq == 0):
x, y = cifar10['test'].next_batch(300)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
x = x.to(device)
y = y.to(device)
out = network.forward(x)
loss_test = criterion(out, y.argmax(dim=1))
print("TEST Batch: {} Loss {}".format(i, loss_test))
acc_test = accuracy(out, y)
print("TEST Accuracy: {}".format(acc_test))
plotting_accuracy_test.append(acc_test)
plotting_loss_test.append(loss_test.item())
plotting_accuracy.append(acc)
plotting_loss.append(loss.item())
plt.plot(plotting_accuracy, label='train accuracy')
plt.plot(plotting_accuracy_test, label='test accuracy')
# plt.plot(plotting_loss, label='train loss')
# plt.plot(plotting_loss_test, label='test loss')
plt.legend()
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
import matplotlib.pyplot as plt
if not torch.cuda.is_available():
print("WARNING: CUDA DEVICE IS NOT AVAILABLE, WILL TRAIN ON CPU")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
train = data['train']
test = data['test']
images_test_np = test.images
labels_test_np = test.labels
vgg = ConvNet(3, 10)
vgg.to(device)
if MEM_DEBUG:
print("Memory after VGG loaded: Alloc: {} MB, Cached: {} MB".format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(vgg.parameters())
loss_train = np.zeros((int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
loss_test = np.zeros((int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
accuracy_test = np.zeros(
(int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
for i in range(0, FLAGS.max_steps):
print('iter', i + 1, end='\r')
images_np, labels_np = train.next_batch(FLAGS.batch_size)
images = torch.from_numpy(images_np).to(device)
labels = torch.from_numpy(np.argmax(labels_np, axis=1)).to(device)
if MEM_DEBUG:
print("\nMemory after train loaded: Alloc: {} MB, Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
optimizer.zero_grad()
pred = vgg(images)
loss = criterion(pred, labels.long())
loss.backward()
optimizer.step()
del images
del labels
if (i + 1) % FLAGS.eval_freq == 0:
if MEM_DEBUG:
print("Memory entering the eval: Alloc: {} MB, Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
loss_train[i // FLAGS.eval_freq] = loss.item()
images_test = torch.from_numpy(images_test_np).to(device)
labels_test = torch.from_numpy(np.argmax(labels_test_np,
axis=1)).to(device)
if MEM_DEBUG:
print("Memory after test loaded: Alloc: {} MB Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
vgg.eval()
with torch.no_grad():
pred_test = vgg(images_test)
vgg.train()
accuracy_test[i // FLAGS.eval_freq] = accuracy(
pred_test, F.one_hot(labels_test)).item()
loss_test[i // FLAGS.eval_freq] = criterion(
pred_test, labels_test.long()).item()
if PRINTS:
print()
print('test_loss:', loss_test[i // FLAGS.eval_freq])
print('test_accuracy:', accuracy_test[i // FLAGS.eval_freq])
print('train_loss:', loss_train[i // FLAGS.eval_freq])
if PLOTS:
fig, ax = plt.subplots(1, 2, figsize=(10, 5))
fig.suptitle('Training curves for Pytorch Convnet')
ax[0].set_title('Loss')
ax[0].set_ylabel('Loss value')
ax[0].set_xlabel('No of batches seen x{}'.format(FLAGS.eval_freq))
ax[0].plot(loss_train, label='Train')
ax[0].plot(loss_test, label='Test')
ax[0].legend()
ax[1].set_title('Accuracy')
ax[1].set_ylabel('Accuracy value')
ax[1].set_xlabel('No of batches seen x{}'.format(FLAGS.eval_freq))
ax[1].plot(accuracy_test, label='Test')
ax[1].legend()
import time
fig_name = 'pt_conv_training_{}_{}_{}_{}.jpg'.format(
FLAGS.max_steps, FLAGS.batch_size, FLAGS.eval_freq, time.time())
plt.savefig(fig_name)
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# set up the data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir, one_hot=False)
test_set = cifar10["test"]
test_batch_size = 2000
n_test_batches = int(test_set.num_examples / test_batch_size)
# set up the model and optimizer
conv_net = ConvNet(n_channels=3, n_classes=10)
loss_module = nn.CrossEntropyLoss()
conv_net.to(device)
optimizer = torch.optim.Adam(conv_net.parameters(), lr=FLAGS.learning_rate)
accuracies = []
losses = []
conv_net.train()
for i in range(FLAGS.max_steps):
# load data
images, labels = cifar10['train'].next_batch(FLAGS.batch_size)
images, labels = torch.from_numpy(images).to(device), torch.from_numpy(
labels).to(device)
# forward pass
model_pred = conv_net(images)
# calculate the loss
loss = loss_module(model_pred, labels)
# backward pass
optimizer.zero_grad()
loss.backward()
# update the parameters
optimizer.step()
# evaluate the model on the data set every eval_freq steps
conv_net.eval()
if i % FLAGS.eval_freq == 0:
test_accuracy = test_conv(conv_net, test_set, n_test_batches,
test_batch_size)
accuracies.append(test_accuracy)
losses.append(loss)
conv_net.train()
plot_curve(accuracies, 'Accuracy')
plot_curve(losses, 'Loss')
def train():
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda")
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
dataset = cifar10_utils.get_cifar10()
training = dataset['train']
test = dataset['test']
model = ConvNet(3, 10)
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
ce = torch.nn.CrossEntropyLoss()
test_accuracy = []
loss_list = []
for epoch in np.arange(0, FLAGS.max_steps):
x, y = training.next_batch(FLAGS.batch_size)
x = Variable(torch.tensor(x).to(device))
y = Variable(torch.tensor(y).to(device))
optimizer.zero_grad()
model.train()
yh = model.forward(x)
loss = ce(yh, torch.max(y, 1)[1])
loss_list.append(loss.item())
loss.backward()
optimizer.step()
if (epoch + 1) % int(FLAGS.eval_freq) == 0:
acc = []
with torch.no_grad():
for _ in np.arange(0, (test.num_examples // FLAGS.batch_size)):
x, y = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
y = torch.tensor(y).to(device)
model.eval()
yh = model.forward(x)
acc.append(accuracy(yh, y))
test_accuracy.append(np.mean(acc))
print(np.mean(acc))
import seaborn as sns
import matplotlib.pyplot as plt
f, axes = plt.subplots(1, 2)
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, EVAL_FREQ_DEFAULT),
test_accuracy,
ax=axes[0])
ax.set_title('Test accuracy')
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, 1),
loss_list,
ax=axes[1])
ax.set_title('Loss')
figure = ax.get_figure()
figure.savefig("cnn-pytorch-results")
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(FLAGS.max_steps): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
# out = model.forward(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])
# plt.show()
plt.savefig('/home/lgpu0376/code/output_dir/loss_acc_graph.png')
# plt.savefig('loss_acc_graph.png')
with open('/home/lgpu0376/code/output_dir/output.out', 'w+') as f:
f.write(f'Final loss: {loss_history[-1]}')
f.write(f'\nFinal acc: {acc_history[-1]}')
