def train():
"""
Performs training and evaluation of ConvNet model.
"""
# Set the random seeds for reproducibility
np.random.seed(42)
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py')
myConvNet = ConvNet(3, 10)
loss_criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(myConvNet.parameters())
accuracies = {'train': [], 'test': []}
loss_curve = {'train': [], 'test': []}
for j in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).contiguous()
y = torch.from_numpy(y).contiguous()
optimizer.zero_grad()
outputs = myConvNet(x)
loss = loss_criterion(outputs, torch.argmax(y, 1))
loss.backward()
optimizer.step()
if j % FLAGS.eval_freq == 0:
accuracies['train'].append(accuracy(outputs.detach().numpy(), y.detach().numpy()))
loss_curve['train'].append(loss.detach().numpy())
x, y = cifar10['test'].images, cifar10['test'].labels
x = torch.from_numpy(x)
y = torch.from_numpy(y)
x = x[:1000]
y = y[:1000]
outputs = myConvNet(x)
loss = loss_criterion(outputs, torch.argmax(y, 1))
loss_curve['test'].append(loss.detach().numpy())
print(j)
print(accuracy(outputs.detach().numpy(), y.detach().numpy()))
accuracies['test'].append(accuracy(outputs.detach().numpy(), y.detach().numpy()))
accuracies['train'].append(accuracy(outputs.detach().numpy(), y.detach().numpy()))
loss_curve['train'].append(loss.detach().numpy())
x, y = cifar10['test'].images, cifar10['test'].labels
x = torch.from_numpy(x)
y = torch.from_numpy(y)
x= x[:1000]
y = y[:1000]
outputs = myConvNet(x)
loss = loss_criterion(outputs, torch.argmax(y, 1))
loss_curve['test'].append(loss.detach().numpy())
print(accuracy(outputs.detach().numpy(), y.detach().numpy()))
accuracies['test'].append(accuracy(outputs.detach().numpy(), y.detach().numpy()))
plot_results(accuracies, loss_curve)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
cifar10 = cifar10_utils.get_cifar10()
n_channels = 3
n_classes = 10
nr_iterations = FLAGS.max_steps+1
convnet = ConvNet(n_channels, n_classes).to(device)
optimizer = optim.Adam(convnet.parameters(), lr=FLAGS.learning_rate)
loss = nn.CrossEntropyLoss()
accuracies_test = []
accuracies_train = []
losses = []
for i in range(nr_iterations):
x, y = cifar10['train'].next_batch(FLAGS.batch_size) # (batch_size, 3, 32, 32) (batch_size, 10)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(y).type(torch.LongTensor).to(device)
_, y_target = y.max(1)
optimizer.zero_grad()
prediction = convnet(x)
cross_entropy_loss = loss(prediction, y_target)
losses.append(cross_entropy_loss.item())
cross_entropy_loss.backward()
optimizer.step()
del x, y_target
if i % FLAGS.eval_freq == 0:
x_test, y_test = cifar10['test'].images, cifar10['test'].labels
x_test = torch.from_numpy(x_test).to(device)
y_test = torch.from_numpy(y_test).type(torch.LongTensor).to(device)
pred_test = convnet(x_test)
acc_test = accuracy(pred_test, y_test)
acc_train = accuracy(prediction, y)
accuracies_test.append(acc_test )
accuracies_train.append(acc_train)
print('accuracy at step', i, ': ', acc_test)
del x_test, y_test, pred_test, prediction
def train():
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
dataset = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
a, b, c = dataset['train'].images.shape[1:]
n_classes = dataset['train'].labels.shape[1]
cnn = ConvNet(3, n_classes).to(device)
optimizer = optim.Adam(cnn.parameters(), lr=FLAGS.learning_rate)
crossentropy = nn.CrossEntropyLoss()
n_test = dataset['test'].images.shape[0]
for step in range(FLAGS.max_steps):
input, labels = dataset['train'].next_batch(FLAGS.batch_size)
labels = np.argmax(labels, axis=1)
input, labels = torch.from_numpy(input).to(device), torch.from_numpy(
labels).long().to(device)
predictions = cnn.forward(input)
loss = crossentropy(predictions, labels)
# clean up old gradients
cnn.zero_grad()
loss.backward()
optimizer.step()
if (step == FLAGS.max_steps - 1 or step % FLAGS.eval_freq == 0):
test_loss = []
test_accuracy = []
for i in range(0, n_test, FLAGS.batch_size):
test_input, test_labels = dataset['test'].next_batch(
FLAGS.batch_size)
test_input = torch.from_numpy(test_input).to(device)
test_labels = torch.from_numpy(np.argmax(
test_labels, axis=1)).long().to(device)
test_prediction = cnn.forward(test_input)
test_loss.append(
crossentropy(test_prediction, test_labels).item())
test_accuracy.append(accuracy(test_prediction, test_labels))
sys.stdout = open(
str(FLAGS.learning_rate) + '_' + str(FLAGS.max_steps) + '_' +
str(FLAGS.batch_size) + '_' + str(FLAGS.batch_size) +
'conv.txt', 'a')
print("{},{:f},{:f}".format(step, np.mean(test_loss),
np.mean(test_accuracy)))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
n_inputs = 3 * 32 * 32
n_classes = 10
batches_per_epoch = (int)(data['test'].images.shape[0] /
FLAGS.batch_size) # need this for test set
model = ConvNet(n_inputs, n_classes).to(device)
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
max_accuracy = 0.0
start_time = time.perf_counter()
for step in range(1, FLAGS.max_steps + 1):
x, y = get_batch(data, 'train', FLAGS.batch_size, device)
predictions = model.forward(x)
training_loss = loss_fn(predictions, y.argmax(dim=1))
optimizer.zero_grad()
training_loss.backward()
optimizer.step()
if step == 1 or step % FLAGS.eval_freq == 0:
with torch.no_grad():
test_loss = 0
test_acc = 0
for test_batch in range(batches_per_epoch):
x, y = get_batch(data, 'test', FLAGS.batch_size, device)
predictions = model(x)
test_loss += loss_fn(predictions,
y.argmax(dim=1)) / batches_per_epoch
test_acc += accuracy(predictions, y) / batches_per_epoch
if test_acc > max_accuracy:
max_accuracy = test_acc
print(
"step %d/%d: training loss: %.3f test loss: %.3f accuracy: %.1f%%"
% (step, FLAGS.max_steps, training_loss, test_loss,
test_acc * 100))
time_taken = time.perf_counter() - start_time
print("Done. Scored %.1f%% in %.1f seconds." %
(max_accuracy * 100, time_taken))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
ce_loss = nn.CrossEntropyLoss()
convnet = ConvNet(3, 10)
optimizer = optim.Adam(convnet.parameters(), lr=FLAGS.learning_rate)
c10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
test_data = c10['test'].images[:32]
test_data = torch.tensor(test_data)
targets = c10['test'].labels[:32]
acc_values = []
loss_values = []
for i in range(FLAGS.max_steps): #range(FLAGS.max_steps)
x, y = c10['train'].next_batch(FLAGS.batch_size)
y = y.argmax(axis=1)
x = torch.tensor(x)
y = torch.tensor(y)
optimizer.zero_grad()
out = convnet(x)
loss = ce_loss(out, y)
loss.backward()
optimizer.step()
loss_values.append(loss.item())
# evaluate
if i % FLAGS.eval_freq == 0:
with torch.no_grad():
predictions = convnet(test_data).detach().numpy()
acc = accuracy(predictions, targets)
print('acc', acc, 'loss', loss.item())
acc_values.append(acc)
# save loss and accuracy to file
with open('accuracy_cnn.txt', 'a') as f_acc:
print(acc_values, file=f_acc)
with open('loss_cnn.txt', 'a') as f_loss:
print(loss_values, file=f_loss)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
print(device)
# Get Images
cifar10 = cifar10_utils.read_data_sets(FLAGS.data_dir)
# Create MLP Instance
trainDataSet = cifar10['train']
testDataSet = cifar10['test']
mlp = ConvNet(cifar10['train'].images[0].shape[2],
np.shape(cifar10['test'].labels)[1]).to(device)
loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(mlp.parameters(), lr=FLAGS.learning_rate)
aggregate_counter = 0
for i in range(FLAGS.max_steps):
# np.random.shuffle(cifar10['train'])
accuracies_train = []
loss_train = []
counter = 0
batch = trainDataSet.next_batch(FLAGS.batch_size)
x = torch.from_numpy(batch[0]).to(device)
# x = torch.from_numpy(x.reshape(x.shape[0], x.shape[2], x.shape[3], x.shape[1])).to(device)
y_numpy = batch[1]
y = torch.from_numpy(batch[1]).to(device)
optimizer.zero_grad()
prob = mlp(x)
prob_num = prob.cpu().clone().detach().numpy()
predictions = (prob_num == prob_num.max(axis=1)[:, None]).astype(int)
current_accuracy = accuracy(predictions, y_numpy)
print(current_accuracy)
accuracies_train.append(current_accuracy)
current_loss = loss(prob, torch.max(y, 1)[1])
current_loss.backward()
optimizer.step()
current_loss = current_loss.cpu().detach().numpy()
loss_train.append(current_loss)
writer.add_scalar('Train/Loss', current_loss, i)
writer.add_scalar('Train/Accuracy', current_accuracy, i)
if i % FLAGS.eval_freq == 0:
test_dataset(mlp, testDataSet, loss, i)
aggregate_counter += counter
print("ITERATION FINISHED", i, " ", np.mean(accuracies_train))
test_dataset(mlp, testDataSet, loss, FLAGS.max_steps + 1)
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py')
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
n_channels = np.size(x, 1)
net = ConvNet(n_channels, 10).to(device)
crossEntropy = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
loss_list = []
accuracy_list = []
test_eval_list = []
for i in range(FLAGS.max_steps):
x = Variable(torch.from_numpy(x), requires_grad = True).to(device)
predictions = net(x).to(device)
numpy_predictions = predictions.cpu().data[:].numpy()
label_index = torch.LongTensor(np.argmax(y, axis = 1)).to(device)
loss = crossEntropy(predictions, label_index)
if i % FLAGS.eval_freq == 0:
current_accuracy = accuracy(numpy_predictions, y)
current_test_accuracy = test(net)
current_loss = loss.cpu().data.numpy()
loss_list.append(current_loss)
accuracy_list.append(current_accuracy)
test_eval_list.append(current_test_accuracy)
print ('Training epoch %d out of %d. Loss %.3f, Train accuracy %.3f, Test accuracy %.3f' % (i, FLAGS.max_steps, current_loss, current_accuracy, current_test_accuracy))
optimizer.zero_grad()
loss.backward()
optimizer.step()
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
# save model
torch.save(net, MODEL_DIRECTORY + CNN_PYTORCH_FILE)
test_accuracy = test(net)
print('Test accuracy %.3f' % (test_accuracy))
def fit(**hyperparameter):
model = ConvNet(3, 10).to(device)
loss_module = nn.CrossEntropyLoss()
optimizer = hyperparameter['optimizer'](
model.parameters(), lr=hyperparameter['learning_rate'])
accuracies = dict(train_scores=list(), val_scores=list())
losses = dict(train_scores=list(), val_scores=list())
for i in range(FLAGS.max_steps):
x, y = train_data.next_batch(FLAGS.batch_size)
x, y = torch.from_numpy(x).float().to(device), torch.from_numpy(
y).long().to(device)
preds = model(x)
preds = preds.squeeze(dim=1)
if i % FLAGS.eval_freq == FLAGS.eval_freq - 1:
if i % FLAGS.eval_freq == FLAGS.eval_freq - 1:
x_train, y_train = train_data.next_batch(2000)
x_test, y_test = test_data.next_batch(2000)
evaluate(model, x_train, y_train, x_test, y_test,
loss_module, accuracies, losses)
_, y = torch.max(y, dim=1)
loss = loss_module(preds, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
plot_history(accuracies, "Accuracies", model_name="ConvNet")
plot_history(losses,
"Losses",
title="Train and Validation Losses of ",
model_name="ConvNet")
# Test
with torch.no_grad():
x, y = test_data.next_batch(2000)
x, y = torch.from_numpy(x).float().to(device), torch.from_numpy(
y).long().to(device)
preds = model(x)
preds = preds.squeeze(dim=1)
print("Test Accuracy: ", accuracy(preds, y))
return accuracy(preds, y)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if FLAGS.batch_size:
batch_size = int(FLAGS.batch_size)
cifar10 = cifar10_utils.get_cifar10();
convNet = ConvNet(3, 10);
print(convNet);
lossfunc = nn.CrossEntropyLoss();
optimizer = torch.optim.Adam(convNet.parameters(),lr=LEARNING_RATE_DEFAULT)
cifar10_train = cifar10['train'];
# get all test image labels and features:
cifar10_test = cifar10['test'];
while (cifar10_train.epochs_completed < 10):
x, y = cifar10_train.next_batch(batch_size);
x_test, y_target = cifar10_test.next_batch(batch_size);
x = torch.autograd.Variable(torch.from_numpy(x));
y = torch.autograd.Variable(torch.from_numpy(y).long());
x_test = torch.autograd.Variable(torch.from_numpy(x_test));
#x_test = x_test.reshape((batch_size, -1));
optimizer.zero_grad();
out = convNet(x);
loss = lossfunc(out,torch.max(y, 1)[1]);
loss.backward();
optimizer.step();
y_test = convNet(x_test);
rate = accuracy(y_test, y_target)
#print('loss: ', crossentropy_loss);
print("Accuracy:", rate)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
if torch.cuda.is_available():
dev = "cuda:0"
else:
dev = "cpu"
device = torch.device(dev)
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
"""
Initialize data module
"""
cifar10=cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
x, y = cifar10['train'].next_batch(1)
x_test, y_test = cifar10['test'].next_batch(10000)
x_test = torch.tensor(x_test)
y_test = torch.tensor(y_test)
"""
initialize the network
"""
network = ConvNet(x.shape[1], y.shape[1]).to(device)
crossEntropy = nn.CrossEntropyLoss()
optimizer = None
optimizer = torch.optim.Adam(network.parameters(), lr=FLAGS.learning_rate, amsgrad=True)
store_loss = None
for i in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
y = torch.LongTensor(y).to(device)
prediction = network.forward(x)
loss = crossEntropy.forward(prediction, torch.max(y, 1)[1])
optimizer.zero_grad()
loss.backward()
optimizer.step()
store_loss = loss.cpu()
del loss
del x
del y
del prediction
prediction_collection = None
if i%FLAGS.eval_freq == 0:
with torch.no_grad():
print('Loss after '+str(i)+' steps '+str(store_loss))
for j in range(100):
test_data = x_test[j*100:j*100+100].to(device)
prediction = network.forward(test_data)
prediction = nn.functional.softmax(prediction)
del test_data
if j == 0:
prediction_collection = prediction
else:
prediction_collection = torch.cat((prediction_collection, prediction), 0)
del prediction
print('Accuracy after '+ str(i) +' steps ' + str(accuracy(prediction_collection, y_test)))
prediction_collection = None
with torch.no_grad():
print('final Loss',store_loss)
for j in range(100):
test_data = x_test[j*100:j*100+100].to(device)
prediction = network.forward(test_data).cpu()
prediction = nn.functional.softmax(prediction)
if j == 0:
prediction_collection = prediction
else:
prediction_collection = torch.cat((prediction_collection, prediction), 0)
del prediction
print('Final accuracy')
print(accuracy(prediction_collection, y_test))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# will be used to compute accuracy and loss for the whole train and test sets
batch_size_acc = 500
data_accuracy_loss = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
X_train_acc, y_train_acc = data_accuracy_loss['train'].images, data_accuracy_loss['train'].labels
X_test_acc, y_test_acc = data_accuracy_loss['test'].images, data_accuracy_loss['test'].labels
steps_train = int(X_train_acc.shape[0] / batch_size_acc)
steps_test = int(X_test_acc.shape[0] / batch_size_acc)
#
data = cifar10_utils.get_cifar10(data_dir = FLAGS.data_dir)
n_classes = data['train'].labels.shape[1]
n_inputs = data['train'].images.shape[1]*data['train'].images.shape[2]*data['train'].images.shape[3]
batch_size = FLAGS.batch_size
m_steps = FLAGS.max_steps
alpha = FLAGS.learning_rate
cnn = ConvNet(3, 10)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr=alpha)
x_ax = []
acc_train = []
acc_test = []
loss_train = []
loss_test = []
for step in range(m_steps):
x, y = data['train'].next_batch(batch_size)
n = x.shape
x = torch.from_numpy(x)
y_pred = cnn(x)
labels = torch.LongTensor(y)
loss = criterion(y_pred, torch.max(labels, 1)[1])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % FLAGS.eval_freq == 0:
print('Iteration ',step)
x_ax.append(step)
acc_ = []
loss_ = []
for i in range(steps_train):
x_acc = X_train_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
y_acc = y_train_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
x_acc = torch.from_numpy(x_acc)
y_acc = torch.LongTensor(y_acc)
y_pred = cnn.forward(x_acc)
acc_.append(accuracy(y_pred, y_acc))
loss_.append(float(criterion(y_pred, torch.max(y_acc, 1)[1])))
acc_train.append(np.mean(acc_))
loss_train.append(np.mean(loss_))
acc_ = []
loss_ = []
for i in range(steps_test):
x_acc = X_test_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
y_acc = y_test_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
x_acc = torch.from_numpy(x_acc)
y_acc = torch.LongTensor(y_acc)
y_pred = cnn.forward(x_acc)
acc_.append(accuracy(y_pred, y_acc))
loss_.append(float(criterion(y_pred, torch.max(y_acc, 1)[1])))
acc_test.append(np.mean(acc_))
loss_test.append(np.mean(loss_))
print('Max train accuracy ', max(acc_train))
print('Max test accuracy ', max(acc_test))
print('Min train loss ', min(loss_train))
print('Min test loss ', min(loss_test))
x_ax = np.array(x_ax)
acc_test = np.array(acc_test)
acc_train = np.array(acc_train)
loss_test = np.array(loss_test)
loss_train = np.array(loss_train)
print('Max train accuracy ', max(acc_train))
print('Max test accuracy ', max(acc_test))
print('Min train loss ', min(loss_train))
print('Min test loss ', min(loss_test))
fig = plt.figure()
ax = plt.axes()
plt.title("CNN Pytorch. Accuracy curves")
ax.plot(x_ax, acc_train, label='train');
ax.plot(x_ax, acc_test, label='test');
ax.set_xlabel('Step');
ax.set_ylabel('Accuracy');
plt.legend();
plt.savefig('accuracy_cnn.jpg')
fig = plt.figure()
ax = plt.axes()
plt.title("CNN Pytorch. Loss curves")
ax.plot(x_ax, loss_train, label='train');
ax.plot(x_ax, loss_test, label='train');
ax.set_xlabel('Step');
ax.set_ylabel('Loss');
ax.set_ylim(top=10, bottom=0)
plt.legend();
plt.savefig('loss_cnn.jpg')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
epoch_num = 3
########################
# PUT YOUR CODE HERE #
#######################
model = ConvNet(3, 10)
#Obtain Dataset
train_dataset = cifar10_utils.get_cifar10()['train']
val_dataset = cifar10_utils.get_cifar10()['validation']
test_dataset = cifar10_utils.get_cifar10()['test']
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=BATCH_SIZE_DEFAULT,
drop_last=True)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
for epoch in range(epoch_num):
model.train()
losses = []
accs = []
for i_iter in range(
int(train_dataset.num_examples / BATCH_SIZE_DEFAULT)):
images, labels = train_dataset.next_batch(BATCH_SIZE_DEFAULT)
images, labels = torch.tensor(images), torch.tensor(
labels, dtype=torch.long)
pred = model(images)
labels = torch.argmax(labels, dim=1)
loss = criterion(pred, labels)
pred_result = torch.argmax(pred, dim=1)
acc = accuracy(pred_result, labels)
accs.append(acc)
model.zero_grad()
loss.backward()
optimizer.step()
losses.append(loss)
if i_iter % 100 == 0:
msg = 'Epoch:[{}/{}] Iter: [{}/{}], Loss: {: .6f}, ACC:[{: .6f}]'.format(
epoch, epoch_num, i_iter,
int(train_dataset.num_examples / BATCH_SIZE_DEFAULT),
sum(losses) / len(losses),
sum(accs) / len(accs))
print(msg)
with open('./log.txt', 'a') as f:
f.write(msg)
f.write('\n')
msg_epoch = '--------Epoch: [{}/{}], Loss: {: .6f}, ACC:[{: .6f}]-------'.format(
epoch, epoch_num,
sum(losses) / len(losses),
sum(accs) / len(accs))
print(msg_epoch)
with open('./log.txt', 'a') as f:
f.write(msg)
f.write('\n')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
# Device configuration
use_cuda = torch.cuda.is_available()
if use_cuda:
print('Running in GPU model')
device = torch.device('cuda' if use_cuda else 'cpu')
dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
# load dataset
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
# get batches
batches = []
# initializing loss and accuracy arrays
accuracies = []
losses = []
for i in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size) # (batch_size, 3, 32, 32) (batch_size, 10)
batches.append((x,y))
# get output size
out_size = batches[-1][1].shape[1]
# get intput size
in_size = batches[-1][0].shape[1]
# initialize network
net = ConvNet(in_size, out_size).to(device)
net = nn.DataParallel(net)
# intialize loss function
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
# make steps
for s in range(FLAGS.max_steps):
x,t = batches[s]
# Forward pass
y = net(torch.from_numpy(x).type(dtype))
t = torch.from_numpy(t).type(dtype)
t = torch.max(t,1)[1]
loss = criterion(y, t)
losses.append(loss.cpu().detach().numpy())
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
if s % FLAGS.eval_freq == 0:
x, t = cifar10['test'].images, cifar10['test'].labels
y = net(torch.from_numpy(x).type(dtype))
acc = accuracy(y.cpu().detach().numpy(),t)
accuracies.append(acc)
print('accuracy at step',s,': ',acc)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
with open("jobs/status.txt", "w") as f:
f.write("Start training\n")
## Prepare all functions
output_dir = FLAGS.output_dir
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
learning_rate = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
optimizer = FLAGS.optimizer
# Obtain dataset
dataset = cifar10_utils.get_cifar10(data_dir)
n_channels = dataset['train'].images[0].shape[0]
n_classes = dataset['train'].labels[0].shape[0]
n_test = dataset['test'].images.shape[0]
# Initialise VGG network
dev = 'cuda' if torch.cuda.is_available() else 'cpu'
device = torch.device(dev)
print("Device: " + dev)
net = ConvNet(n_channels, n_classes).to(device)
loss_fn = F.cross_entropy
print("Network architecture:\n\t{}\nLoss module:\n\t{}".format(
str(net), str(loss_fn)))
# Evaluation vars
train_loss = []
gradient_norms = []
train_acc = []
test_acc = []
iteration = 0
# Training
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
while iteration < max_steps:
iteration += 1
# Sample a mini-batch
x, y = dataset['train'].next_batch(batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(y).argmax(dim=1).long().to(device)
# Forward propagation
prediction = net.forward(x)
loss = loss_fn(prediction, y)
acc = accuracy(prediction, y)
train_acc.append((iteration, acc))
train_loss.append((iteration, loss))
# Backprop
optimizer.zero_grad()
loss.backward()
# Weight update in linear modules
optimizer.step()
norm = 0
for params in net.parameters():
norm += params.reshape(-1).pow(2).sum()
gradient_norms.append((iteration, norm.reshape(-1)))
# Evaluation
with torch.no_grad():
if iteration % eval_freq == 0:
x = torch.from_numpy(dataset['test'].images).to(device)
y = torch.from_numpy(
dataset['test'].labels).argmax(dim=1).long().to(device)
prediction = net.forward(x)
acc = accuracy(prediction, y)
test_acc.append((iteration, acc))
print("Iteration: {}\t\tTest accuracy: {}".format(
iteration, acc))
# Save raw output
now = datetime.datetime.now()
time_stamp = "{}{}{}{}{}".format(now.year, now.month, now.day, now.hour,
now.minute)
net_name = "cnn"
if not os.path.isdir(os.path.join(output_dir, net_name, time_stamp)):
os.makedirs(os.path.join(output_dir, net_name, time_stamp))
metrics = {
"train_loss": train_loss,
"gradient_norms": gradient_norms,
"train_acc": train_acc,
"test_acc": test_acc
}
raw_data = {"net": net, "metrics": metrics}
pickle.dump(
raw_data,
open(os.path.join(output_dir, net_name, time_stamp, "raw_data"), "wb"))
# Save plots
# Loss
fig, ax = plt.subplots()
iter = [i for (i, q) in train_loss]
loss = [q for (i, q) in train_loss]
ax.plot(iter, loss)
ax.set(xlabel='Iteration',
ylabel='Loss (log)',
title='Batch training loss')
ax.set_yscale('log')
ax.grid()
fig.savefig(os.path.join(output_dir, net_name, time_stamp, "loss.png"))
# gradient norm
fig, ax = plt.subplots()
iter = [i for (i, q) in gradient_norms]
norm = [q for (i, q) in gradient_norms]
ax.plot(iter, norm)
ax.set(xlabel='Iteration', ylabel='Norm', title='Gradient norm')
ax.grid()
fig.savefig(
os.path.join(output_dir, net_name, time_stamp, "gradient_norm.png"))
# accuracies
fig, ax = plt.subplots()
iter = [i for (i, q) in train_acc]
accu = [q for (i, q) in train_acc]
ax.plot(iter, accu, label='Train')
iter = [i for (i, q) in test_acc]
accu = [q for (i, q) in test_acc]
ax.plot(iter, accu, label='Test')
ax.set(xlabel='Iteration',
ylabel='Accuracy',
title='Train and test accuracy')
ax.legend()
ax.grid()
fig.savefig(os.path.join(output_dir, net_name, time_stamp, "accuracy.png"))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
# select which device to train the model on
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# compute the input size of the MLP
input_size, n_classes = 3 * 32 * 32, 10
# if no pretrained model is passed to the commandline the convnet model will be initialized
if FLAGS.model == "custom":
model = ConvNet(3, n_classes).to(device)
else:
# check if the requested pretrained is available
assert FLAGS.model in pretrained_dict, "Model not available in pre_trained dict, given: {}, available: {}".format(
FLAGS.model, pretrained_dict.keys())
model = pretrained_dict[FLAGS.model](pretrained=True)
# image transforms needed to be alligned with the type of input the pretrained model is used to
normalize = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
resize = T.Resize(256)
centerCrop = T.CenterCrop(224)
# break out the last layer and add a linear layer that has 10 outputs instead of 1000
layers = []
first_linear = False
for child in model.children():
if list(child.children()) == []:
if isinstance(child, nn.Linear) and not first_linear:
layers.append(nn.Flatten())
first_linear = True
layers.append(child)
else:
for grandchild in child.children():
if isinstance(grandchild, nn.Linear) and not first_linear:
layers.append(nn.Flatten())
first_linear = True
layers.append(grandchild)
model = nn.Sequential(
*layers[:-1],
nn.Linear(in_features=layers[-1].in_features, out_features=10))
model.to(device)
# freeze the layers that are pretrained
for child in list(model.children())[:-1]:
for param in child.parameters():
param.requires_grad = False
# define the dataset, loss function and optimizer
dataset = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_fn = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=FLAGS.learning_rate)
for step in range(FLAGS.max_steps):
X_train, y_train = dataset['train'].next_batch(FLAGS.batch_size)
optimizer.zero_grad()
# from to normalize the data for pretrained model https://discuss.pytorch.org/t/how-to-efficiently-normalize-a-batch-of-tensor-to-0-1/65122
if FLAGS.model != "custom":
X_train, y_train = torch.tensor(X_train).float(), torch.tensor(
y_train).float().to(device)
X_train -= X_train.min(1, keepdim=True)[0]
X_train /= X_train.max(1, keepdim=True)[0]
X_train = torch.tensor([
normalize(T.ToTensor()(centerCrop(resize(
T.ToPILImage()(x))))).numpy() for x in X_train
]).to(device)
else:
# move to correct device and shape for MLP
X_train, y_train = torch.tensor(X_train).float().to(
device), torch.tensor(y_train).float().to(device)
predictions = model(X_train)
train_loss = loss_fn(predictions, y_train.argmax(1).long())
train_loss.backward()
optimizer.step()
# add the loss and accuracy to the lists for plotting
train_overall_loss.append(train_loss.cpu().detach().sum())
train_overall_accuracy.append(
accuracy(predictions.cpu().detach(),
y_train.cpu().detach()))
train_x_axis.append(step)
# test the model when eval freq is reached or if it is the last step
if not step % FLAGS.eval_freq or step + 1 == FLAGS.max_steps:
model.eval()
test_accuracies, test_losses_list = [], []
# test batchwise since it doesnot fit my gpu
for X_test, y_test in cifar_test_generator(dataset):
if FLAGS.model != "custom":
X_test, y_test = torch.tensor(X_test).float(
), torch.tensor(y_test).float().to(device)
X_test -= X_test.min(1, keepdim=True)[0]
X_test /= X_test.max(1, keepdim=True)[0]
X_test = torch.tensor([
normalize(T.ToTensor()(centerCrop(
resize(T.ToPILImage()(x))))).numpy()
for x in X_test
]).to(device)
else:
# move to correct device and shape for MLP
X_test, y_test = torch.tensor(X_test).float().to(
device), torch.tensor(y_test).float().to(device)
predictions = model(X_test)
test_loss = loss_fn(predictions, y_test.argmax(1).long())
test_accuracy = accuracy(predictions, y_test)
# add the values to compute the average loss and accuracy for the entire testset
test_accuracies.append(test_accuracy.cpu().detach())
test_losses_list.append(test_loss.cpu().detach().sum())
print(
"[{:5}/{:5}] Train loss {:.5f} Test loss {:.5f} Test accuracy {:.5f}"
.format(step, FLAGS.max_steps, train_loss, test_loss,
sum(test_accuracies) / len(test_accuracies)))
test_overall_accuracy.append(
sum(test_accuracies) / len(test_accuracies))
test_overall_loss.append(
sum(test_losses_list) / len(test_losses_list))
test_x_axis.append(step)
model.train()
# freeze the pretrained layers
if FLAGS.model != "custom":
for child in list(model.children())[:-1]:
for param in child.parameters():
param.requires_grad = False
plt.plot(train_x_axis, train_overall_loss, label="Avg Train loss")
plt.plot(test_x_axis, test_overall_loss, label="Avg Test loss")
plt.legend()
plt.savefig("convnet_loss_curve")
plt.show()
plt.plot(train_x_axis,
train_overall_accuracy,
label="Train batch accuracy")
plt.plot(test_x_axis, test_overall_accuracy, label="Test set accuracy")
plt.legend()
plt.savefig("convnet_accuracy_curve")
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# Load data
cifar10 = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir,
one_hot=False,
validation_size=5000)
train_set = cifar10['train']
test_set = cifar10['test']
val_set = cifar10['validation']
# Initialize model
n_channels = len(train_set.images[0].shape)
n_classes = train_set.labels.max() + 1
# set device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = ConvNet(n_channels, n_classes)
model = model.to(device)
model = nn.DataParallel(model)
cross_entropy = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
total_loss = 0
losses = []
val_acc = []
train_acc = []
for i in range(FLAGS.max_steps + 1):
# prepare batch
x, y = train_set.next_batch(FLAGS.batch_size)
x, y = torch.tensor(x), torch.tensor(y, dtype=torch.long)
x, y = x.to(device), y.to(device)
# forward pass
out = model(x)
loss = cross_entropy(out, y)
total_loss += loss.item()
# keep track of training accuracy
train_acc.append(accuracy(out, y))
# backward pass
model.zero_grad()
loss.backward()
optimizer.step()
if i % FLAGS.eval_freq == 0 and i != 0:
with torch.no_grad():
val_inputs = test_set.images
val_inputs = torch.tensor(val_inputs)
val_inputs = val_inputs.to(device)
pred = model(val_inputs)
targ = torch.tensor(test_set.labels)
targ = targ.to(device)
acc = accuracy(pred, targ)
losses.append(total_loss)
val_acc.append(acc)
print()
print("- - - - - - - - - -")
print('- STEPS:\t\t\t', i)
print('- TRAIN ACC: \t\t\t', np.array(train_acc).mean())
print('- VALIDATION ACC:\t\t', acc)
print("- - - - - - - - - -")
train_acc = []
total_loss = 0
print("Loss over time: \t", losses)
print("Val acc over time: \t", val_acc)
with open('cnn_data.dill', 'wb') as f:
dill.dump({'train_loss': losses, 'val_acc': val_acc}, f)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).float().to(device)
y = torch.from_numpy(y).float().to(device)
n_classes = y.shape[1]
n_channels = x.shape[1]
CNN = ConvNet(n_channels, n_classes)
CNN.to(device)
if OPTIMIZER_DEFAULT == 'SGD':
optimizer = optim.SGD(CNN.parameters())
elif OPTIMIZER_DEFAULT == 'ADAM':
optimizer = optim.Adam(CNN.parameters())
else:
print('Try SGD or ADAM...')
loss = nn.CrossEntropyLoss()
l_list = list()
t_list = list()
train_acc = list()
test_acc = list()
iterations = list()
print('\nTraining...')
for i in range(FLAGS.max_steps):
optimizer.zero_grad()
s_pred = CNN(x)
f_loss = loss(s_pred, y.argmax(dim=1))
f_loss.backward()
optimizer.step()
if i % FLAGS.eval_freq == 0:
iterations.append(i + 1)
l_list.append(round(f_loss.item(), 3))
train_acc.append(accuracy(s_pred, y))
test_size = cifar10['test'].labels.shape[0]
iter_num = 10
tmp_size = int(test_size / iter_num)
tmp_correct = 0
tmp_loss = 0
for j in range(iter_num):
t_x, t_y = cifar10['test'].next_batch(tmp_size)
t_x = torch.from_numpy(t_x).float().to(device)
t_y = torch.from_numpy(t_y).float().to(device)
t_pred = CNN(t_x)
tmp_loss += loss(t_pred, t_y.argmax(dim=1)).item()
tmp_correct += accuracy(t_pred, t_y) * tmp_size
test_acc.append(tmp_correct / test_size)
t_list.append(round(tmp_loss / iter_num, 3))
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).float().to(device)
y = torch.from_numpy(y).float().to(device)
print('Done!\n')
print('Training Losses:', l_list)
print('Test Losses:', t_list)
print('Training Accuracies:', train_acc)
print('Test Accuracies:', test_acc)
print('Best Test Accuracy:', max(test_acc))
fig, axs = plt.subplots(1, 2, figsize=(10, 5))
axs[0].plot(iterations, train_acc, iterations, test_acc)
axs[0].set_xlabel('Iteration')
axs[0].set_ylabel('Accuracy')
axs[0].legend(('train', 'test'))
axs[1].plot(iterations, l_list, iterations, t_list)
axs[1].set_xlabel('Iteration')
axs[1].set_ylabel('Loss')
axs[1].legend(('train', 'test'))
fig.tight_layout()
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if FLAGS.data_dir:
DATA_DIR_DEFAULT = FLAGS.data_dir
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
batch_size = FLAGS.batch_size
learning_rate = FLAGS.learning_rate
cifar_data = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_data = cifar_data['train']
test_data = cifar_data['test']
input_channels = train_data.images.shape[1]
n_classes = train_data.labels.shape[1]
criterion = nn.CrossEntropyLoss()
model = ConvNet(input_channels, n_classes).to(device)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
# Train and Test losses
losses = [[], []]
# Train and Test accuracies
accuracies = [[], []]
# True iteration for plotting
iterations = []
for iteration in np.arange(FLAGS.max_steps):
x, y = train_data.next_batch(batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).type(torch.LongTensor).to(device)
train_output = model.forward(x)
loss = criterion(train_output, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if iteration % FLAGS.eval_freq == 0 or iteration == FLAGS.max_steps - 1:
iterations.append(iteration)
x_test, y_test = test_data.next_batch(10 * batch_size)
x_test = torch.from_numpy(x_test).to(device)
y_test = torch.from_numpy(np.argmax(y_test, axis=1)).type(torch.LongTensor).to(device)
# Second forward pass for test set
with torch.no_grad():
test_output = model.forward(x_test)
# Calculate losses
train_loss = criterion.forward(train_output, y)
losses[0].append(train_loss)
test_loss = criterion.forward(test_output, y_test)
losses[1].append(test_loss)
# Calculate accuracies
train_acc = accuracy(train_output, y)
test_acc = accuracy(test_output, y_test)
accuracies[0].append(train_acc)
accuracies[1].append(test_acc)
print("Iteration {}, Train loss: {}, Train accuracy: {}, Test accuracy: {}".format(iteration, train_loss,
train_acc, test_acc))
fig = plt.figure(figsize=(25, 10), dpi=200)
fig.suptitle('PyTorch ConvNet: Losses and Accuracies', fontsize=40)
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)
ax1.plot(iterations, losses[0], linewidth=4, color="g", label="Train loss")
ax1.plot(iterations, losses[1], linewidth=4, color="c", label="Test loss")
ax2.plot(iterations, accuracies[0], linewidth=4, color="g", label="Train accuracy")
ax2.plot(iterations, accuracies[1], linewidth=4, color="c", label="Test accuracy")
ax1.set_xlabel('$Iteration$', fontsize=28)
ax1.set_ylabel('$Loss$', fontsize=28)
ax2.set_xlabel('$Iteration$', fontsize=28)
ax2.set_ylabel('$Accuracy$', fontsize=28)
ax1.legend(fontsize=22)
ax2.legend(fontsize=22)
plt.savefig("../figures/pytorch_convnet.png")
plt.show()
with open('results_convnet.pkl', 'wb') as f:
pickle.dump([losses, accuracies], f)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# ## Prepare all functions
# # Get number of units in each hidden layer specified in the string such as 100,100
# if FLAGS.dnn_hidden_units:
# dnn_hidden_units = FLAGS.dnn_hidden_units.split(",")
# dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units]
# else:
# dnn_hidden_units = []
# Set path to data
data_dir = FLAGS.data_dir
data = cifar10_utils.get_cifar10(data_dir)
# Prepare the test set
input_dims_test = data['test'].images.shape
height = input_dims_test[2]
width = input_dims_test[3]
channels = input_dims_test[1]
# num_images_test = input_dims_test[0]
# image_dims_ravel = height * width * channels
X_test = data["test"].images
Y_test = data["test"].labels
# Make acceptable input for test
# X_test = X_test.reshape((num_images_test, image_dims_ravel))
# make usable by pytorch
X_test = torch.tensor(X_test, requires_grad=False).type(dtype).to(device)
Y_test = torch.tensor(Y_test, requires_grad=False).type(dtype).to(device)
# Determine the channels
n_channels = channels
model = ConvNet(n_channels=n_channels, n_classes=10)
accuracy_train_log = list()
accuracy_test_log = list()
loss_train_log = list()
loss_test_log = list()
# FLAGS hold command line arguments
batch_size = FLAGS.batch_size
numb_iterations = FLAGS.max_steps
learning_rate = FLAGS.learning_rate
evaluation_freq = FLAGS.eval_freq
logging.info(f"learning rate: %2d " % learning_rate)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
for step in range(numb_iterations):
X_batch, Y_batch = data['train'].next_batch(batch_size)
# X_batch = X_batch.reshape((batch_size, image_dims_ravel))
# Convert to tensors which are handled by the device
X_batch = torch.from_numpy(X_batch).type(dtype).to(device)
Y_batch = torch.from_numpy(Y_batch).type(dtype).to(device)
# why do we need this again?
optimizer.zero_grad()
targs = Y_batch.argmax(dim=1)
outputs = model.forward(X_batch)
loss_current = criterion(outputs, targs)
loss_current.backward()
optimizer.step()
running_loss = loss_current.item()
if step % evaluation_freq == 0:
loss_train_log.append(running_loss)
accuracy_train_log.append(accuracy(outputs, Y_batch))
logging.info(f"train performance: loss = %4f, accuracy = %4f ",
loss_train_log[-1], accuracy_train_log[-1])
# Get performance on the test set
targs_test = Y_test.argmax(dim=1)
outputs = model(X_test)
loss_test_log.append(criterion(outputs, targs_test))
accuracy_test_log.append(accuracy(outputs, Y_test))
logging.info(f"test performance: loss = %4f , accuracy = %4f\n",
loss_test_log[-1], accuracy_test_log[-1])
# TODO: implement early stopping ?
list_acc = list()
list_loss = list()
for i in range(0, 10):
selection = random.sample(range(1, 1000), 32)
targs_test = Y_test[selection].argmax(dim=1)
outputs_test = model(X_test[selection])
loss_current_test = criterion(outputs_test,
targs_test).detach().item()
acc_current_test = accuracy(outputs_test, Y_test[selection])
list_loss.append(loss_current_test)
list_acc.append(acc_current_test)
path = "./mlp_results_pytorch/"
date_time = datetime.now().replace(
second=0, microsecond=0).strftime(format="%Y-%m-%d-%H-%M")
np.save(os.path.join(path, date_time + "accuracy_test"), accuracy_test_log)
np.save(os.path.join(path, date_time + "loss_test"), loss_test_log)
np.save(os.path.join(path, date_time + "loss_train"), loss_train_log)
np.save(os.path.join(path, date_time + "accuracy_train"),
accuracy_train_log)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
def init_weights(m):
print(m)
if type(m) == nn.Linear:
m.weight.data.uniform_(0.0, 1.0)
print(m.weight)
m.bias.data.fill_(0.0)
print(m.bias)
lr = FLAGS.learning_rate
eval_freq = FLAGS.eval_freq
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
input_size = 32 * 32 * 3
output_size = 10
# load dataset
raw_data = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_data = raw_data['train']
validation_data = raw_data["validation"]
test_data = raw_data['test']
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = ConvNet(n_channels=3, n_classes=10).to(device)
print(model.layers)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
loss_target = nn.CrossEntropyLoss()
csv_data = [[
'step', 'train_loss', 'test_loss', 'train_accuracy', 'test_accuracy'
]]
print("initial weights as normal distribution and bias as zeros")
# model.layers.apply(init_weights)
for step in range(max_steps):
x, y = train_data.next_batch(batch_size)
# x = x.reshape(batch_size, input_size)
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
# train
# x = Variable(torch.from_numpy(x))
output = model.forward(x)
loss = loss_target.forward(output, y.argmax(dim=1))
# somehow we need to divide the loss by the output size to get the same loss
loss_avg = loss.item()
# model.zero_grad()
optimizer.zero_grad()
loss.backward()
# only need to update weights for linear module for each step
optimizer.step()
# with torch.no_grad():
# for param in model.parameters():
# param.data -= lr * param.grad
train_acc = accuracy(output, y)
# with the \r and end = '' trick, we can print on the same line
print('\r[{}/{}] train_loss: {} train_accuracy: {}'.format(
step + 1, max_steps, round(loss_avg, 3), round(train_acc, 3)),
end='')
# evaluate
if step % eval_freq == 0 or step >= (max_steps - 1):
test_batch_acc = test_loss_batch = 0
test_batch_size = 100
total_test_samples = test_data.num_examples
batch_num = math.ceil(total_test_samples / test_batch_size)
for i in range(batch_num):
x, y = test_data.next_batch(test_batch_size)
# x = x.reshape(test_data.num_examples, input_size)
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
output = model.forward(x)
test_loss_batch += loss_target.forward(output,
y.argmax(dim=1)).item()
test_batch_acc += accuracy(output, y)
test_loss = test_loss_batch / batch_num
test_acc = test_batch_acc / batch_num
csv_data.append([step, loss_avg, test_loss, train_acc, test_acc])
print(' test_loss: {}, test_accuracy: {}'.format(
round(test_loss, 3), round(test_acc, 3)))
with open(
'results/train_summary_torchconvnet_{}.csv'.format(int(
time.time())), 'w') as csv_file:
writer = csv.writer(csv_file)
writer.writerows(csv_data)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# initialize empty dictionaries
x, y, accu, loss = ({} for _ in range(4))
# retrieve data
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
# determine shapes
image_shape = data['test'].images[0].shape
nr_pixels = image_shape[0] * image_shape[1] * image_shape[2]
nr_labels = data['test'].labels.shape[1]
nr_test = data['test'].images.shape[0]
# set standards
tensor = torch.FloatTensor
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# save in variables
for tag in data:
nr_images = data[tag].images.shape[0]
x[tag] = torch.tensor(data[tag].images)
y[tag] = torch.tensor(data[tag].labels)
accu[tag] = []
loss[tag] = []
x['test'] = x['test'].type(tensor).to(device)
y['test'] = y['test'].type(tensor).to(device)
# create neural network
neural_network = ConvNet(nr_pixels, nr_labels).to(device)
cross_entropy = nn.CrossEntropyLoss().to(device)
parameter_optimizer = torch.optim.Adam(params=neural_network.parameters(), \
lr=FLAGS.learning_rate)
dx = 1
i = 0
logs = ['\n'.join([key + ' : ' + str(value)
for key, value in vars(FLAGS).items()])]
while i < FLAGS.max_steps and np.linalg.norm(dx) > 1e-5:
i += 1
# sample batch from data
rand_idx = np.random.randint(x['train'].shape[0], size=FLAGS.batch_size)
x_batch = x['train'][rand_idx].type(tensor).to(device)
y_batch = y['train'][rand_idx].type(tensor).to(device)
parameter_optimizer.zero_grad()
nn_out = neural_network.forward(x_batch)
ce_out = cross_entropy.forward(nn_out, y_batch.argmax(dim=1))
ce_out.backward()
parameter_optimizer.step()
if i % FLAGS.eval_freq == 0:
# save train accuracy and loss
accu['train'].append(accuracy(nn_out, y_batch))
loss['train'].append(ce_out)
# calculate and save test accuracy and loss
nn_out = neural_network.forward(x['test'])
ce_out = cross_entropy.forward(nn_out, y['test'].argmax(dim=1))
accu['test'].append(accuracy(nn_out, y['test']))
loss['test'].append(ce_out.item())
# show results in command prompt and save log
s = 'iteration ' + str(i) + ' | train acc/loss ' + \
str('{:.3f}'.format(accu['train'][-1])) + '/' + \
str('{:.3f}'.format(loss['train'][-1])) + ' | test acc/loss ' \
+ str('{:.3f}'.format(accu['test'][-1])) + '/' + \
str('{:.3f}'.format(loss['test'][-1]))
logs.append(s)
print(s)
#sys.stdout.write("\r%s" % s)
#sys.stdout.flush()
t = str(time.time())
# write logs
with open('results/logs_' + t + '.txt', 'w') as f:
f.writelines(['%s\n' % item for item in logs])
# write data to file
with open('results/data_' + t + '.txt', 'w') as f:
f.write('train accuracy')
f.writelines([',%s' % str(item) for item in accu['train']])
f.write('\ntrain loss')
f.writelines([',%s' % str(item) for item in loss['train']])
f.write('\ntest accuracy')
f.writelines([',%s' % str(item) for item in accu['test']])
f.write('\ntest loss')
f.writelines([',%s' % str(item) for item in loss['test']])
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_list = []
batch_list = []
accuracy_list = []
batch_list_test = []
use_cuda = torch.cuda.is_available()
if use_cuda:
print('Running in GPU model')
device = torch.device('cuda' if use_cuda else 'cpu')
dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
for iter in range(313):
x, y = cifar10['test'].next_batch(FLAGS.batch_size)
y = np.argmax(y, axis=1)
batch_list_test.append((x, y))
for iter in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
y = np.argmax(y, axis=1)
batch_list.append((x, y))
print('Batch list completed')
in_features = 3
out_features = 10 #num_classes
net = ConvNet(in_features, out_features).to(device)
lossfunc = nn.CrossEntropyLoss()
optimiser = optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
net.train()
for i in range(FLAGS.max_steps):
inputs, labels = batch_list[i]
# inputs = torch.from_numpy(inputs)
inputs = torch.tensor(inputs)
labels = torch.from_numpy(labels).long()
inputs, labels = inputs.to(device), labels.to(device)
optimiser.zero_grad()
outputs = net.forward(inputs.float())
loss = lossfunc(outputs, labels)
loss_list.append(loss)
loss.backward()
optimiser.step()
if (i + 1) % FLAGS.eval_freq == 0:
net.eval()
total = 0
for data in batch_list_test:
x_test, y_test = data
x_test = torch.from_numpy(x_test).type(dtype).to(device)
y_test = torch.from_numpy(y_test).long().to(device)
predicted = net.forward(x_test)
acc = accuracy(predicted, y_test)
total += acc
accuracy_list.append(total / len(batch_list_test))
print('Accuracy on test set at step {} is {}'.format(
i, total / len(batch_list_test)))
print('Loss of training is {}'.format(loss.item()))
plt.subplot(2, 1, 1)
plt.plot(
np.arange(len(accuracy_list) * FLAGS.eval_freq, step=FLAGS.eval_freq),
accuracy_list, 'o-')
plt.xlabel('Step')
plt.ylabel('Accuracy')
#
plt.subplot(2, 1, 2)
plt.plot(np.arange(len(loss_list)), loss_list)
plt.xlabel('Step')
plt.ylabel('Loss')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
dataset_dict = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_loader = dataset_dict['train']
test_loader = dataset_dict['test']
test_images = Variable(torch.tensor(test_loader.images))
test_labels = torch.tensor(test_loader.labels)
model = ConvNet(n_channels=3, n_classes=10).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
criterion = torch.nn.CrossEntropyLoss()
test_accs = []
train_accs = []
losses = []
for epoch in range(FLAGS.max_steps):
model.train()
batch_x, batch_y = train_loader.next_batch(FLAGS.batch_size)
batch_x, batch_y = Variable(
torch.tensor(batch_x).to(device)), Variable(
torch.tensor(batch_y).to(device))
optimizer.zero_grad()
out = model.forward(batch_x)
loss = criterion(out, batch_y.max(1)[1])
# l2_reg = torch.tensor(0).float().to(device)
# for param in model.parameters():
# l2_reg += torch.norm(param, 2)
# loss = torch.add(loss, l2_reg/100)
# l1_reg = torch.tensor(0).float().to(device)
# for param in model.parameters():
# l1_reg += torch.norm(param, 1)
# loss = torch.add(loss, l1_reg / 100000)
losses.append(round(float(loss), 3))
loss.backward()
optimizer.step()
if epoch % FLAGS.eval_freq == 0:
with torch.no_grad():
model.eval()
# print accuracy on test and train set
train_acc = accuracy(out, batch_y)
#COMPUTE TEST ACCURACY
all_predictions = torch.tensor([]).float().to(device)
for i in range(FLAGS.batch_size, test_images.shape[0],
FLAGS.batch_size):
out = model.forward(
test_images[i - FLAGS.batch_size:i].to(device))
all_predictions = torch.cat((all_predictions, out))
if i < test_images.shape[0]:
out = model.forward(test_images[i:].to(device))
all_predictions = torch.cat((all_predictions, out))
test_acc = accuracy(all_predictions, test_labels.to(device))
print(
'Train Epoch: {}/{}\tLoss: {:.6f}\tTrain accuracy: {:.6f}\tTest accuracy: {:.6f}'
.format(epoch, FLAGS.max_steps, loss, train_acc, test_acc))
test_accs.append(float(test_acc))
train_accs.append(float(train_acc))
with torch.no_grad():
# COMPUTE TEST ACCURACY
all_predictions = torch.tensor([]).float().to(device)
for i in range(FLAGS.batch_size, test_images.shape[0],
FLAGS.batch_size):
out = model.forward(test_images[i - FLAGS.batch_size:i].to(device))
all_predictions = torch.cat((all_predictions, out))
if i < test_images.shape[0]:
out = model.forward(test_images[i:].to(device))
all_predictions = torch.cat((all_predictions, out))
test_acc = accuracy(all_predictions, test_labels.to(device))
print('FINAL Test accuracy: {:.6f}'.format(test_acc))
import matplotlib.pyplot as plt
plt.figure()
plt.plot([i for i in range(0, epoch + 1, EVAL_FREQ_DEFAULT)], train_accs)
plt.plot([i for i in range(0, epoch + 1, EVAL_FREQ_DEFAULT)], test_accs)
plt.legend(["train", "test"])
plt.ylabel("accuracy")
plt.xlabel("epoch")
plt.savefig("cnn_accuracy")
plt.figure()
plt.plot([i for i in range(0, epoch + 1)], losses)
plt.legend(["loss"])
plt.ylabel("loss")
plt.xlabel("epoch")
plt.savefig("cnn_loss")
def train(_run):
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
def get_xy_tensors(batch):
x, y = batch
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
return x, y
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
datasets = cifar10_utils.read_data_sets(DATA_DIR_DEFAULT, one_hot=False)
train_data = datasets['train']
test_data = datasets['test']
model = ConvNet(n_channels=3, n_classes=10).to(device)
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
log_every = 100
avg_loss = 0
avg_acc = 0
for step in range(FLAGS.max_steps):
x, y = get_xy_tensors(train_data.next_batch(FLAGS.batch_size))
# Forward and backward passes
optimizer.zero_grad()
out = model.forward(x)
loss = loss_fn(out, y)
loss.backward()
# Parameter updates
optimizer.step()
avg_loss += loss.item() / log_every
avg_acc += accuracy(out, y) / log_every
if step % log_every == 0:
print('[{}/{}] train loss: {:.6f} train acc: {:.6f}'.format(
step, FLAGS.max_steps, avg_loss, avg_acc))
_run.log_scalar('train-loss', avg_loss, step)
_run.log_scalar('train-acc', avg_acc, step)
avg_loss = 0
avg_acc = 0
# Evaluate
if step % FLAGS.eval_freq == 0 or step == (FLAGS.max_steps - 1):
n_test_iters = int(
np.ceil(test_data.num_examples / TEST_BATCH_SIZE))
test_loss = 0
test_acc = 0
for i in range(n_test_iters):
x, y = get_xy_tensors(test_data.next_batch(TEST_BATCH_SIZE))
model.eval()
out = model.forward(x)
model.train()
test_loss += loss_fn(out, y).item() / n_test_iters
test_acc += accuracy(out, y) / n_test_iters
print('[{}/{}] test accuracy: {:6f}'.format(
step, FLAGS.max_steps, test_acc))
_run.log_scalar('test-loss', test_loss, step)
_run.log_scalar('test-acc', test_acc, step)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
#load data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
#hyperparameters
eta = FLAGS.learning_rate
eps = 1e-6 # convergence criterion
max_steps = FLAGS.max_steps
b_size = FLAGS.batch_size
#test_data
x_test = cifar10["test"].images
y_test = cifar10["test"].labels
#get usefull dimensions
n_channels = np.size(x_test, 1)
n_classes = np.size(y_test, 1)
n_batches = np.size(x_test, 0) // b_size
#load whole train data ############################################################
x_train = cifar10["train"].images
x_train = torch.tensor(x_train, requires_grad=False).type(dtype).to(device)
n_train_batches = np.size(x_train, 0) // b_size
#initialize the ConvNet model
model = ConvNet(n_channels, n_classes)
get_loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=eta)
model.to(device)
train_loss = []
test_loss = []
train_acc = []
test_acc = []
for step in range(max_steps):
#get batch
x, y = cifar10['train'].next_batch(b_size)
x = torch.tensor(x).type(dtype).to(device)
y = torch.tensor(y).type(dtype).to(device)
#forward pass
pred = model.forward(x)
#get training loss
current_loss = get_loss(pred, y.argmax(dim=1))
optimizer.zero_grad()
#get training loss gradient
current_loss.backward()
#get training accuracy
current_train_acc = accuracy(pred, y)
optimizer.step()
#free memory up
pred.detach()
x.detach()
y.detach()
#select evaluation step
if (step % FLAGS.eval_freq) == 0:
# c_train_loss = current_loss.data.item()
# train_loss.append(c_train_loss)
# train_acc.append(current_train_acc)
c_train_loss = 0
current_train_acc = 0
c_test_loss = 0
current_test_acc = 0
#loop through train set in batches ######################################################
for test_batch in range(n_train_batches):
#load test data
x_train, y_train = cifar10['train'].next_batch(b_size)
x_train = torch.tensor(
x_train, requires_grad=False).type(dtype).to(device)
y_train = torch.tensor(
y_train, requires_grad=False).type(dtype).to(device)
#get test batch results
train_pred = model.forward(x_train)
current_train_loss = get_loss(train_pred,
y_train.argmax(dim=1))
c_train_loss += current_train_loss.data.item()
current_train_acc += accuracy(train_pred, y_train)
#free memory up
train_pred.detach()
x_train.detach()
y_train.detach()
#loop through test set in batches
for test_batch in range(n_batches):
#load test data
x_test, y_test = cifar10['test'].next_batch(b_size)
x_test = torch.tensor(
x_test, requires_grad=False).type(dtype).to(device)
y_test = torch.tensor(
y_test, requires_grad=False).type(dtype).to(device)
#get test batch results
test_pred = model.forward(x_test)
current_test_loss = get_loss(test_pred, y_test.argmax(dim=1))
c_test_loss += current_test_loss.data.item()
current_test_acc += accuracy(test_pred, y_test)
#free memory up
test_pred.detach()
x_test.detach()
y_test.detach()
#get full training set results #########################################################
c_train_loss = c_train_loss / n_train_batches
current_train_acc = current_train_acc / n_train_batches
train_loss.append(c_train_loss)
train_acc.append(current_train_acc)
#get full test set results
c_test_loss = c_test_loss / n_batches
current_test_acc = current_test_acc / n_batches
test_loss.append(c_test_loss)
test_acc.append(current_test_acc)
print('\nStep ', step, '\n------------\nTraining Loss = ',
round(c_train_loss, 4),
', Train Accuracy = ', current_train_acc, '\nTest Loss = ',
round(c_test_loss, 4), ', Test Accuracy = ',
round(current_test_acc, 4))
if step > 0 and abs(test_loss[(int(step / FLAGS.eval_freq))] -
test_loss[int(step / FLAGS.eval_freq) -
1]) < eps:
break
plot_graphs(train_loss,
'Training Loss',
'orange',
test_loss,
'Test Loss',
'blue',
title='Adams optimization',
ylabel='Loss',
xlabel='Steps')
plot_graphs(train_acc,
'Training Accuracy',
'darkorange',
test_acc,
'Test Accuracy',
'darkred',
title='Adams optimization',
ylabel='Accuracy',
xlabel='Steps')
#save results:
np.save('train_loss', train_loss)
np.save('train_acc', train_acc)
np.save('test_loss', test_loss)
np.save('test_acc', test_acc)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
learning_rate = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py')
convnet = ConvNet(3, 10).cuda()
opt = optim.Adam(convnet.parameters(), lr=learning_rate)
loss_function = nn.CrossEntropyLoss()
train_losses = []
accuracies = []
steps = []
for step in range(max_steps):
total_loss = 0
x, y = cifar10['train'].next_batch(batch_size)
x_tensor = torch.from_numpy(x).cuda()
y_tensor = torch.from_numpy(y).cuda()
out = convnet(x_tensor)
loss = loss_function(out, torch.max(y_tensor, 1)[1])
total_loss += loss
opt.zero_grad()
loss.backward()
opt.step()
train_losses.append(total_loss)
print('Step: {} Loss: {:.4f}'.format(step + 1, total_loss))
if (step + 1) % eval_freq == 0:
test_pred_seq = []
test_labels_seq = []
while (cifar10['test']._epochs_completed == 0):
test_x, test_y = cifar10['test'].next_batch(batch_size)
test_x_tensor = torch.from_numpy(test_x).cuda()
test_y_tensor = torch.from_numpy(test_y).cuda()
test_out = convnet(test_x_tensor)
test_out_cpu = test_out.cpu().detach().numpy()
test_y_tensor_cpu = test_y_tensor.cpu().detach().numpy()
test_pred_seq.append(test_out_cpu)
test_labels_seq.append(test_y_tensor_cpu)
test_pred = np.vstack(test_pred_seq)
test_labels = np.vstack(test_labels_seq)
test_accuracy = accuracy(test_pred, test_labels)
accuracies.append(test_accuracy)
steps.append(step + 1)
cifar10['test']._epochs_completed = 0
print('Step: {} Accuracy {:.2f}'.format(step + 1, test_accuracy))
plt.plot(range(max_steps), train_losses)
plt.xlabel("Step")
plt.ylabel("Training loss")
plt.show()
plt.plot(steps, accuracies)
plt.xlabel("Step")
plt.ylabel("Test Accuracy")
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
import matplotlib.pyplot as plt
if not torch.cuda.is_available():
print("WARNING: CUDA DEVICE IS NOT AVAILABLE, WILL TRAIN ON CPU")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
train = data['train']
test = data['test']
images_test_np = test.images
labels_test_np = test.labels
vgg = ConvNet(3, 10)
vgg.to(device)
if MEM_DEBUG:
print("Memory after VGG loaded: Alloc: {} MB, Cached: {} MB".format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(vgg.parameters())
loss_train = np.zeros((int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
loss_test = np.zeros((int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
accuracy_test = np.zeros(
(int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
for i in range(0, FLAGS.max_steps):
print('iter', i + 1, end='\r')
images_np, labels_np = train.next_batch(FLAGS.batch_size)
images = torch.from_numpy(images_np).to(device)
labels = torch.from_numpy(np.argmax(labels_np, axis=1)).to(device)
if MEM_DEBUG:
print("\nMemory after train loaded: Alloc: {} MB, Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
optimizer.zero_grad()
pred = vgg(images)
loss = criterion(pred, labels.long())
loss.backward()
optimizer.step()
del images
del labels
if (i + 1) % FLAGS.eval_freq == 0:
if MEM_DEBUG:
print("Memory entering the eval: Alloc: {} MB, Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
loss_train[i // FLAGS.eval_freq] = loss.item()
images_test = torch.from_numpy(images_test_np).to(device)
labels_test = torch.from_numpy(np.argmax(labels_test_np,
axis=1)).to(device)
if MEM_DEBUG:
print("Memory after test loaded: Alloc: {} MB Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
vgg.eval()
with torch.no_grad():
pred_test = vgg(images_test)
vgg.train()
accuracy_test[i // FLAGS.eval_freq] = accuracy(
pred_test, F.one_hot(labels_test)).item()
loss_test[i // FLAGS.eval_freq] = criterion(
pred_test, labels_test.long()).item()
if PRINTS:
print()
print('test_loss:', loss_test[i // FLAGS.eval_freq])
print('test_accuracy:', accuracy_test[i // FLAGS.eval_freq])
print('train_loss:', loss_train[i // FLAGS.eval_freq])
if PLOTS:
fig, ax = plt.subplots(1, 2, figsize=(10, 5))
fig.suptitle('Training curves for Pytorch Convnet')
ax[0].set_title('Loss')
ax[0].set_ylabel('Loss value')
ax[0].set_xlabel('No of batches seen x{}'.format(FLAGS.eval_freq))
ax[0].plot(loss_train, label='Train')
ax[0].plot(loss_test, label='Test')
ax[0].legend()
ax[1].set_title('Accuracy')
ax[1].set_ylabel('Accuracy value')
ax[1].set_xlabel('No of batches seen x{}'.format(FLAGS.eval_freq))
ax[1].plot(accuracy_test, label='Test')
ax[1].legend()
import time
fig_name = 'pt_conv_training_{}_{}_{}_{}.jpg'.format(
FLAGS.max_steps, FLAGS.batch_size, FLAGS.eval_freq, time.time())
plt.savefig(fig_name)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
cifar10 = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train_data = cifar10['train']
n_channels = train_data.images.shape[0]
n_classes = train_data.labels.shape[1]
net = ConvNet(n_channels, n_classes)
net.to(device)
params = net.parameters()
optimizer = torch.optim.Adam(params, lr=FLAGS.learning_rate)
criterion = torch.nn.CrossEntropyLoss()
rloss = 0
train_acc_plot = []
test_acc_plot = []
loss_train = []
loss_test = []
print(f'[DEBUG] start training.... Max steps {FLAGS.max_steps}')
for i in range(0, FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x, y = torch.from_numpy(x).float().to(device), torch.from_numpy(
y).float().to(device)
out = net.forward(x)
loss = criterion(out, y.argmax(1))
optimizer.zero_grad()
loss.backward()
optimizer.step()
rloss += loss.item()
if i % FLAGS.eval_freq == 0:
train_accuracy = accuracy(out, y)
with torch.no_grad():
test_accuracys, test_losses = [], []
for j in range(0, FLAGS.max_steps):
test_x, test_y = cifar10['test'].next_batch(
FLAGS.batch_size)
test_x, test_y = torch.from_numpy(test_x).float().to(
device), torch.from_numpy(test_y).float().to(device)
test_out = net.forward(test_x)
test_loss = criterion(test_out, test_y.argmax(1))
test_accuracy = accuracy(test_out, test_y)
if device == 'cpu':
test_losses.append(test_loss)
else:
test_losses.append(test_loss.cpu().data.numpy())
test_accuracys.append(test_accuracy)
t_acc = np.array(test_accuracys).mean()
t_loss = np.array(test_losses).mean()
train_acc_plot.append(train_accuracy)
test_acc_plot.append(t_acc)
loss_train.append(rloss / (i + 1))
loss_test.append(t_loss)
print(
f"iter {i}, train_loss_avg {rloss/(i + 1)}, test_loss_avg {t_loss}, train_acc {train_accuracy}, test_acc_avg {t_acc}"
)
print('[DEBUG] Done training')
if FLAGS.plot:
print('[DEBUG] Start plotting...')
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
ax1.plot(np.arange(len(train_acc_plot)),
train_acc_plot,
label='training')
ax1.plot(np.arange(len(test_acc_plot)), test_acc_plot, label='testing')
ax1.set_title('Training evaluation with batch size ' +
str(FLAGS.batch_size) + '\n learning rate ' +
str(FLAGS.learning_rate) + '\n best accuracy ' +
str(max(test_acc_plot)))
ax1.set_ylabel('Accuracy')
ax1.legend()
ax2.plot(np.arange(len(loss_train)), loss_train, label='Train Loss')
ax2.plot(np.arange(len(loss_test)), loss_test, label='Test Loss')
ax2.set_title('Loss evaluation')
ax2.set_ylabel('Loss')
ax2.legend()
plt.xlabel('Iteration')
plt.savefig('convnet.png')
def train():
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda")
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
dataset = cifar10_utils.get_cifar10()
training = dataset['train']
test = dataset['test']
model = ConvNet(3, 10)
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
ce = torch.nn.CrossEntropyLoss()
test_accuracy = []
loss_list = []
for epoch in np.arange(0, FLAGS.max_steps):
x, y = training.next_batch(FLAGS.batch_size)
x = Variable(torch.tensor(x).to(device))
y = Variable(torch.tensor(y).to(device))
optimizer.zero_grad()
model.train()
yh = model.forward(x)
loss = ce(yh, torch.max(y, 1)[1])
loss_list.append(loss.item())
loss.backward()
optimizer.step()
if (epoch + 1) % int(FLAGS.eval_freq) == 0:
acc = []
with torch.no_grad():
for _ in np.arange(0, (test.num_examples // FLAGS.batch_size)):
x, y = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
y = torch.tensor(y).to(device)
model.eval()
yh = model.forward(x)
acc.append(accuracy(yh, y))
test_accuracy.append(np.mean(acc))
print(np.mean(acc))
import seaborn as sns
import matplotlib.pyplot as plt
f, axes = plt.subplots(1, 2)
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, EVAL_FREQ_DEFAULT),
test_accuracy,
ax=axes[0])
ax.set_title('Test accuracy')
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, 1),
loss_list,
ax=axes[1])
ax.set_title('Loss')
figure = ax.get_figure()
figure.savefig("cnn-pytorch-results")
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
print(train.images.shape)
# transform = transforms.Compose(
# [transforms.Resize((224, 224)),
# transforms.ToTensor(),
# transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
if FLAGS.model == 'ALEX':
train.images = torch.Tensor([resize(img) for img in train.images])
test.images = torch.Tensor([resize(img) for img in test.images])
print('doneee')
model = models.alexnet(pretrained=True)
torch.save(model.state_dict(), 'alexnet.txt')
# model = torch.load('alexnet.txt')
for param in model.features.parameters():
param.requires_grad = False
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.classifier.parameters(),
lr=FLAGS.learning_rate)
else:
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(),
lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(2): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
if FLAGS.model == 'ADAM': x = resizer(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
if FLAGS.model == 'ADAM': batch_x = resizer(batch_x)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])