def train():
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
dataset = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
a, b, c = dataset['train'].images.shape[1:]
n_classes = dataset['train'].labels.shape[1]
cnn = ConvNet(3, n_classes).to(device)
optimizer = optim.Adam(cnn.parameters(), lr=FLAGS.learning_rate)
crossentropy = nn.CrossEntropyLoss()
n_test = dataset['test'].images.shape[0]
for step in range(FLAGS.max_steps):
input, labels = dataset['train'].next_batch(FLAGS.batch_size)
labels = np.argmax(labels, axis=1)
input, labels = torch.from_numpy(input).to(device), torch.from_numpy(
labels).long().to(device)
predictions = cnn.forward(input)
loss = crossentropy(predictions, labels)
# clean up old gradients
cnn.zero_grad()
loss.backward()
optimizer.step()
if (step == FLAGS.max_steps - 1 or step % FLAGS.eval_freq == 0):
test_loss = []
test_accuracy = []
for i in range(0, n_test, FLAGS.batch_size):
test_input, test_labels = dataset['test'].next_batch(
FLAGS.batch_size)
test_input = torch.from_numpy(test_input).to(device)
test_labels = torch.from_numpy(np.argmax(
test_labels, axis=1)).long().to(device)
test_prediction = cnn.forward(test_input)
test_loss.append(
crossentropy(test_prediction, test_labels).item())
test_accuracy.append(accuracy(test_prediction, test_labels))
sys.stdout = open(
str(FLAGS.learning_rate) + '_' + str(FLAGS.max_steps) + '_' +
str(FLAGS.batch_size) + '_' + str(FLAGS.batch_size) +
'conv.txt', 'a')
print("{},{:f},{:f}".format(step, np.mean(test_loss),
np.mean(test_accuracy)))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
n_inputs = 3 * 32 * 32
n_classes = 10
batches_per_epoch = (int)(data['test'].images.shape[0] /
FLAGS.batch_size) # need this for test set
model = ConvNet(n_inputs, n_classes).to(device)
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
max_accuracy = 0.0
start_time = time.perf_counter()
for step in range(1, FLAGS.max_steps + 1):
x, y = get_batch(data, 'train', FLAGS.batch_size, device)
predictions = model.forward(x)
training_loss = loss_fn(predictions, y.argmax(dim=1))
optimizer.zero_grad()
training_loss.backward()
optimizer.step()
if step == 1 or step % FLAGS.eval_freq == 0:
with torch.no_grad():
test_loss = 0
test_acc = 0
for test_batch in range(batches_per_epoch):
x, y = get_batch(data, 'test', FLAGS.batch_size, device)
predictions = model(x)
test_loss += loss_fn(predictions,
y.argmax(dim=1)) / batches_per_epoch
test_acc += accuracy(predictions, y) / batches_per_epoch
if test_acc > max_accuracy:
max_accuracy = test_acc
print(
"step %d/%d: training loss: %.3f test loss: %.3f accuracy: %.1f%%"
% (step, FLAGS.max_steps, training_loss, test_loss,
test_acc * 100))
time_taken = time.perf_counter() - start_time
print("Done. Scored %.1f%% in %.1f seconds." %
(max_accuracy * 100, time_taken))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
cifar10 = cifar10_utils.get_cifar10("cifar10/cifar-10-batches-py")
# x_train, y_train = cifar10['train'].next_batch(32)
# print(x_train.shape)
use_cuda = torch.cuda.is_available()
if use_cuda:
print('Running on GPU')
device = torch.device('cuda' if use_cuda else 'cpu')
dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
conv_net = ConvNet(n_channels=3, n_classes=10).to(device)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(conv_net.parameters(), lr=FLAGS.learning_rate)
loss_list = []
for step in range(FLAGS.max_steps):
# Get batch and reshape input to vector
x_train, y_train = cifar10['train'].next_batch(FLAGS.batch_size)
# x_train = np.reshape(x_train, (batch_size,-1))
print("ok")
net_output = conv_net.forward(torch.from_numpy(x_train).type(dtype))
# print(net_output.shape)
# print("batch accuracy : "+str(accuracy(net_output.cpu().detach().numpy(),y_train)))
y_train = torch.from_numpy(y_train).type(dtype)
loss = criterion(net_output, torch.max(y_train, 1)[1])
loss_list.append(loss.cpu().detach().numpy())
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (step + 1) % FLAGS.eval_freq == 0:
# print("in test")
net_test_output_list = []
for i in range(100):
x_test, y_test = cifar10['test'].next_batch(100)
# y_test = torch.from_numpy(y_test).type(dtype)
# print(x_test.shape)
# x_test = np.reshape(x_test, (x_test.shape[0],-1))
net_test_output = conv_net.forward(
torch.from_numpy(x_test).type(dtype))
net_test_output_list.append(
accuracy(net_test_output.cpu().detach().numpy(), y_test))
print("test set accuracy for step " + str(step + 1) + " : " +
str(sum(net_test_output_list) / len(net_test_output_list)))
print("loss : ", sum(loss_list) / len(loss_list))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# ## Prepare all functions
# # Get number of units in each hidden layer specified in the string such as 100,100
# if FLAGS.dnn_hidden_units:
# dnn_hidden_units = FLAGS.dnn_hidden_units.split(",")
# dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units]
# else:
# dnn_hidden_units = []
# Set path to data
data_dir = FLAGS.data_dir
data = cifar10_utils.get_cifar10(data_dir)
# Prepare the test set
input_dims_test = data['test'].images.shape
height = input_dims_test[2]
width = input_dims_test[3]
channels = input_dims_test[1]
# num_images_test = input_dims_test[0]
# image_dims_ravel = height * width * channels
X_test = data["test"].images
Y_test = data["test"].labels
# Make acceptable input for test
# X_test = X_test.reshape((num_images_test, image_dims_ravel))
# make usable by pytorch
X_test = torch.tensor(X_test, requires_grad=False).type(dtype).to(device)
Y_test = torch.tensor(Y_test, requires_grad=False).type(dtype).to(device)
# Determine the channels
n_channels = channels
model = ConvNet(n_channels=n_channels, n_classes=10)
accuracy_train_log = list()
accuracy_test_log = list()
loss_train_log = list()
loss_test_log = list()
# FLAGS hold command line arguments
batch_size = FLAGS.batch_size
numb_iterations = FLAGS.max_steps
learning_rate = FLAGS.learning_rate
evaluation_freq = FLAGS.eval_freq
logging.info(f"learning rate: %2d " % learning_rate)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
for step in range(numb_iterations):
X_batch, Y_batch = data['train'].next_batch(batch_size)
# X_batch = X_batch.reshape((batch_size, image_dims_ravel))
# Convert to tensors which are handled by the device
X_batch = torch.from_numpy(X_batch).type(dtype).to(device)
Y_batch = torch.from_numpy(Y_batch).type(dtype).to(device)
# why do we need this again?
optimizer.zero_grad()
targs = Y_batch.argmax(dim=1)
outputs = model.forward(X_batch)
loss_current = criterion(outputs, targs)
loss_current.backward()
optimizer.step()
running_loss = loss_current.item()
if step % evaluation_freq == 0:
loss_train_log.append(running_loss)
accuracy_train_log.append(accuracy(outputs, Y_batch))
logging.info(f"train performance: loss = %4f, accuracy = %4f ",
loss_train_log[-1], accuracy_train_log[-1])
# Get performance on the test set
targs_test = Y_test.argmax(dim=1)
outputs = model(X_test)
loss_test_log.append(criterion(outputs, targs_test))
accuracy_test_log.append(accuracy(outputs, Y_test))
logging.info(f"test performance: loss = %4f , accuracy = %4f\n",
loss_test_log[-1], accuracy_test_log[-1])
# TODO: implement early stopping ?
list_acc = list()
list_loss = list()
for i in range(0, 10):
selection = random.sample(range(1, 1000), 32)
targs_test = Y_test[selection].argmax(dim=1)
outputs_test = model(X_test[selection])
loss_current_test = criterion(outputs_test,
targs_test).detach().item()
acc_current_test = accuracy(outputs_test, Y_test[selection])
list_loss.append(loss_current_test)
list_acc.append(acc_current_test)
path = "./mlp_results_pytorch/"
date_time = datetime.now().replace(
second=0, microsecond=0).strftime(format="%Y-%m-%d-%H-%M")
np.save(os.path.join(path, date_time + "accuracy_test"), accuracy_test_log)
np.save(os.path.join(path, date_time + "loss_test"), loss_test_log)
np.save(os.path.join(path, date_time + "loss_train"), loss_train_log)
np.save(os.path.join(path, date_time + "accuracy_train"),
accuracy_train_log)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
dataset_dict = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_loader = dataset_dict['train']
test_loader = dataset_dict['test']
test_images = Variable(torch.tensor(test_loader.images))
test_labels = torch.tensor(test_loader.labels)
model = ConvNet(n_channels=3, n_classes=10).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
criterion = torch.nn.CrossEntropyLoss()
test_accs = []
train_accs = []
losses = []
for epoch in range(FLAGS.max_steps):
model.train()
batch_x, batch_y = train_loader.next_batch(FLAGS.batch_size)
batch_x, batch_y = Variable(
torch.tensor(batch_x).to(device)), Variable(
torch.tensor(batch_y).to(device))
optimizer.zero_grad()
out = model.forward(batch_x)
loss = criterion(out, batch_y.max(1)[1])
# l2_reg = torch.tensor(0).float().to(device)
# for param in model.parameters():
# l2_reg += torch.norm(param, 2)
# loss = torch.add(loss, l2_reg/100)
# l1_reg = torch.tensor(0).float().to(device)
# for param in model.parameters():
# l1_reg += torch.norm(param, 1)
# loss = torch.add(loss, l1_reg / 100000)
losses.append(round(float(loss), 3))
loss.backward()
optimizer.step()
if epoch % FLAGS.eval_freq == 0:
with torch.no_grad():
model.eval()
# print accuracy on test and train set
train_acc = accuracy(out, batch_y)
#COMPUTE TEST ACCURACY
all_predictions = torch.tensor([]).float().to(device)
for i in range(FLAGS.batch_size, test_images.shape[0],
FLAGS.batch_size):
out = model.forward(
test_images[i - FLAGS.batch_size:i].to(device))
all_predictions = torch.cat((all_predictions, out))
if i < test_images.shape[0]:
out = model.forward(test_images[i:].to(device))
all_predictions = torch.cat((all_predictions, out))
test_acc = accuracy(all_predictions, test_labels.to(device))
print(
'Train Epoch: {}/{}\tLoss: {:.6f}\tTrain accuracy: {:.6f}\tTest accuracy: {:.6f}'
.format(epoch, FLAGS.max_steps, loss, train_acc, test_acc))
test_accs.append(float(test_acc))
train_accs.append(float(train_acc))
with torch.no_grad():
# COMPUTE TEST ACCURACY
all_predictions = torch.tensor([]).float().to(device)
for i in range(FLAGS.batch_size, test_images.shape[0],
FLAGS.batch_size):
out = model.forward(test_images[i - FLAGS.batch_size:i].to(device))
all_predictions = torch.cat((all_predictions, out))
if i < test_images.shape[0]:
out = model.forward(test_images[i:].to(device))
all_predictions = torch.cat((all_predictions, out))
test_acc = accuracy(all_predictions, test_labels.to(device))
print('FINAL Test accuracy: {:.6f}'.format(test_acc))
import matplotlib.pyplot as plt
plt.figure()
plt.plot([i for i in range(0, epoch + 1, EVAL_FREQ_DEFAULT)], train_accs)
plt.plot([i for i in range(0, epoch + 1, EVAL_FREQ_DEFAULT)], test_accs)
plt.legend(["train", "test"])
plt.ylabel("accuracy")
plt.xlabel("epoch")
plt.savefig("cnn_accuracy")
plt.figure()
plt.plot([i for i in range(0, epoch + 1)], losses)
plt.legend(["loss"])
plt.ylabel("loss")
plt.xlabel("epoch")
plt.savefig("cnn_loss")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# will be used to compute accuracy and loss for the whole train and test sets
batch_size_acc = 500
data_accuracy_loss = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
X_train_acc, y_train_acc = data_accuracy_loss['train'].images, data_accuracy_loss['train'].labels
X_test_acc, y_test_acc = data_accuracy_loss['test'].images, data_accuracy_loss['test'].labels
steps_train = int(X_train_acc.shape[0] / batch_size_acc)
steps_test = int(X_test_acc.shape[0] / batch_size_acc)
#
data = cifar10_utils.get_cifar10(data_dir = FLAGS.data_dir)
n_classes = data['train'].labels.shape[1]
n_inputs = data['train'].images.shape[1]*data['train'].images.shape[2]*data['train'].images.shape[3]
batch_size = FLAGS.batch_size
m_steps = FLAGS.max_steps
alpha = FLAGS.learning_rate
cnn = ConvNet(3, 10)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr=alpha)
x_ax = []
acc_train = []
acc_test = []
loss_train = []
loss_test = []
for step in range(m_steps):
x, y = data['train'].next_batch(batch_size)
n = x.shape
x = torch.from_numpy(x)
y_pred = cnn(x)
labels = torch.LongTensor(y)
loss = criterion(y_pred, torch.max(labels, 1)[1])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % FLAGS.eval_freq == 0:
print('Iteration ',step)
x_ax.append(step)
acc_ = []
loss_ = []
for i in range(steps_train):
x_acc = X_train_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
y_acc = y_train_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
x_acc = torch.from_numpy(x_acc)
y_acc = torch.LongTensor(y_acc)
y_pred = cnn.forward(x_acc)
acc_.append(accuracy(y_pred, y_acc))
loss_.append(float(criterion(y_pred, torch.max(y_acc, 1)[1])))
acc_train.append(np.mean(acc_))
loss_train.append(np.mean(loss_))
acc_ = []
loss_ = []
for i in range(steps_test):
x_acc = X_test_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
y_acc = y_test_acc[i * batch_size_acc:(i + 1) * batch_size_acc]
x_acc = torch.from_numpy(x_acc)
y_acc = torch.LongTensor(y_acc)
y_pred = cnn.forward(x_acc)
acc_.append(accuracy(y_pred, y_acc))
loss_.append(float(criterion(y_pred, torch.max(y_acc, 1)[1])))
acc_test.append(np.mean(acc_))
loss_test.append(np.mean(loss_))
print('Max train accuracy ', max(acc_train))
print('Max test accuracy ', max(acc_test))
print('Min train loss ', min(loss_train))
print('Min test loss ', min(loss_test))
x_ax = np.array(x_ax)
acc_test = np.array(acc_test)
acc_train = np.array(acc_train)
loss_test = np.array(loss_test)
loss_train = np.array(loss_train)
print('Max train accuracy ', max(acc_train))
print('Max test accuracy ', max(acc_test))
print('Min train loss ', min(loss_train))
print('Min test loss ', min(loss_test))
fig = plt.figure()
ax = plt.axes()
plt.title("CNN Pytorch. Accuracy curves")
ax.plot(x_ax, acc_train, label='train');
ax.plot(x_ax, acc_test, label='test');
ax.set_xlabel('Step');
ax.set_ylabel('Accuracy');
plt.legend();
plt.savefig('accuracy_cnn.jpg')
fig = plt.figure()
ax = plt.axes()
plt.title("CNN Pytorch. Loss curves")
ax.plot(x_ax, loss_train, label='train');
ax.plot(x_ax, loss_test, label='train');
ax.set_xlabel('Step');
ax.set_ylabel('Loss');
ax.set_ylim(top=10, bottom=0)
plt.legend();
plt.savefig('loss_cnn.jpg')
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
print(train.images.shape)
# transform = transforms.Compose(
# [transforms.Resize((224, 224)),
# transforms.ToTensor(),
# transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
if FLAGS.model == 'ALEX':
train.images = torch.Tensor([resize(img) for img in train.images])
test.images = torch.Tensor([resize(img) for img in test.images])
print('doneee')
model = models.alexnet(pretrained=True)
torch.save(model.state_dict(), 'alexnet.txt')
# model = torch.load('alexnet.txt')
for param in model.features.parameters():
param.requires_grad = False
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.classifier.parameters(),
lr=FLAGS.learning_rate)
else:
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(),
lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(2): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
if FLAGS.model == 'ADAM': x = resizer(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
if FLAGS.model == 'ADAM': batch_x = resizer(batch_x)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
def init_weights(m):
print(m)
if type(m) == nn.Linear:
m.weight.data.uniform_(0.0, 1.0)
print(m.weight)
m.bias.data.fill_(0.0)
print(m.bias)
lr = FLAGS.learning_rate
eval_freq = FLAGS.eval_freq
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
input_size = 32 * 32 * 3
output_size = 10
# load dataset
raw_data = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_data = raw_data['train']
validation_data = raw_data["validation"]
test_data = raw_data['test']
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = ConvNet(n_channels=3, n_classes=10).to(device)
print(model.layers)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
loss_target = nn.CrossEntropyLoss()
csv_data = [[
'step', 'train_loss', 'test_loss', 'train_accuracy', 'test_accuracy'
]]
print("initial weights as normal distribution and bias as zeros")
# model.layers.apply(init_weights)
for step in range(max_steps):
x, y = train_data.next_batch(batch_size)
# x = x.reshape(batch_size, input_size)
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
# train
# x = Variable(torch.from_numpy(x))
output = model.forward(x)
loss = loss_target.forward(output, y.argmax(dim=1))
# somehow we need to divide the loss by the output size to get the same loss
loss_avg = loss.item()
# model.zero_grad()
optimizer.zero_grad()
loss.backward()
# only need to update weights for linear module for each step
optimizer.step()
# with torch.no_grad():
# for param in model.parameters():
# param.data -= lr * param.grad
train_acc = accuracy(output, y)
# with the \r and end = '' trick, we can print on the same line
print('\r[{}/{}] train_loss: {} train_accuracy: {}'.format(
step + 1, max_steps, round(loss_avg, 3), round(train_acc, 3)),
end='')
# evaluate
if step % eval_freq == 0 or step >= (max_steps - 1):
test_batch_acc = test_loss_batch = 0
test_batch_size = 100
total_test_samples = test_data.num_examples
batch_num = math.ceil(total_test_samples / test_batch_size)
for i in range(batch_num):
x, y = test_data.next_batch(test_batch_size)
# x = x.reshape(test_data.num_examples, input_size)
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
output = model.forward(x)
test_loss_batch += loss_target.forward(output,
y.argmax(dim=1)).item()
test_batch_acc += accuracy(output, y)
test_loss = test_loss_batch / batch_num
test_acc = test_batch_acc / batch_num
csv_data.append([step, loss_avg, test_loss, train_acc, test_acc])
print(' test_loss: {}, test_accuracy: {}'.format(
round(test_loss, 3), round(test_acc, 3)))
with open(
'results/train_summary_torchconvnet_{}.csv'.format(int(
time.time())), 'w') as csv_file:
writer = csv.writer(csv_file)
writer.writerows(csv_data)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_list = []
batch_list = []
accuracy_list = []
batch_list_test = []
use_cuda = torch.cuda.is_available()
if use_cuda:
print('Running in GPU model')
device = torch.device('cuda' if use_cuda else 'cpu')
dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
for iter in range(313):
x, y = cifar10['test'].next_batch(FLAGS.batch_size)
y = np.argmax(y, axis=1)
batch_list_test.append((x, y))
for iter in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
y = np.argmax(y, axis=1)
batch_list.append((x, y))
print('Batch list completed')
in_features = 3
out_features = 10 #num_classes
net = ConvNet(in_features, out_features).to(device)
lossfunc = nn.CrossEntropyLoss()
optimiser = optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
net.train()
for i in range(FLAGS.max_steps):
inputs, labels = batch_list[i]
# inputs = torch.from_numpy(inputs)
inputs = torch.tensor(inputs)
labels = torch.from_numpy(labels).long()
inputs, labels = inputs.to(device), labels.to(device)
optimiser.zero_grad()
outputs = net.forward(inputs.float())
loss = lossfunc(outputs, labels)
loss_list.append(loss)
loss.backward()
optimiser.step()
if (i + 1) % FLAGS.eval_freq == 0:
net.eval()
total = 0
for data in batch_list_test:
x_test, y_test = data
x_test = torch.from_numpy(x_test).type(dtype).to(device)
y_test = torch.from_numpy(y_test).long().to(device)
predicted = net.forward(x_test)
acc = accuracy(predicted, y_test)
total += acc
accuracy_list.append(total / len(batch_list_test))
print('Accuracy on test set at step {} is {}'.format(
i, total / len(batch_list_test)))
print('Loss of training is {}'.format(loss.item()))
plt.subplot(2, 1, 1)
plt.plot(
np.arange(len(accuracy_list) * FLAGS.eval_freq, step=FLAGS.eval_freq),
accuracy_list, 'o-')
plt.xlabel('Step')
plt.ylabel('Accuracy')
#
plt.subplot(2, 1, 2)
plt.plot(np.arange(len(loss_list)), loss_list)
plt.xlabel('Step')
plt.ylabel('Loss')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# print(FLAGS.batch_size)
# print(FLAGS.eval_freq)
# print(FLAGS.learning_rate)
# print(FLAGS.max_steps)
cifar10 = cifar10_utils.get_cifar10()
if torch.cuda.is_available():
# print(torch.device('cpu'), torch.device("cuda"))
device = torch.device("cuda")
else:
device = torch.device("cpu")
network = ConvNet(3, 10)
network.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(network.parameters(), lr=FLAGS.learning_rate)
plotting_accuracy = []
plotting_loss = []
plotting_accuracy_test = []
plotting_loss_test = []
for i in range(1, FLAGS.max_steps - 1):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
x = x.to(device)
y = y.to(device)
out = network.forward(x)
loss = criterion(out, y.argmax(dim=1))
# print("Batch: {} Loss {}".format(i, loss))
acc = accuracy(out, y)
# print("Accuracy: {}".format(acc))
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (i % FLAGS.eval_freq == 0):
x, y = cifar10['test'].next_batch(300)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
x = x.to(device)
y = y.to(device)
out = network.forward(x)
loss_test = criterion(out, y.argmax(dim=1))
print("TEST Batch: {} Loss {}".format(i, loss_test))
acc_test = accuracy(out, y)
print("TEST Accuracy: {}".format(acc_test))
plotting_accuracy_test.append(acc_test)
plotting_loss_test.append(loss_test.item())
plotting_accuracy.append(acc)
plotting_loss.append(loss.item())
plt.plot(plotting_accuracy, label='train accuracy')
plt.plot(plotting_accuracy_test, label='test accuracy')
# plt.plot(plotting_loss, label='train loss')
# plt.plot(plotting_loss_test, label='test loss')
plt.legend()
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
model = ConvNet(3, 10)
print(model)
cv_size = 10000
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py',
validation_size=cv_size)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
log = defaultdict(list)
for step in range(FLAGS.max_steps):
optimizer.zero_grad()
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
h = model.forward(x)
loss = criterion(h, y.argmax(1))
loss.backward()
optimizer.step()
if step % FLAGS.eval_freq == 0:
log['train_loss'].append(loss.item())
log['train_acc'].append(accuracy(h, y))
model.eval()
x, y = cifar10['validation'].next_batch(cv_size)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
h = model.forward(x)
loss = criterion(h, y.argmax(1))
log['cv_loss'].append(loss.item())
log['cv_acc'].append(accuracy(h, y))
model.train()
print(
f"Step {step} | "
f"Training loss: {log['train_loss'][-1]:.5f}, "
f"accuracy: {100 * log['train_acc'][-1]:.1f}% | "
f"CV loss: {log['cv_loss'][-1]:.5f}, "
f"accuracy: {100 * log['cv_acc'][-1]:.1f}%")
model.eval()
x, y = cifar10['test'].next_batch(cifar10['test'].num_examples)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
h = model.forward(x)
loss = criterion(h, y.argmax(1))
print(f"Test loss: {loss.item()}, accuracy: {100 * accuracy(h, y):.1f}%")
# Plot loss and accuracy.
plt.subplot(121)
plt.title("Loss")
plt.plot(log['train_loss'], label="Training")
plt.plot(log['cv_loss'], label="Cross Validation")
plt.xlabel("Step")
plt.legend()
plt.subplot(122)
plt.title("Accuracy")
plt.plot(log['train_acc'], label="Training")
plt.plot(log['cv_acc'], label="Cross Validation")
plt.xlabel("Step")
plt.legend()
plt.legend()
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# Preparation for training
print('- Init parameters')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
train_data = data['train']
test_data = data['test']
w, h, d = train_data.images[0].shape
n_classes = train_data.labels[0].shape[0]
criterion = nn.CrossEntropyLoss()
model = ConvNet(w * h * d, n_classes).to(device)
optimizer = torch.optim.Adam(model.parameters())
train_losses = []
test_losses = []
test_accuracies = []
# Train
print('- Start Training')
for step in range(FLAGS.max_steps):
x_batch, x_labels = next_batch_in_tensors(train_data, FLAGS.batch_size,
device)
optimizer.zero_grad()
out = model.forward(x_batch)
loss = criterion(out, x_labels.argmax(dim=1))
loss.backward()
optimizer.step()
train_losses.append(loss.data[0].item())
if (step % FLAGS.eval_freq == 0) or (step == FLAGS.max_steps - 1):
print(' - step: {}'.format(step))
# Test current
test_x, test_labels = next_batch_in_tensors(
test_data, FLAGS.batch_size, device)
out_test = model(test_x)
loss_test = criterion(out_test, test_labels.argmax(dim=1))
acc = accuracy(out_test, test_labels)
test_losses.append(loss_test.data[0].item())
test_accuracies.append(acc.item())
# Save stuff
filepath = '../models/cnn/'
if not os.path.exists(filepath):
os.makedirs(filepath)
torch.save(model, '{}model.pt'.format(filepath))
with open('{}train_loss'.format(filepath), 'wb+') as f:
pickle.dump(train_losses, f)
with open('{}test_loss'.format(filepath), 'wb+') as f:
pickle.dump(test_losses, f)
with open('{}accuracies'.format(filepath), 'wb+') as f:
pickle.dump(test_accuracies, f)
print(test_accuracies[-1])
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
# set cuda
cuda = False
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# import the test data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_function = torch.nn.CrossEntropyLoss()
neuralnet = ConvNet(3, 10)
if cuda:
neuralnet.cuda() # run on GPU
sgd_back = torch.optim.Adam(neuralnet.parameters(), lr=FLAGS.learning_rate)
#lists with losses and accuracies
train_losses = []
train_accs = []
test_losses = []
test_accs = []
graph_x = []
# for i in range(50):
for i in range(FLAGS.max_steps):
neuralnet.train()
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
if cuda:
x = torch.from_numpy(x).cuda()
y = torch.from_numpy(y).cuda()
else:
x = torch.from_numpy(x)
y = torch.from_numpy(y)
# predict on the train data and calculate the gradients
out = neuralnet.forward(x) #output after the softmax
train_loss = loss_function(out, y.argmax(dim=1))
sgd_back.zero_grad()
train_loss.backward()
sgd_back.step()
# save the losses for every eval_freqth loop
if i % FLAGS.eval_freq == 0 or i == (FLAGS.max_steps - 1):
# if i % 1 == 0 or i == (FLAGS.max_steps - 1):
neuralnet.eval()
with torch.no_grad():
test_x, test_y = cifar10['test'].next_batch(1000)
if cuda:
torch.cuda.empty_cache()
test_x = torch.from_numpy(test_x).cuda()
test_y = torch.from_numpy(test_y).cuda()
else:
test_x = torch.from_numpy(test_x)
test_y = torch.from_numpy(test_y)
train_out = neuralnet.forward(x) # output after the softmax
test_out = neuralnet.forward(test_x)
if cuda:
torch.cuda.empty_cache()
train_loss = loss_function(train_out, y.argmax(dim=1))
test_loss = loss_function(test_out, test_y.argmax(dim=1))
train_acc = accuracy(out, y)
test_acc = accuracy(test_out, test_y)
train_losses.append(train_loss)
train_accs.append(train_acc)
test_losses.append(test_loss)
test_accs.append(test_acc)
graph_x.append(i)
print("iteration:", i)
print("Test accuracy:", test_accs[-1])
print("Test loss:", test_losses[-1])
plt.figure()
plt.subplot(1, 2, 1)
plt.plot(graph_x, train_losses, label="train loss")
plt.plot(graph_x, test_losses, label="test loss")
plt.title('Losses')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(graph_x, train_accs, label="train acc")
plt.plot(graph_x, test_accs, label="test acc")
plt.title('Accuracies')
plt.legend()
print("Final test accuracy:", test_accs[-1])
print("Final test loss:", test_losses[-1])
plt.savefig("conv_acc_and_loss.png")
with open('conv_acc_and_loss.txt', 'w') as f:
f.write("test_losses \n")
f.write(str(test_losses) + "\n \n")
f.write("test accs \n")
f.write(str(test_accs))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
print()
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train = cifar10['train']
input_channels = 3
output_class = 10
net = ConvNet(input_channels, output_class)
net.to(device)
if OPTIMIZER_DEFAULT == 'ADAM':
optimizer = torch.optim.Adam(net.parameters(),lr=FLAGS.learning_rate)
CrossEntropyLoss = nn.CrossEntropyLoss()
loss_iter = []
loss_mean = []
acc_iter = []
loss_sum = 0.0
for iter_n in np.arange(0,FLAGS.max_steps):
x, labels = train.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
labels = np.argwhere(labels>0)
labels = torch.from_numpy(labels[:,1]).to(device)
optimizer.zero_grad()
net.train()
predictions = net.forward(x)
loss = CrossEntropyLoss(predictions,labels.long())
loss_iter.append(loss.item())
loss_sum += loss.item()
loss_mean.append(np.mean(loss_iter[:-50:-1]))
loss.backward()
optimizer.step()
print("Iter: {}, training Loss: {}".format(iter_n, "%.3f"%loss.item()))
if (iter_n+1) % int(FLAGS.eval_freq) == 0:
print("....Testing.....\n")
test = cifar10['test']
acc = []
with torch.no_grad():
for _ in np.arange(0,(test.num_examples//FLAGS.batch_size)):
x, t = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
t = torch.tensor(t).to(device)
net.eval()
y = net.forward(x)
acc.append(accuracy(y,t))
acc_iter.append(np.mean(acc))
print("Testing accuracy: {}".format("%.3f"%np.mean(acc)))
print()
plt.plot(loss_iter,'r-',alpha=0.1, label="Batch loss")
plt.plot(loss_mean,'b-', label="Average loss")
plt.legend()
plt.xlabel("Iterations")
plt.ylabel("Loss")
plt.title("Training Loss")
plt.grid(True)
plt.show()
plt.close()
plt.plot(acc_iter,'g-')
plt.xlabel("Iterations")
plt.ylabel("Accuracy")
plt.grid(True)
plt.title("Test Accuracy")
plt.show()
plt.close()
print()
print("COMPLETED")
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(FLAGS.max_steps): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
# out = model.forward(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])
# plt.show()
plt.savefig('/home/lgpu0376/code/output_dir/loss_acc_graph.png')
# plt.savefig('loss_acc_graph.png')
with open('/home/lgpu0376/code/output_dir/output.out', 'w+') as f:
f.write(f'Final loss: {loss_history[-1]}')
f.write(f'\nFinal acc: {acc_history[-1]}')
def train(_run):
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
def get_xy_tensors(batch):
x, y = batch
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
return x, y
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
datasets = cifar10_utils.read_data_sets(DATA_DIR_DEFAULT, one_hot=False)
train_data = datasets['train']
test_data = datasets['test']
model = ConvNet(n_channels=3, n_classes=10).to(device)
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
log_every = 100
avg_loss = 0
avg_acc = 0
for step in range(FLAGS.max_steps):
x, y = get_xy_tensors(train_data.next_batch(FLAGS.batch_size))
# Forward and backward passes
optimizer.zero_grad()
out = model.forward(x)
loss = loss_fn(out, y)
loss.backward()
# Parameter updates
optimizer.step()
avg_loss += loss.item() / log_every
avg_acc += accuracy(out, y) / log_every
if step % log_every == 0:
print('[{}/{}] train loss: {:.6f} train acc: {:.6f}'.format(
step, FLAGS.max_steps, avg_loss, avg_acc))
_run.log_scalar('train-loss', avg_loss, step)
_run.log_scalar('train-acc', avg_acc, step)
avg_loss = 0
avg_acc = 0
# Evaluate
if step % FLAGS.eval_freq == 0 or step == (FLAGS.max_steps - 1):
n_test_iters = int(
np.ceil(test_data.num_examples / TEST_BATCH_SIZE))
test_loss = 0
test_acc = 0
for i in range(n_test_iters):
x, y = get_xy_tensors(test_data.next_batch(TEST_BATCH_SIZE))
model.eval()
out = model.forward(x)
model.train()
test_loss += loss_fn(out, y).item() / n_test_iters
test_acc += accuracy(out, y) / n_test_iters
print('[{}/{}] test accuracy: {:6f}'.format(
step, FLAGS.max_steps, test_acc))
_run.log_scalar('test-loss', test_loss, step)
_run.log_scalar('test-acc', test_acc, step)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
cifar10 = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train_data = cifar10['train']
n_channels = train_data.images.shape[0]
n_classes = train_data.labels.shape[1]
net = ConvNet(n_channels, n_classes)
net.to(device)
params = net.parameters()
optimizer = torch.optim.Adam(params, lr=FLAGS.learning_rate)
criterion = torch.nn.CrossEntropyLoss()
rloss = 0
train_acc_plot = []
test_acc_plot = []
loss_train = []
loss_test = []
print(f'[DEBUG] start training.... Max steps {FLAGS.max_steps}')
for i in range(0, FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x, y = torch.from_numpy(x).float().to(device), torch.from_numpy(
y).float().to(device)
out = net.forward(x)
loss = criterion(out, y.argmax(1))
optimizer.zero_grad()
loss.backward()
optimizer.step()
rloss += loss.item()
if i % FLAGS.eval_freq == 0:
train_accuracy = accuracy(out, y)
with torch.no_grad():
test_accuracys, test_losses = [], []
for j in range(0, FLAGS.max_steps):
test_x, test_y = cifar10['test'].next_batch(
FLAGS.batch_size)
test_x, test_y = torch.from_numpy(test_x).float().to(
device), torch.from_numpy(test_y).float().to(device)
test_out = net.forward(test_x)
test_loss = criterion(test_out, test_y.argmax(1))
test_accuracy = accuracy(test_out, test_y)
if device == 'cpu':
test_losses.append(test_loss)
else:
test_losses.append(test_loss.cpu().data.numpy())
test_accuracys.append(test_accuracy)
t_acc = np.array(test_accuracys).mean()
t_loss = np.array(test_losses).mean()
train_acc_plot.append(train_accuracy)
test_acc_plot.append(t_acc)
loss_train.append(rloss / (i + 1))
loss_test.append(t_loss)
print(
f"iter {i}, train_loss_avg {rloss/(i + 1)}, test_loss_avg {t_loss}, train_acc {train_accuracy}, test_acc_avg {t_acc}"
)
print('[DEBUG] Done training')
if FLAGS.plot:
print('[DEBUG] Start plotting...')
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
ax1.plot(np.arange(len(train_acc_plot)),
train_acc_plot,
label='training')
ax1.plot(np.arange(len(test_acc_plot)), test_acc_plot, label='testing')
ax1.set_title('Training evaluation with batch size ' +
str(FLAGS.batch_size) + '\n learning rate ' +
str(FLAGS.learning_rate) + '\n best accuracy ' +
str(max(test_acc_plot)))
ax1.set_ylabel('Accuracy')
ax1.legend()
ax2.plot(np.arange(len(loss_train)), loss_train, label='Train Loss')
ax2.plot(np.arange(len(loss_test)), loss_test, label='Test Loss')
ax2.set_title('Loss evaluation')
ax2.set_ylabel('Loss')
ax2.legend()
plt.xlabel('Iteration')
plt.savefig('convnet.png')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# get device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# loop through data
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py',
validation_size=2000)
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
# get channels
n_channels = np.size(x, 1)
# create model
net = ConvNet(n_channels, 10)
net.to(device)
# get loss function and optimizer
crossEntropy = nn.CrossEntropyLoss()
# keep track of loss and accuracy
loss_list = []
loss_val_list = []
accuracy_train_list = []
accuracy_val_list = []
# set optimizer
optimizer = torch.optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
# loop for the amount of steps
for i in range(FLAGS.max_steps):
# create torch compatible input
x = Variable(torch.from_numpy(x), requires_grad=True)
x = x.to(device)
# perform forward pass
out = net(x)
# convert one hot to indices and create torch compatible input
label_index = np.argmax(y, axis=1)
label_index = torch.LongTensor(label_index)
label_index = label_index.to(device)
# apply cross entropy
loss = crossEntropy(out, label_index)
# show progress and run network on validation set
if i % FLAGS.eval_freq == 0:
# convert output to numpy array, differs when using cuda compared to cpu
if torch.cuda.is_available():
train_out = out.cpu()
out_numpy = train_out.data[:].numpy()
else:
out_numpy = out.data[:].numpy()
# calculate accuracy
accuracy_train = accuracy(out_numpy, y)
# don't track the gradients
with torch.no_grad():
# load validation data
x_val, y_val = cifar10['validation'].next_batch(
FLAGS.batch_size)
# create torch compatible input
x_val = Variable(torch.from_numpy(x_val), requires_grad=False)
x_val = x_val.to(device)
# run on validation set
val_out = net.forward(x_val)
# convert one hot to indices and create torch compatible input
y_val_index = np.argmax(y_val, axis=1)
y_val_index = torch.LongTensor(y_val_index)
y_val_index = y_val_index.to(device)
# apply cross entropy
loss_val = crossEntropy.forward(val_out, y_val_index)
# convert output to numpy array, differs when using cuda compared to cpu
if torch.cuda.is_available():
val_out = val_out.cpu()
val_out_numpy = val_out.data[:].numpy()
loss_val = loss_val.cpu()
loss_val = loss_val.data.numpy()
loss_train = loss.cpu()
loss_train = loss_train.data.numpy()
else:
val_out_numpy = val_out.data[:].numpy()
loss_val = loss_val.data.numpy()
loss_train = loss.data.numpy()
accuracy_val = accuracy(val_out_numpy, y_val)
# save variables
accuracy_train_list.append(accuracy_train)
accuracy_val_list.append(accuracy_val)
loss_list.append(loss_train)
loss_val_list.append(loss_val)
# print progress
print(
"##############################################################"
)
print("Epoch ", i)
print(
"---------------------------------------------------------------"
)
print("The ACCURACY on the TRAIN set is currently: ",
accuracy_train)
print(
"---------------------------------------------------------------"
)
print("The ACCURACY on the VALIDATION set is currently:",
accuracy_val)
print(
"---------------------------------------------------------------"
)
print("The LOSS on the TRAIN set is currently:", loss_train)
print(
"---------------------------------------------------------------"
)
print("The LOSS on the VALIDATION set is currently:", loss_val)
print(
"---------------------------------------------------------------"
)
print(
"###############################################################"
)
print("\n")
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
# insert new databatch for next loop
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
# run test data through network without tracking the gradients
with torch.no_grad():
# test
x, y = cifar10['test'].images, cifar10['test'].labels
# convert variable to torch compatible input
x = Variable(torch.from_numpy(x), requires_grad=False)
x = x.to(device)
# get output
out = net(x)
# convert output to numpy array, differs when using cuda compared to cpu
if torch.cuda.is_available():
out = out.cpu()
out_numpy = out.data[:].numpy()
else:
out_numpy = out.data[:].numpy()
# calculate accuracy
test_accuracy = accuracy(out_numpy, y)
print("The accuracy on the test set is:")
print(test_accuracy)
# save test, training and validation accuracies and losses to make a plot afterwards
lists = [
accuracy_train_list, accuracy_val_list, loss_list, loss_val_list,
test_accuracy
]
pickle.dump(lists, open("lists.p", "wb"))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# initialize empty dictionaries
x, y, accu, loss = ({} for _ in range(4))
# retrieve data
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
# determine shapes
image_shape = data['test'].images[0].shape
nr_pixels = image_shape[0] * image_shape[1] * image_shape[2]
nr_labels = data['test'].labels.shape[1]
nr_test = data['test'].images.shape[0]
# set standards
tensor = torch.FloatTensor
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# save in variables
for tag in data:
nr_images = data[tag].images.shape[0]
x[tag] = torch.tensor(data[tag].images)
y[tag] = torch.tensor(data[tag].labels)
accu[tag] = []
loss[tag] = []
x['test'] = x['test'].type(tensor).to(device)
y['test'] = y['test'].type(tensor).to(device)
# create neural network
neural_network = ConvNet(nr_pixels, nr_labels).to(device)
cross_entropy = nn.CrossEntropyLoss().to(device)
parameter_optimizer = torch.optim.Adam(params=neural_network.parameters(), \
lr=FLAGS.learning_rate)
dx = 1
i = 0
logs = ['\n'.join([key + ' : ' + str(value)
for key, value in vars(FLAGS).items()])]
while i < FLAGS.max_steps and np.linalg.norm(dx) > 1e-5:
i += 1
# sample batch from data
rand_idx = np.random.randint(x['train'].shape[0], size=FLAGS.batch_size)
x_batch = x['train'][rand_idx].type(tensor).to(device)
y_batch = y['train'][rand_idx].type(tensor).to(device)
parameter_optimizer.zero_grad()
nn_out = neural_network.forward(x_batch)
ce_out = cross_entropy.forward(nn_out, y_batch.argmax(dim=1))
ce_out.backward()
parameter_optimizer.step()
if i % FLAGS.eval_freq == 0:
# save train accuracy and loss
accu['train'].append(accuracy(nn_out, y_batch))
loss['train'].append(ce_out)
# calculate and save test accuracy and loss
nn_out = neural_network.forward(x['test'])
ce_out = cross_entropy.forward(nn_out, y['test'].argmax(dim=1))
accu['test'].append(accuracy(nn_out, y['test']))
loss['test'].append(ce_out.item())
# show results in command prompt and save log
s = 'iteration ' + str(i) + ' | train acc/loss ' + \
str('{:.3f}'.format(accu['train'][-1])) + '/' + \
str('{:.3f}'.format(loss['train'][-1])) + ' | test acc/loss ' \
+ str('{:.3f}'.format(accu['test'][-1])) + '/' + \
str('{:.3f}'.format(loss['test'][-1]))
logs.append(s)
print(s)
#sys.stdout.write("\r%s" % s)
#sys.stdout.flush()
t = str(time.time())
# write logs
with open('results/logs_' + t + '.txt', 'w') as f:
f.writelines(['%s\n' % item for item in logs])
# write data to file
with open('results/data_' + t + '.txt', 'w') as f:
f.write('train accuracy')
f.writelines([',%s' % str(item) for item in accu['train']])
f.write('\ntrain loss')
f.writelines([',%s' % str(item) for item in loss['train']])
f.write('\ntest accuracy')
f.writelines([',%s' % str(item) for item in accu['test']])
f.write('\ntest loss')
f.writelines([',%s' % str(item) for item in loss['test']])
def train():
"""
Performs training and evaluation of ConvNet model.
"""
# DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# initialize tensorboard
run_id = datetime.now().strftime("%Y-%m-%d_%H-%M-%S_convnet")
log_dir = 'tensorboard/' + run_id
writer = SummaryWriter(log_dir=log_dir)
# get the dataset
data_set = cifar10_utils.get_cifar10(FLAGS.data_dir)
# get the necessary components
classifier = ConvNet(n_channels, n_classes).to(device)
loss_function = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(classifier.parameters(),
lr=FLAGS.learning_rate)
n_batches = {
'train': int(data_set['train']._num_examples / FLAGS.batch_size),
'validation':
int(data_set['validation']._num_examples / FLAGS.batch_size),
'test': int(data_set['test']._num_examples / FLAGS.batch_size)
}
# list of training accuracies and losses
train_accuracies = []
train_losses = []
# list of test accuracies and losses
test_accuracies = []
test_losses = []
epoch_test_accuracy = 0
epoch_test_loss = 0
# training loop
for step in range(FLAGS.max_steps):
# get current batch...
images, labels = data_set['train'].next_batch(FLAGS.batch_size)
# ...in the gpu
images = torch.from_numpy(images).type(dtype).to(device=device)
labels = torch.from_numpy(labels).type(dtype).to(device=device)
# forward pass
predictions = classifier.forward(images)
# compute loss
class_labels = labels.argmax(dim=1)
loss = loss_function(predictions, class_labels)
# reset gradients before backwards pass
optimizer.zero_grad()
# backward pass
loss.backward()
# update weights
optimizer.step()
# get accuracy and loss for the batch
train_accuracy = accuracy(predictions, labels)
train_accuracies.append(train_accuracy)
writer.add_scalar("Training accuracy vs steps", train_accuracy, step)
train_losses.append(loss.item())
writer.add_scalar("Training loss vs steps", loss.item(), step)
if ((step + 1) % 50) == 0 or step == 0:
print("\nStep", step + 1)
print("\tTRAIN:", round(train_accuracy * 100, 1), "%")
# run evaluation every eval_freq epochs
if (step + 1) % FLAGS.eval_freq == 0 or (step + 1) == FLAGS.max_steps:
# list of test batch accuracies and losses for this step
step_test_accuracies = []
step_test_losses = []
# get accuracy on the test set
for batch in range(n_batches['test']):
# get current batch...
images, labels = data_set['test'].next_batch(FLAGS.batch_size)
# ...in the gpu
images = torch.from_numpy(images).type(dtype).to(device=device)
labels = torch.from_numpy(labels).type(dtype).to(device=device)
# forward pass
predictions = classifier(images)
# compute loss
class_labels = labels.argmax(dim=1)
loss = loss_function(predictions, class_labels)
# get accuracy and loss for the batch
step_test_accuracies.append(accuracy(predictions, labels))
step_test_losses.append(loss.item())
# store accuracy and loss
epoch_test_accuracy = np.mean(step_test_accuracies)
test_accuracies.append(epoch_test_accuracy)
epoch_test_loss = np.mean(step_test_losses)
test_losses.append(epoch_test_loss)
print("\tTEST:", round(epoch_test_accuracy * 100, 1), "%")
writer.add_scalar("Test accuracy vs epochs", epoch_test_accuracy, step)
writer.add_scalar("Test loss vs epochs", epoch_test_loss, step)
# save results
results = {
'train_accuracies': train_accuracies,
'train_losses': train_losses,
'test_accuracies': test_accuracies,
'test_losses': test_losses,
'eval_freq': FLAGS.eval_freq
}
if not os.path.exists("results/"):
os.makedirs("results/")
with open("results/" + run_id + "_results.pkl", "wb") as file:
pkl.dump(results, file)
writer.close()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if FLAGS.data_dir:
DATA_DIR_DEFAULT = FLAGS.data_dir
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
batch_size = FLAGS.batch_size
learning_rate = FLAGS.learning_rate
cifar_data = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_data = cifar_data['train']
test_data = cifar_data['test']
input_channels = train_data.images.shape[1]
n_classes = train_data.labels.shape[1]
criterion = nn.CrossEntropyLoss()
model = ConvNet(input_channels, n_classes).to(device)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
# Train and Test losses
losses = [[], []]
# Train and Test accuracies
accuracies = [[], []]
# True iteration for plotting
iterations = []
for iteration in np.arange(FLAGS.max_steps):
x, y = train_data.next_batch(batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).type(torch.LongTensor).to(device)
train_output = model.forward(x)
loss = criterion(train_output, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if iteration % FLAGS.eval_freq == 0 or iteration == FLAGS.max_steps - 1:
iterations.append(iteration)
x_test, y_test = test_data.next_batch(10 * batch_size)
x_test = torch.from_numpy(x_test).to(device)
y_test = torch.from_numpy(np.argmax(y_test, axis=1)).type(torch.LongTensor).to(device)
# Second forward pass for test set
with torch.no_grad():
test_output = model.forward(x_test)
# Calculate losses
train_loss = criterion.forward(train_output, y)
losses[0].append(train_loss)
test_loss = criterion.forward(test_output, y_test)
losses[1].append(test_loss)
# Calculate accuracies
train_acc = accuracy(train_output, y)
test_acc = accuracy(test_output, y_test)
accuracies[0].append(train_acc)
accuracies[1].append(test_acc)
print("Iteration {}, Train loss: {}, Train accuracy: {}, Test accuracy: {}".format(iteration, train_loss,
train_acc, test_acc))
fig = plt.figure(figsize=(25, 10), dpi=200)
fig.suptitle('PyTorch ConvNet: Losses and Accuracies', fontsize=40)
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)
ax1.plot(iterations, losses[0], linewidth=4, color="g", label="Train loss")
ax1.plot(iterations, losses[1], linewidth=4, color="c", label="Test loss")
ax2.plot(iterations, accuracies[0], linewidth=4, color="g", label="Train accuracy")
ax2.plot(iterations, accuracies[1], linewidth=4, color="c", label="Test accuracy")
ax1.set_xlabel('$Iteration$', fontsize=28)
ax1.set_ylabel('$Loss$', fontsize=28)
ax2.set_xlabel('$Iteration$', fontsize=28)
ax2.set_ylabel('$Accuracy$', fontsize=28)
ax1.legend(fontsize=22)
ax2.legend(fontsize=22)
plt.savefig("../figures/pytorch_convnet.png")
plt.show()
with open('results_convnet.pkl', 'wb') as f:
pickle.dump([losses, accuracies], f)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# Get everything ready
data = cifar10_utils.get_cifar10()
n_classes = data['train'].labels.shape[1] # 10
n_channels = data['train'].images.shape[1] # 3
cnn = ConvNet(n_channels, n_classes)
loss_module = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr = FLAGS.learning_rate)
test_accuracies = []
train_losses = []
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
cnn.to(device)
# Iterate over the batches
for iteration in range(0, FLAGS.max_steps):
optimizer.zero_grad()
# Evaluate on whole test set
if (iteration%FLAGS.eval_freq == 0):
print("Iteration {}...".format(iteration))
epochs = data['test'].epochs_completed
batch_accuracies = []
while (epochs-data['test'].epochs_completed) == 0:
test_batch, test_batch_labels = data['test'].next_batch(FLAGS.batch_size)
test_probabilities = cnn.forward(torch.from_numpy(test_batch).to(device))
acc = accuracy(test_probabilities, torch.from_numpy(test_batch_labels).to(device))
batch_accuracies.append(acc.item())
test_accuracy = np.mean(batch_accuracies)
print("Test accuracy:", test_accuracy)
test_accuracies.append(test_accuracy)
# Train on batch
train_batch, train_batch_labels = data['train'].next_batch(FLAGS.batch_size)
train_probabilities = cnn.forward(torch.from_numpy(train_batch).to(device))
loss = loss_module(train_probabilities, torch.argmax(torch.from_numpy(train_batch_labels), dim=1).long().to(device))
if (iteration%FLAGS.eval_freq == 0):
train_losses.append(loss.item())
loss.backward()
optimizer.step()
# Plot results
x = range(0, len(test_accuracies)*FLAGS.eval_freq, FLAGS.eval_freq)
fig, ax = plt.subplots()
ax.plot(x, train_losses)
ax.set(xlabel='batches', ylabel='loss',
title='Loss training set after batches trained')
ax.grid()
fig.savefig("figures/cnn_loss_{0}_{1}_{2}.png".format(FLAGS.learning_rate, FLAGS.max_steps, FLAGS.batch_size))
# plt.show()
x = range(0, len(test_accuracies)*FLAGS.eval_freq, FLAGS.eval_freq)
fig, ax = plt.subplots()
ax.plot(x, test_accuracies)
ax.set(xlabel='batches', ylabel='accuracy',
title='Accuracy test set after batches trained')
ax.grid()
fig.savefig("figures/cnn_results_{0}_{1}_{2}.png".format(FLAGS.learning_rate, FLAGS.max_steps, FLAGS.batch_size))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
with open("jobs/status.txt", "w") as f:
f.write("Start training\n")
## Prepare all functions
output_dir = FLAGS.output_dir
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
learning_rate = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
optimizer = FLAGS.optimizer
# Obtain dataset
dataset = cifar10_utils.get_cifar10(data_dir)
n_channels = dataset['train'].images[0].shape[0]
n_classes = dataset['train'].labels[0].shape[0]
n_test = dataset['test'].images.shape[0]
# Initialise VGG network
dev = 'cuda' if torch.cuda.is_available() else 'cpu'
device = torch.device(dev)
print("Device: " + dev)
net = ConvNet(n_channels, n_classes).to(device)
loss_fn = F.cross_entropy
print("Network architecture:\n\t{}\nLoss module:\n\t{}".format(
str(net), str(loss_fn)))
# Evaluation vars
train_loss = []
gradient_norms = []
train_acc = []
test_acc = []
iteration = 0
# Training
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
while iteration < max_steps:
iteration += 1
# Sample a mini-batch
x, y = dataset['train'].next_batch(batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(y).argmax(dim=1).long().to(device)
# Forward propagation
prediction = net.forward(x)
loss = loss_fn(prediction, y)
acc = accuracy(prediction, y)
train_acc.append((iteration, acc))
train_loss.append((iteration, loss))
# Backprop
optimizer.zero_grad()
loss.backward()
# Weight update in linear modules
optimizer.step()
norm = 0
for params in net.parameters():
norm += params.reshape(-1).pow(2).sum()
gradient_norms.append((iteration, norm.reshape(-1)))
# Evaluation
with torch.no_grad():
if iteration % eval_freq == 0:
x = torch.from_numpy(dataset['test'].images).to(device)
y = torch.from_numpy(
dataset['test'].labels).argmax(dim=1).long().to(device)
prediction = net.forward(x)
acc = accuracy(prediction, y)
test_acc.append((iteration, acc))
print("Iteration: {}\t\tTest accuracy: {}".format(
iteration, acc))
# Save raw output
now = datetime.datetime.now()
time_stamp = "{}{}{}{}{}".format(now.year, now.month, now.day, now.hour,
now.minute)
net_name = "cnn"
if not os.path.isdir(os.path.join(output_dir, net_name, time_stamp)):
os.makedirs(os.path.join(output_dir, net_name, time_stamp))
metrics = {
"train_loss": train_loss,
"gradient_norms": gradient_norms,
"train_acc": train_acc,
"test_acc": test_acc
}
raw_data = {"net": net, "metrics": metrics}
pickle.dump(
raw_data,
open(os.path.join(output_dir, net_name, time_stamp, "raw_data"), "wb"))
# Save plots
# Loss
fig, ax = plt.subplots()
iter = [i for (i, q) in train_loss]
loss = [q for (i, q) in train_loss]
ax.plot(iter, loss)
ax.set(xlabel='Iteration',
ylabel='Loss (log)',
title='Batch training loss')
ax.set_yscale('log')
ax.grid()
fig.savefig(os.path.join(output_dir, net_name, time_stamp, "loss.png"))
# gradient norm
fig, ax = plt.subplots()
iter = [i for (i, q) in gradient_norms]
norm = [q for (i, q) in gradient_norms]
ax.plot(iter, norm)
ax.set(xlabel='Iteration', ylabel='Norm', title='Gradient norm')
ax.grid()
fig.savefig(
os.path.join(output_dir, net_name, time_stamp, "gradient_norm.png"))
# accuracies
fig, ax = plt.subplots()
iter = [i for (i, q) in train_acc]
accu = [q for (i, q) in train_acc]
ax.plot(iter, accu, label='Train')
iter = [i for (i, q) in test_acc]
accu = [q for (i, q) in test_acc]
ax.plot(iter, accu, label='Test')
ax.set(xlabel='Iteration',
ylabel='Accuracy',
title='Train and test accuracy')
ax.legend()
ax.grid()
fig.savefig(os.path.join(output_dir, net_name, time_stamp, "accuracy.png"))
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
lr = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
optim = FLAGS.optimizer
#fetch data
cifar10 = cifar10_utils.get_cifar10(data_dir)
n_classes = 10
n_channels = 3
eval_rounds = int(np.ceil(cifar10['test']._num_examples / batch_size))
model = ConvNet(n_channels, n_classes)
ce = torch.nn.CrossEntropyLoss()
pars = model.parameters()
# optimizer
optim_pars = {'params': pars, 'lr': lr, 'weight_decay': FLAGS.weight_decay}
if optim == 'adadelta':
optimizer = torch.optim.Adadelta(**optim_pars)
elif optim == 'adagrad':
optimizer = torch.optim.Adagrad(**optim_pars)
elif optim == 'rmsprop':
optimizer = torch.optim.RMSprop(**optim_pars)
elif optim == 'adam':
optimizer = torch.optim.Adam(**optim_pars)
else: # SGD
optimizer = torch.optim.SGD(**optim_pars)
model.to(device)
eval_i = 0
cols = ['train_acc', 'test_acc', 'train_loss', 'test_loss', 'secs']
# train
results = []
name = f'convnet-pytorch-{optim}'
with SummaryWriter(name) as w:
for step in tqdm(range(FLAGS.max_steps)):
optimizer.zero_grad()
X, y = cifar10['train'].next_batch(batch_size)
X = torch.tensor(X).type(dtype).to(device)
train_predictions = model.forward(X)
X.detach()
y = torch.tensor(y).type(dtype).to(device)
train_acc = accuracy(train_predictions, y)
idx_train = torch.argmax(y, dim=-1).long()
y.detach()
train_loss = ce(train_predictions, idx_train)
train_predictions.detach()
# stop if loss has converged!
check = 10
if len(results) >= 2 * check:
threshold = 1e-6
losses = [result['test_loss'] for result in results]
current = np.mean(losses[-check:])
prev = np.mean(losses[-2 * check:-check])
if (prev - current) < threshold:
break
# # at each epoch, we divide the learning rate by this if the dev accuracy decreases
# if dev_acc > prev_acc:
# lr /= learning_decay
# prev_acc = dev_acc
train_loss.backward()
optimizer.step()
# evaluate
if step % FLAGS.eval_freq == 0:
time = int(step / FLAGS.eval_freq)
start = timer()
test_accs = []
test_losses = []
for t in range(eval_rounds):
X, y = cifar10['test'].next_batch(batch_size)
X = torch.tensor(
X, requires_grad=False).type(dtype).to(device)
y = torch.tensor(
y, requires_grad=False).type(dtype).to(device)
test_predictions = model.forward(X)
X.detach()
test_accs.append(accuracy(test_predictions, y))
test_losses.append(
ce(test_predictions, y.argmax(dim=1)).item())
test_predictions.detach()
y.detach()
end = timer()
secs = end - start
test_acc = np.mean(test_accs)
test_loss = np.mean(test_losses)
vals = [train_acc, test_acc, train_loss, test_loss, secs]
stats = dict(
zip(cols, [
np.asscalar(i.detach().cpu().numpy().take(0))
if isinstance(i, torch.Tensor) else np.asscalar(i)
if isinstance(i, (np.ndarray, np.generic)) else i
for i in vals
]))
print(
yaml.dump({
k: round(i, 3) if isinstance(i, float) else i
for k, i in stats.items()
}))
w.add_scalars('metrics', stats, time)
results.append(stats)
df = pd.DataFrame(results, columns=cols)
meta = {
'framework': 'pytorch',
'algo': 'convnet',
'optimizer': optim,
'batch_size': FLAGS.batch_size,
'learning_rate': FLAGS.learning_rate,
'dnn_hidden_units': '',
'weight_decay': FLAGS.weight_decay,
'max_steps': FLAGS.max_steps,
}
for k, v in meta.items():
df[k] = v
output_file = 'results/results.csv' # f'{name}.csv'
if os.path.isfile(output_file):
df.to_csv(f'{name}.csv', header=False, mode='a')
else:
df.to_csv(f'{name}.csv', header=True, mode='w')
torch.save(model.state_dict(), f'{name}.pth')
print('done!')
return test_loss
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
if torch.cuda.is_available():
dev = "cuda:0"
else:
dev = "cpu"
device = torch.device(dev)
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
"""
Initialize data module
"""
cifar10=cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
x, y = cifar10['train'].next_batch(1)
x_test, y_test = cifar10['test'].next_batch(10000)
x_test = torch.tensor(x_test)
y_test = torch.tensor(y_test)
"""
initialize the network
"""
network = ConvNet(x.shape[1], y.shape[1]).to(device)
crossEntropy = nn.CrossEntropyLoss()
optimizer = None
optimizer = torch.optim.Adam(network.parameters(), lr=FLAGS.learning_rate, amsgrad=True)
store_loss = None
for i in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
y = torch.LongTensor(y).to(device)
prediction = network.forward(x)
loss = crossEntropy.forward(prediction, torch.max(y, 1)[1])
optimizer.zero_grad()
loss.backward()
optimizer.step()
store_loss = loss.cpu()
del loss
del x
del y
del prediction
prediction_collection = None
if i%FLAGS.eval_freq == 0:
with torch.no_grad():
print('Loss after '+str(i)+' steps '+str(store_loss))
for j in range(100):
test_data = x_test[j*100:j*100+100].to(device)
prediction = network.forward(test_data)
prediction = nn.functional.softmax(prediction)
del test_data
if j == 0:
prediction_collection = prediction
else:
prediction_collection = torch.cat((prediction_collection, prediction), 0)
del prediction
print('Accuracy after '+ str(i) +' steps ' + str(accuracy(prediction_collection, y_test)))
prediction_collection = None
with torch.no_grad():
print('final Loss',store_loss)
for j in range(100):
test_data = x_test[j*100:j*100+100].to(device)
prediction = network.forward(test_data).cpu()
prediction = nn.functional.softmax(prediction)
if j == 0:
prediction_collection = prediction
else:
prediction_collection = torch.cat((prediction_collection, prediction), 0)
del prediction
print('Final accuracy')
print(accuracy(prediction_collection, y_test))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on
the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
#all external parameters in a readable format.
lr = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
#fetch data
data = cifar10_utils.get_cifar10(data_dir)
n_classes = 10
n_channels = 3
#number of iterations to train the data in the whole dataset:
n_iter = 1 # int(np.ceil(data["train"]._num_examples/batch_size))
#number of evaluations
num_evals = int(np.ceil(data['test']._num_examples / batch_size))
#load model
cnn_model = ConvNet(n_channels, n_classes)
#Loss function
loss_XE = torch.nn.CrossEntropyLoss()
#keep track of how loss and accuracy evolves over time.
loss_train = np.zeros(max_steps + 1) #loss on training data
acc_train = np.zeros(max_steps + 1) #accuracy on training data
loss_eval = np.zeros(max_steps + 1) #loss on test data
acc_eval = np.zeros(max_steps + 1) #accuracy on test data
#Optimizer
optmizer = optim.Adam(cnn_model.parameters(), lr=lr)
#let's put some gpus to work!
cnn_model.to(device)
#index to keep track of the evaluations.
eval_i = 0
#Train shit
for s in range(max_steps):
for n in range(n_iter):
#fetch next batch of data
X, y = data['train'].next_batch(batch_size)
#use torch tensor + gpu
X = torch.from_numpy(X).type(dtype).to(device)
y = torch.from_numpy(y).type(dtype).to(device)
#reset gradient to zero before gradient descent.
optmizer.zero_grad()
#calculate loss
probs = cnn_model(X) #automatically calls .forward()
loss = loss_XE(probs, y.argmax(dim=1))
#backward propagation
loss.backward()
optmizer.step()
#stores the loss and accuracy of the trainind data for later analysis.
loss_train[eval_i] += loss.item() / num_evals #
acc_train[eval_i] += accuracy(probs, y) / num_evals
probs.detach()
if (s % eval_freq == 0) | (s == (max_steps - 1)):
#calculate accuracy for the whole data set
for t in range(num_evals):
#fetch all the data
X, y = data['test'].next_batch(batch_size)
#use torch tensor + gpu, no gradient needed.
X = torch.tensor(X, requires_grad=False).type(dtype).to(device)
y = torch.tensor(y, requires_grad=False).type(dtype).to(device)
#actually calculates loss and accuracy for the batch
probs = cnn_model.forward(X)
loss_eval[eval_i] += loss_XE(
probs,
y.argmax(dim=1)).item() # detach().data.cpu().item()
acc_eval[eval_i] += accuracy(probs, y)
probs.detach()
#frees memory
X.detach()
y.detach()
#average the losses and accuracies across test batches
loss_eval[eval_i] /= num_evals
acc_eval[eval_i] /= num_evals
#print performance
print(f"step {s} out of {max_steps}")
print(
f" loss: {loss_eval[eval_i]}, accuracy: {acc_eval[eval_i]}")
print(
f" loss: {loss_train[eval_i]}, accuracy: {acc_train[eval_i]}"
)
#save the results
# np.save("loss_eval", loss_eval)
# np.save("accuracy_eval", acc_eval)
#increments eval counter
eval_i += 1
#Save intermediary results for later analysis
print("saving results in folder...")
np.save("loss_train", loss_train)
np.save("accuracy_train", acc_train)
np.save("loss_eval", loss_eval)
np.save("accuracy_eval", acc_eval)
print("savign model")
torch.save(cnn_model.state_dict(), cnn_model.__class__.__name__ + ".pt")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
#load data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
#hyperparameters
eta = FLAGS.learning_rate
eps = 1e-6 # convergence criterion
max_steps = FLAGS.max_steps
b_size = FLAGS.batch_size
#test_data
x_test = cifar10["test"].images
y_test = cifar10["test"].labels
#get usefull dimensions
n_channels = np.size(x_test, 1)
n_classes = np.size(y_test, 1)
n_batches = np.size(x_test, 0) // b_size
#load whole train data ############################################################
x_train = cifar10["train"].images
x_train = torch.tensor(x_train, requires_grad=False).type(dtype).to(device)
n_train_batches = np.size(x_train, 0) // b_size
#initialize the ConvNet model
model = ConvNet(n_channels, n_classes)
get_loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=eta)
model.to(device)
train_loss = []
test_loss = []
train_acc = []
test_acc = []
for step in range(max_steps):
#get batch
x, y = cifar10['train'].next_batch(b_size)
x = torch.tensor(x).type(dtype).to(device)
y = torch.tensor(y).type(dtype).to(device)
#forward pass
pred = model.forward(x)
#get training loss
current_loss = get_loss(pred, y.argmax(dim=1))
optimizer.zero_grad()
#get training loss gradient
current_loss.backward()
#get training accuracy
current_train_acc = accuracy(pred, y)
optimizer.step()
#free memory up
pred.detach()
x.detach()
y.detach()
#select evaluation step
if (step % FLAGS.eval_freq) == 0:
# c_train_loss = current_loss.data.item()
# train_loss.append(c_train_loss)
# train_acc.append(current_train_acc)
c_train_loss = 0
current_train_acc = 0
c_test_loss = 0
current_test_acc = 0
#loop through train set in batches ######################################################
for test_batch in range(n_train_batches):
#load test data
x_train, y_train = cifar10['train'].next_batch(b_size)
x_train = torch.tensor(
x_train, requires_grad=False).type(dtype).to(device)
y_train = torch.tensor(
y_train, requires_grad=False).type(dtype).to(device)
#get test batch results
train_pred = model.forward(x_train)
current_train_loss = get_loss(train_pred,
y_train.argmax(dim=1))
c_train_loss += current_train_loss.data.item()
current_train_acc += accuracy(train_pred, y_train)
#free memory up
train_pred.detach()
x_train.detach()
y_train.detach()
#loop through test set in batches
for test_batch in range(n_batches):
#load test data
x_test, y_test = cifar10['test'].next_batch(b_size)
x_test = torch.tensor(
x_test, requires_grad=False).type(dtype).to(device)
y_test = torch.tensor(
y_test, requires_grad=False).type(dtype).to(device)
#get test batch results
test_pred = model.forward(x_test)
current_test_loss = get_loss(test_pred, y_test.argmax(dim=1))
c_test_loss += current_test_loss.data.item()
current_test_acc += accuracy(test_pred, y_test)
#free memory up
test_pred.detach()
x_test.detach()
y_test.detach()
#get full training set results #########################################################
c_train_loss = c_train_loss / n_train_batches
current_train_acc = current_train_acc / n_train_batches
train_loss.append(c_train_loss)
train_acc.append(current_train_acc)
#get full test set results
c_test_loss = c_test_loss / n_batches
current_test_acc = current_test_acc / n_batches
test_loss.append(c_test_loss)
test_acc.append(current_test_acc)
print('\nStep ', step, '\n------------\nTraining Loss = ',
round(c_train_loss, 4),
', Train Accuracy = ', current_train_acc, '\nTest Loss = ',
round(c_test_loss, 4), ', Test Accuracy = ',
round(current_test_acc, 4))
if step > 0 and abs(test_loss[(int(step / FLAGS.eval_freq))] -
test_loss[int(step / FLAGS.eval_freq) -
1]) < eps:
break
plot_graphs(train_loss,
'Training Loss',
'orange',
test_loss,
'Test Loss',
'blue',
title='Adams optimization',
ylabel='Loss',
xlabel='Steps')
plot_graphs(train_acc,
'Training Accuracy',
'darkorange',
test_acc,
'Test Accuracy',
'darkred',
title='Adams optimization',
ylabel='Accuracy',
xlabel='Steps')
#save results:
np.save('train_loss', train_loss)
np.save('train_acc', train_acc)
np.save('test_loss', test_loss)
np.save('test_acc', test_acc)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# raise NotImplementedError
loss_train = []
acc_train = []
acc_test = []
#Basic declarations
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
cifar10_set = cifar10_utils.get_cifar10(FLAGS.data_dir)
# get the number of outputs
_, n_outputs = cifar10_set['train']._labels.shape
#Declare model
cnn = ConvNet(3, n_outputs).to(device)
optimizer = torch.optim.Adam(cnn.parameters(), lr=FLAGS.learning_rate)
loss_funct = nn.CrossEntropyLoss()
#Loop through data to train
for i in range(0, FLAGS.max_steps + 1):
x, t = cifar10_set['train'].next_batch(FLAGS.batch_size)
x = torch.tensor(x, dtype=torch.float32).to(device)
y = cnn.forward(x)
loss = loss_funct(y, torch.LongTensor(np.argmax(t, 1)).to(device))
optimizer.zero_grad()
loss.backward()
optimizer.step()
#Evaluate and save for plotting
if i % FLAGS.eval_freq == 0:
tmp_test_acc = []
loss_train.append(loss)
acc_train.append(accuracy(y.cpu().detach().numpy(), t))
# x,t = cifar10_set['test'].images, cifar10_set['test'].labels
# x = torch.tensor(x, dtype=torch.float32).to(device)
for j in range(0, int(cifar10_set['test']._num_examples / 50)):
x, t = cifar10_set['train'].next_batch(50)
x = torch.tensor(x, dtype=torch.float32).to(device)
y = cnn.forward(x)
tmp_test_acc.append(accuracy(y.cpu().detach().numpy(), t))
acc_test.append(np.array(tmp_test_acc).mean())
print("The accuracy at step, " + str(i) + " is : " +
str(acc_test[-1]))
#Plotting the accuracy of test and train:
plt.figure(0)
plt.plot(np.arange(0,
len(acc_train) * FLAGS.eval_freq * FLAGS.batch_size,
FLAGS.eval_freq * FLAGS.batch_size) /
cifar10_set['train'].num_examples,
acc_train,
label='Train')
plt.plot(np.arange(0,
len(acc_train) * FLAGS.eval_freq * FLAGS.batch_size,
FLAGS.eval_freq * FLAGS.batch_size) /
cifar10_set['train'].num_examples,
acc_test,
label='Test')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Accuracy of Train and Test Set Through Training')
plt.legend()
plt.savefig('cnn_accuracy_correct.png')
# plt.show()
plt.figure(1)
plt.plot(np.arange(0,
len(loss_train) * FLAGS.eval_freq * FLAGS.batch_size,
FLAGS.eval_freq * FLAGS.batch_size) /
cifar10_set['train'].num_examples,
loss_train,
label='Train')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.title('Loss Through Training')
plt.savefig('cnn_loss_correct.png')
def train():
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda")
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
dataset = cifar10_utils.get_cifar10()
training = dataset['train']
test = dataset['test']
model = ConvNet(3, 10)
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
ce = torch.nn.CrossEntropyLoss()
test_accuracy = []
loss_list = []
for epoch in np.arange(0, FLAGS.max_steps):
x, y = training.next_batch(FLAGS.batch_size)
x = Variable(torch.tensor(x).to(device))
y = Variable(torch.tensor(y).to(device))
optimizer.zero_grad()
model.train()
yh = model.forward(x)
loss = ce(yh, torch.max(y, 1)[1])
loss_list.append(loss.item())
loss.backward()
optimizer.step()
if (epoch + 1) % int(FLAGS.eval_freq) == 0:
acc = []
with torch.no_grad():
for _ in np.arange(0, (test.num_examples // FLAGS.batch_size)):
x, y = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
y = torch.tensor(y).to(device)
model.eval()
yh = model.forward(x)
acc.append(accuracy(yh, y))
test_accuracy.append(np.mean(acc))
print(np.mean(acc))
import seaborn as sns
import matplotlib.pyplot as plt
f, axes = plt.subplots(1, 2)
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, EVAL_FREQ_DEFAULT),
test_accuracy,
ax=axes[0])
ax.set_title('Test accuracy')
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, 1),
loss_list,
ax=axes[1])
ax.set_title('Loss')
figure = ax.get_figure()
figure.savefig("cnn-pytorch-results")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
# use GPU if available
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
## Prepare all functions
lr = FLAGS.learning_rate
max_steps = FLAGS.max_steps
batch_size = FLAGS.batch_size
eval_freq = FLAGS.eval_freq
data_dir = FLAGS.data_dir
train_treshold = 1e-6 # if train loss below that threshold, training stops
# load input data
cifar10 = cifar10_utils.get_cifar10(data_dir, one_hot=True)
# get test data
x_test = cifar10["test"].images
y_test = cifar10["test"].labels
train_data = cifar10["train"]
# determine dimension of data
x_dim = x_test.shape
n_test_samples = x_dim[0] # number of test samples
# images of size 32 x 32 x 3
n_inputs = x_dim[1] * x_dim[2] * x_dim[3] # channels * height * width
n_classes = y_test.shape[1]
# reshape data to tensor representation
#x_test = x_test.reshape((n_test_samples, n_inputs))
x_test_torch = torch.tensor(x_test, dtype=torch.float, device=device)
y_test_torch = torch.tensor(y_test, dtype=torch.float, device=device)
# initialize ConvNet model
convnet_model = ConvNet(
n_channels=x_dim[1],
n_classes=n_classes
).to(device)
# define loss function
loss_fn = nn.CrossEntropyLoss()
# define optimizer
optimizer = torch.optim.Adam(convnet_model.parameters(), lr=lr)
# evaluation metrics
acc_train = []
acc_test = []
loss_train = []
loss_test = []
best_acc = 0.0
#results = []
# train the model
print("Start training")
for step in range(max_steps):
# get mini-batch
x_train, y_train = train_data.next_batch(batch_size)
#x_train = x_train.reshape((batch_size, n_inputs))
# transform to tensor representation
x_train_torch = torch.tensor(x_train, dtype=torch.float, device=device)
y_train_torch = torch.tensor(y_train, dtype=torch.float, device=device) # labels for mb training set
# set gradients to zero
optimizer.zero_grad()
# forward pass mb to get predictions as output
out = convnet_model.forward(x_train_torch)
# compute loss
loss_mb = loss_fn.forward(out, y_train_torch.argmax(dim=1))
# backward pass
loss_mb.backward()
optimizer.step()
# evaluate training and validation set (pretty much the same as with Numpy)
# perhaps modify learning rate?
if (step % eval_freq == 0) or (step == max_steps - 1):
print(f"Step: {step}")
# compute and store training metrics
loss_train.append(loss_mb.item())
acc_train.append(accuracy(out, y_train_torch))
print("TRAIN acc: {0:.4f} & loss: {1:.4f}".format(acc_train[-1], loss_train[-1]))
# compute and store test metrics
# Note that we use the test set as validation set!! Only as an exception :P
out_test = convnet_model.forward(x_test_torch)
loss_val = loss_fn.forward(out_test, y_test_torch.argmax(dim=1))
loss_test.append(loss_val.item())
acc_test.append(accuracy(out_test, y_test_torch))
print("TEST acc: {0:.4f} & loss: {1:.4f}".format(acc_test[-1], loss_test[-1]))
#results.append([step, acc_train[-1], loss_train[-1], acc_test[-1], loss_test[-1]])
if acc_test[-1] > best_acc:
best_acc = acc_test[-1]
print("New BEST acc: {0:.4f}".format(best_acc))
# Early stop when training loss below threshold?
if len(loss_train) > 20:
prev_losses = loss_test[-2]
cur_losses = loss_test[-1]
if (prev_losses - cur_losses) < train_treshold:
print("Training stopped early at step {0}".format(step + 1))
break
print("Finished training")
print("BEST acc: {0:.4f}".format(best_acc))
res_path = Path.cwd().parent / 'conv_pytorch_results'
if not res_path.exists():
res_path.mkdir(parents=True)
print("Saving results to {0}".format(res_path))
#Save an array to a binary file in NumPy .npy format.
#np.save(res_path / 'loss_train', loss_train)
#np.save(res_path / 'acc_train', acc_train)
#np.save(res_path / 'loss_test', loss_test)
#np.save(res_path / 'acc_test', acc_test)
#Save array to csv file
np.savetxt(res_path / 'loss_train.csv', loss_train, delimiter=',')
np.savetxt(res_path / 'acc_train.csv', acc_train, delimiter=',')
np.savetxt(res_path / 'loss_test.csv', loss_test, delimiter=',')
np.savetxt(res_path / 'acc_test.csv', acc_test, delimiter=',')