def train():
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda")
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
dataset = cifar10_utils.get_cifar10()
training = dataset['train']
test = dataset['test']
model = ConvNet(3, 10)
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
ce = torch.nn.CrossEntropyLoss()
test_accuracy = []
loss_list = []
for epoch in np.arange(0, FLAGS.max_steps):
x, y = training.next_batch(FLAGS.batch_size)
x = Variable(torch.tensor(x).to(device))
y = Variable(torch.tensor(y).to(device))
optimizer.zero_grad()
model.train()
yh = model.forward(x)
loss = ce(yh, torch.max(y, 1)[1])
loss_list.append(loss.item())
loss.backward()
optimizer.step()
if (epoch + 1) % int(FLAGS.eval_freq) == 0:
acc = []
with torch.no_grad():
for _ in np.arange(0, (test.num_examples // FLAGS.batch_size)):
x, y = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
y = torch.tensor(y).to(device)
model.eval()
yh = model.forward(x)
acc.append(accuracy(yh, y))
test_accuracy.append(np.mean(acc))
print(np.mean(acc))
import seaborn as sns
import matplotlib.pyplot as plt
f, axes = plt.subplots(1, 2)
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, EVAL_FREQ_DEFAULT),
test_accuracy,
ax=axes[0])
ax.set_title('Test accuracy')
ax = sns.lineplot(np.arange(0, MAX_STEPS_DEFAULT, 1),
loss_list,
ax=axes[1])
ax.set_title('Loss')
figure = ax.get_figure()
figure.savefig("cnn-pytorch-results")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
dataset_dict = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train_loader = dataset_dict['train']
test_loader = dataset_dict['test']
test_images = Variable(torch.tensor(test_loader.images))
test_labels = torch.tensor(test_loader.labels)
model = ConvNet(n_channels=3, n_classes=10).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
criterion = torch.nn.CrossEntropyLoss()
test_accs = []
train_accs = []
losses = []
for epoch in range(FLAGS.max_steps):
model.train()
batch_x, batch_y = train_loader.next_batch(FLAGS.batch_size)
batch_x, batch_y = Variable(
torch.tensor(batch_x).to(device)), Variable(
torch.tensor(batch_y).to(device))
optimizer.zero_grad()
out = model.forward(batch_x)
loss = criterion(out, batch_y.max(1)[1])
# l2_reg = torch.tensor(0).float().to(device)
# for param in model.parameters():
# l2_reg += torch.norm(param, 2)
# loss = torch.add(loss, l2_reg/100)
# l1_reg = torch.tensor(0).float().to(device)
# for param in model.parameters():
# l1_reg += torch.norm(param, 1)
# loss = torch.add(loss, l1_reg / 100000)
losses.append(round(float(loss), 3))
loss.backward()
optimizer.step()
if epoch % FLAGS.eval_freq == 0:
with torch.no_grad():
model.eval()
# print accuracy on test and train set
train_acc = accuracy(out, batch_y)
#COMPUTE TEST ACCURACY
all_predictions = torch.tensor([]).float().to(device)
for i in range(FLAGS.batch_size, test_images.shape[0],
FLAGS.batch_size):
out = model.forward(
test_images[i - FLAGS.batch_size:i].to(device))
all_predictions = torch.cat((all_predictions, out))
if i < test_images.shape[0]:
out = model.forward(test_images[i:].to(device))
all_predictions = torch.cat((all_predictions, out))
test_acc = accuracy(all_predictions, test_labels.to(device))
print(
'Train Epoch: {}/{}\tLoss: {:.6f}\tTrain accuracy: {:.6f}\tTest accuracy: {:.6f}'
.format(epoch, FLAGS.max_steps, loss, train_acc, test_acc))
test_accs.append(float(test_acc))
train_accs.append(float(train_acc))
with torch.no_grad():
# COMPUTE TEST ACCURACY
all_predictions = torch.tensor([]).float().to(device)
for i in range(FLAGS.batch_size, test_images.shape[0],
FLAGS.batch_size):
out = model.forward(test_images[i - FLAGS.batch_size:i].to(device))
all_predictions = torch.cat((all_predictions, out))
if i < test_images.shape[0]:
out = model.forward(test_images[i:].to(device))
all_predictions = torch.cat((all_predictions, out))
test_acc = accuracy(all_predictions, test_labels.to(device))
print('FINAL Test accuracy: {:.6f}'.format(test_acc))
import matplotlib.pyplot as plt
plt.figure()
plt.plot([i for i in range(0, epoch + 1, EVAL_FREQ_DEFAULT)], train_accs)
plt.plot([i for i in range(0, epoch + 1, EVAL_FREQ_DEFAULT)], test_accs)
plt.legend(["train", "test"])
plt.ylabel("accuracy")
plt.xlabel("epoch")
plt.savefig("cnn_accuracy")
plt.figure()
plt.plot([i for i in range(0, epoch + 1)], losses)
plt.legend(["loss"])
plt.ylabel("loss")
plt.xlabel("epoch")
plt.savefig("cnn_loss")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
import matplotlib.pyplot as plt
if not torch.cuda.is_available():
print("WARNING: CUDA DEVICE IS NOT AVAILABLE, WILL TRAIN ON CPU")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
data = cifar10_utils.get_cifar10(FLAGS.data_dir)
train = data['train']
test = data['test']
images_test_np = test.images
labels_test_np = test.labels
vgg = ConvNet(3, 10)
vgg.to(device)
if MEM_DEBUG:
print("Memory after VGG loaded: Alloc: {} MB, Cached: {} MB".format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(vgg.parameters())
loss_train = np.zeros((int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
loss_test = np.zeros((int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
accuracy_test = np.zeros(
(int(np.floor(FLAGS.max_steps / FLAGS.eval_freq), )))
for i in range(0, FLAGS.max_steps):
print('iter', i + 1, end='\r')
images_np, labels_np = train.next_batch(FLAGS.batch_size)
images = torch.from_numpy(images_np).to(device)
labels = torch.from_numpy(np.argmax(labels_np, axis=1)).to(device)
if MEM_DEBUG:
print("\nMemory after train loaded: Alloc: {} MB, Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
optimizer.zero_grad()
pred = vgg(images)
loss = criterion(pred, labels.long())
loss.backward()
optimizer.step()
del images
del labels
if (i + 1) % FLAGS.eval_freq == 0:
if MEM_DEBUG:
print("Memory entering the eval: Alloc: {} MB, Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
loss_train[i // FLAGS.eval_freq] = loss.item()
images_test = torch.from_numpy(images_test_np).to(device)
labels_test = torch.from_numpy(np.argmax(labels_test_np,
axis=1)).to(device)
if MEM_DEBUG:
print("Memory after test loaded: Alloc: {} MB Cached: {} MB".
format(
torch.cuda.memory_allocated(device) / 1e6,
torch.cuda.memory_cached(device) / 1e6))
vgg.eval()
with torch.no_grad():
pred_test = vgg(images_test)
vgg.train()
accuracy_test[i // FLAGS.eval_freq] = accuracy(
pred_test, F.one_hot(labels_test)).item()
loss_test[i // FLAGS.eval_freq] = criterion(
pred_test, labels_test.long()).item()
if PRINTS:
print()
print('test_loss:', loss_test[i // FLAGS.eval_freq])
print('test_accuracy:', accuracy_test[i // FLAGS.eval_freq])
print('train_loss:', loss_train[i // FLAGS.eval_freq])
if PLOTS:
fig, ax = plt.subplots(1, 2, figsize=(10, 5))
fig.suptitle('Training curves for Pytorch Convnet')
ax[0].set_title('Loss')
ax[0].set_ylabel('Loss value')
ax[0].set_xlabel('No of batches seen x{}'.format(FLAGS.eval_freq))
ax[0].plot(loss_train, label='Train')
ax[0].plot(loss_test, label='Test')
ax[0].legend()
ax[1].set_title('Accuracy')
ax[1].set_ylabel('Accuracy value')
ax[1].set_xlabel('No of batches seen x{}'.format(FLAGS.eval_freq))
ax[1].plot(accuracy_test, label='Test')
ax[1].legend()
import time
fig_name = 'pt_conv_training_{}_{}_{}_{}.jpg'.format(
FLAGS.max_steps, FLAGS.batch_size, FLAGS.eval_freq, time.time())
plt.savefig(fig_name)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_list = []
batch_list = []
accuracy_list = []
batch_list_test = []
use_cuda = torch.cuda.is_available()
if use_cuda:
print('Running in GPU model')
device = torch.device('cuda' if use_cuda else 'cpu')
dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
for iter in range(313):
x, y = cifar10['test'].next_batch(FLAGS.batch_size)
y = np.argmax(y, axis=1)
batch_list_test.append((x, y))
for iter in range(FLAGS.max_steps):
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
y = np.argmax(y, axis=1)
batch_list.append((x, y))
print('Batch list completed')
in_features = 3
out_features = 10 #num_classes
net = ConvNet(in_features, out_features).to(device)
lossfunc = nn.CrossEntropyLoss()
optimiser = optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
net.train()
for i in range(FLAGS.max_steps):
inputs, labels = batch_list[i]
# inputs = torch.from_numpy(inputs)
inputs = torch.tensor(inputs)
labels = torch.from_numpy(labels).long()
inputs, labels = inputs.to(device), labels.to(device)
optimiser.zero_grad()
outputs = net.forward(inputs.float())
loss = lossfunc(outputs, labels)
loss_list.append(loss)
loss.backward()
optimiser.step()
if (i + 1) % FLAGS.eval_freq == 0:
net.eval()
total = 0
for data in batch_list_test:
x_test, y_test = data
x_test = torch.from_numpy(x_test).type(dtype).to(device)
y_test = torch.from_numpy(y_test).long().to(device)
predicted = net.forward(x_test)
acc = accuracy(predicted, y_test)
total += acc
accuracy_list.append(total / len(batch_list_test))
print('Accuracy on test set at step {} is {}'.format(
i, total / len(batch_list_test)))
print('Loss of training is {}'.format(loss.item()))
plt.subplot(2, 1, 1)
plt.plot(
np.arange(len(accuracy_list) * FLAGS.eval_freq, step=FLAGS.eval_freq),
accuracy_list, 'o-')
plt.xlabel('Step')
plt.ylabel('Accuracy')
#
plt.subplot(2, 1, 2)
plt.plot(np.arange(len(loss_list)), loss_list)
plt.xlabel('Step')
plt.ylabel('Loss')
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
print(train.images.shape)
# transform = transforms.Compose(
# [transforms.Resize((224, 224)),
# transforms.ToTensor(),
# transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
if FLAGS.model == 'ALEX':
train.images = torch.Tensor([resize(img) for img in train.images])
test.images = torch.Tensor([resize(img) for img in test.images])
print('doneee')
model = models.alexnet(pretrained=True)
torch.save(model.state_dict(), 'alexnet.txt')
# model = torch.load('alexnet.txt')
for param in model.features.parameters():
param.requires_grad = False
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.classifier.parameters(),
lr=FLAGS.learning_rate)
else:
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(),
lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(2): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
if FLAGS.model == 'ADAM': x = resizer(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
if FLAGS.model == 'ADAM': batch_x = resizer(batch_x)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])
def train(_run):
"""
Performs training and evaluation of ConvNet model.
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
def get_xy_tensors(batch):
x, y = batch
x = torch.tensor(x, dtype=torch.float32).to(device)
y = torch.tensor(y, dtype=torch.long).to(device)
return x, y
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
datasets = cifar10_utils.read_data_sets(DATA_DIR_DEFAULT, one_hot=False)
train_data = datasets['train']
test_data = datasets['test']
model = ConvNet(n_channels=3, n_classes=10).to(device)
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
log_every = 100
avg_loss = 0
avg_acc = 0
for step in range(FLAGS.max_steps):
x, y = get_xy_tensors(train_data.next_batch(FLAGS.batch_size))
# Forward and backward passes
optimizer.zero_grad()
out = model.forward(x)
loss = loss_fn(out, y)
loss.backward()
# Parameter updates
optimizer.step()
avg_loss += loss.item() / log_every
avg_acc += accuracy(out, y) / log_every
if step % log_every == 0:
print('[{}/{}] train loss: {:.6f} train acc: {:.6f}'.format(
step, FLAGS.max_steps, avg_loss, avg_acc))
_run.log_scalar('train-loss', avg_loss, step)
_run.log_scalar('train-acc', avg_acc, step)
avg_loss = 0
avg_acc = 0
# Evaluate
if step % FLAGS.eval_freq == 0 or step == (FLAGS.max_steps - 1):
n_test_iters = int(
np.ceil(test_data.num_examples / TEST_BATCH_SIZE))
test_loss = 0
test_acc = 0
for i in range(n_test_iters):
x, y = get_xy_tensors(test_data.next_batch(TEST_BATCH_SIZE))
model.eval()
out = model.forward(x)
model.train()
test_loss += loss_fn(out, y).item() / n_test_iters
test_acc += accuracy(out, y) / n_test_iters
print('[{}/{}] test accuracy: {:6f}'.format(
step, FLAGS.max_steps, test_acc))
_run.log_scalar('test-loss', test_loss, step)
_run.log_scalar('test-acc', test_acc, step)
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
# select which device to train the model on
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# compute the input size of the MLP
input_size, n_classes = 3 * 32 * 32, 10
# if no pretrained model is passed to the commandline the convnet model will be initialized
if FLAGS.model == "custom":
model = ConvNet(3, n_classes).to(device)
else:
# check if the requested pretrained is available
assert FLAGS.model in pretrained_dict, "Model not available in pre_trained dict, given: {}, available: {}".format(
FLAGS.model, pretrained_dict.keys())
model = pretrained_dict[FLAGS.model](pretrained=True)
# image transforms needed to be alligned with the type of input the pretrained model is used to
normalize = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
resize = T.Resize(256)
centerCrop = T.CenterCrop(224)
# break out the last layer and add a linear layer that has 10 outputs instead of 1000
layers = []
first_linear = False
for child in model.children():
if list(child.children()) == []:
if isinstance(child, nn.Linear) and not first_linear:
layers.append(nn.Flatten())
first_linear = True
layers.append(child)
else:
for grandchild in child.children():
if isinstance(grandchild, nn.Linear) and not first_linear:
layers.append(nn.Flatten())
first_linear = True
layers.append(grandchild)
model = nn.Sequential(
*layers[:-1],
nn.Linear(in_features=layers[-1].in_features, out_features=10))
model.to(device)
# freeze the layers that are pretrained
for child in list(model.children())[:-1]:
for param in child.parameters():
param.requires_grad = False
# define the dataset, loss function and optimizer
dataset = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_fn = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=FLAGS.learning_rate)
for step in range(FLAGS.max_steps):
X_train, y_train = dataset['train'].next_batch(FLAGS.batch_size)
optimizer.zero_grad()
# from to normalize the data for pretrained model https://discuss.pytorch.org/t/how-to-efficiently-normalize-a-batch-of-tensor-to-0-1/65122
if FLAGS.model != "custom":
X_train, y_train = torch.tensor(X_train).float(), torch.tensor(
y_train).float().to(device)
X_train -= X_train.min(1, keepdim=True)[0]
X_train /= X_train.max(1, keepdim=True)[0]
X_train = torch.tensor([
normalize(T.ToTensor()(centerCrop(resize(
T.ToPILImage()(x))))).numpy() for x in X_train
]).to(device)
else:
# move to correct device and shape for MLP
X_train, y_train = torch.tensor(X_train).float().to(
device), torch.tensor(y_train).float().to(device)
predictions = model(X_train)
train_loss = loss_fn(predictions, y_train.argmax(1).long())
train_loss.backward()
optimizer.step()
# add the loss and accuracy to the lists for plotting
train_overall_loss.append(train_loss.cpu().detach().sum())
train_overall_accuracy.append(
accuracy(predictions.cpu().detach(),
y_train.cpu().detach()))
train_x_axis.append(step)
# test the model when eval freq is reached or if it is the last step
if not step % FLAGS.eval_freq or step + 1 == FLAGS.max_steps:
model.eval()
test_accuracies, test_losses_list = [], []
# test batchwise since it doesnot fit my gpu
for X_test, y_test in cifar_test_generator(dataset):
if FLAGS.model != "custom":
X_test, y_test = torch.tensor(X_test).float(
), torch.tensor(y_test).float().to(device)
X_test -= X_test.min(1, keepdim=True)[0]
X_test /= X_test.max(1, keepdim=True)[0]
X_test = torch.tensor([
normalize(T.ToTensor()(centerCrop(
resize(T.ToPILImage()(x))))).numpy()
for x in X_test
]).to(device)
else:
# move to correct device and shape for MLP
X_test, y_test = torch.tensor(X_test).float().to(
device), torch.tensor(y_test).float().to(device)
predictions = model(X_test)
test_loss = loss_fn(predictions, y_test.argmax(1).long())
test_accuracy = accuracy(predictions, y_test)
# add the values to compute the average loss and accuracy for the entire testset
test_accuracies.append(test_accuracy.cpu().detach())
test_losses_list.append(test_loss.cpu().detach().sum())
print(
"[{:5}/{:5}] Train loss {:.5f} Test loss {:.5f} Test accuracy {:.5f}"
.format(step, FLAGS.max_steps, train_loss, test_loss,
sum(test_accuracies) / len(test_accuracies)))
test_overall_accuracy.append(
sum(test_accuracies) / len(test_accuracies))
test_overall_loss.append(
sum(test_losses_list) / len(test_losses_list))
test_x_axis.append(step)
model.train()
# freeze the pretrained layers
if FLAGS.model != "custom":
for child in list(model.children())[:-1]:
for param in child.parameters():
param.requires_grad = False
plt.plot(train_x_axis, train_overall_loss, label="Avg Train loss")
plt.plot(test_x_axis, test_overall_loss, label="Avg Test loss")
plt.legend()
plt.savefig("convnet_loss_curve")
plt.show()
plt.plot(train_x_axis,
train_overall_accuracy,
label="Train batch accuracy")
plt.plot(test_x_axis, test_overall_accuracy, label="Test set accuracy")
plt.legend()
plt.savefig("convnet_accuracy_curve")
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
cifar10_data = cifar10_utils.get_cifar10(FLAGS.data_dir)
train_data = cifar10_data['train']
test_data = cifar10_data['test']
test_batch_size = 2000
num_steps = test_data.num_examples / test_batch_size
input_channels = train_data.images[0].shape[0]
output_size = train_data.labels.shape[1]
# Create model and optimizer
model = ConvNet(input_channels, output_size)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
model.train()
# Train & evaluate
eval_loss = 0.0
all_losses = []
lossv = []
train_accv = []
test_accv = []
for step in np.arange(1, FLAGS.max_steps + 1):
data, target = train_data.next_batch(FLAGS.batch_size)
data, target = Variable(torch.from_numpy(data).float()), Variable(torch.from_numpy(target))
optimizer.zero_grad()
prediction = model(data)
loss = criterion(prediction, torch.argmax(target, dim=1))
loss.backward()
optimizer.step()
all_losses.append(loss.item())
# Accuracy evaluation
eval_loss += loss.item()
if step % FLAGS.eval_freq == 0:
with torch.no_grad():
model.eval()
predicted_test = None
for _ in np.arange(num_steps):
test_x, test_y = test_data.next_batch(test_batch_size)
test_x = Variable(torch.from_numpy(test_x).float())
predicted_y = model(test_x)
predicted_y = predicted_y.detach().numpy()
if predicted_test is None:
predicted_test = predicted_y
else:
predicted_test = np.vstack((predicted_test, predicted_y))
accuracy_result = accuracy(predicted_test, test_data.labels)
lossv.append(eval_loss / FLAGS.eval_freq)
train_accv.append(accuracy(prediction.detach().numpy(), target.detach().numpy()))
test_accv.append(accuracy_result)
print('-- Step %d - accuracy: %.3f - loss: %.3f'
% (step, accuracy_result, eval_loss / FLAGS.eval_freq))
eval_loss = 0.0
model.train()
print("Training Done")
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
# set up the data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir, one_hot=False)
test_set = cifar10["test"]
test_batch_size = 2000
n_test_batches = int(test_set.num_examples / test_batch_size)
# set up the model and optimizer
conv_net = ConvNet(n_channels=3, n_classes=10)
loss_module = nn.CrossEntropyLoss()
conv_net.to(device)
optimizer = torch.optim.Adam(conv_net.parameters(), lr=FLAGS.learning_rate)
accuracies = []
losses = []
conv_net.train()
for i in range(FLAGS.max_steps):
# load data
images, labels = cifar10['train'].next_batch(FLAGS.batch_size)
images, labels = torch.from_numpy(images).to(device), torch.from_numpy(
labels).to(device)
# forward pass
model_pred = conv_net(images)
# calculate the loss
loss = loss_module(model_pred, labels)
# backward pass
optimizer.zero_grad()
loss.backward()
# update the parameters
optimizer.step()
# evaluate the model on the data set every eval_freq steps
conv_net.eval()
if i % FLAGS.eval_freq == 0:
test_accuracy = test_conv(conv_net, test_set, n_test_batches,
test_batch_size)
accuracies.append(test_accuracy)
losses.append(loss)
conv_net.train()
plot_curve(accuracies, 'Accuracy')
plot_curve(losses, 'Loss')
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
if torch.cuda.is_available():
torch.backends.cudnn.determinstic = True
torch.backends.cudnn.benchmark = False
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
device = torch.device("cuda:0")
else:
device = torch.device("cpu")
net = ConvNet(3, 10).to(device)
cifar = cifar10_utils.get_cifar10(FLAGS.data_dir)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=FLAGS.learning_rate)
# decay LR by 0.5 every 500 iterations
#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=500, gamma=0.5)
losses = []
accuracies = []
for step in range(FLAGS.max_steps):
# load the next batch
x, y = cifar["train"].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
# Normalize per channel
#x = x / torch.Tensor([62.245, 60.982, 65.468]).view(1, 3, 1, 1)
#x = x.reshape(FLAGS.batch_size, -1)
y = torch.from_numpy(y)
y = torch.argmax(y, dim=1).to(device)
# forward pass
net.train()
out = net(x)
loss = criterion(out, y)
losses.append(loss.item())
# backward pass + weight update
optimizer.zero_grad()
loss.backward()
optimizer.step()
#scheduler.step()
# evaluate every FLAGS.eval_freq iterations
if (step + 1) % FLAGS.eval_freq == 0:
net.eval()
# during test time, batch norm uses the running stats from training,
# so the batch size doesn't matter and we can choose one which divides
# the total number of samples
num_batches = 100
bs = 100
mean_acc = 0
for i in range(num_batches):
x, y = cifar["train"].next_batch(bs)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(y).to(device)
#x = x / torch.Tensor([62.245, 60.982, 65.468]).view(1, 3, 1, 1)
with torch.no_grad():
out = net(x)
mean_acc += accuracy(out, y) / num_batches
accuracies.append(mean_acc)
print("Step {}, accuracy: {:.5f} %".format(step + 1,
mean_acc * 100))
plt.figure()
plt.plot(range(1, len(losses) + 1), losses)
plt.xlabel("Number of batches")
plt.ylabel("Batch loss")
plt.savefig("fig/loss_curve.pdf")
plt.close()
plt.figure()
plt.plot(range(1,
FLAGS.eval_freq * len(accuracies) + 1, FLAGS.eval_freq),
accuracies)
plt.xlabel("Number of batches")
plt.ylabel("Accuracy on the test set")
plt.savefig("fig/accuracy_curve.pdf")
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
########################
# PUT YOUR CODE HERE #
#######################
model = ConvNet(3, 10)
print(model)
cv_size = 10000
cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py',
validation_size=cv_size)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=FLAGS.learning_rate)
log = defaultdict(list)
for step in range(FLAGS.max_steps):
optimizer.zero_grad()
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
h = model.forward(x)
loss = criterion(h, y.argmax(1))
loss.backward()
optimizer.step()
if step % FLAGS.eval_freq == 0:
log['train_loss'].append(loss.item())
log['train_acc'].append(accuracy(h, y))
model.eval()
x, y = cifar10['validation'].next_batch(cv_size)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
h = model.forward(x)
loss = criterion(h, y.argmax(1))
log['cv_loss'].append(loss.item())
log['cv_acc'].append(accuracy(h, y))
model.train()
print(
f"Step {step} | "
f"Training loss: {log['train_loss'][-1]:.5f}, "
f"accuracy: {100 * log['train_acc'][-1]:.1f}% | "
f"CV loss: {log['cv_loss'][-1]:.5f}, "
f"accuracy: {100 * log['cv_acc'][-1]:.1f}%")
model.eval()
x, y = cifar10['test'].next_batch(cifar10['test'].num_examples)
x = torch.from_numpy(x)
y = torch.from_numpy(y)
h = model.forward(x)
loss = criterion(h, y.argmax(1))
print(f"Test loss: {loss.item()}, accuracy: {100 * accuracy(h, y):.1f}%")
# Plot loss and accuracy.
plt.subplot(121)
plt.title("Loss")
plt.plot(log['train_loss'], label="Training")
plt.plot(log['cv_loss'], label="Cross Validation")
plt.xlabel("Step")
plt.legend()
plt.subplot(122)
plt.title("Accuracy")
plt.plot(log['train_acc'], label="Training")
plt.plot(log['cv_acc'], label="Cross Validation")
plt.xlabel("Step")
plt.legend()
plt.legend()
plt.show()
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
# set cuda
cuda = False
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
# import the test data
cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir)
loss_function = torch.nn.CrossEntropyLoss()
neuralnet = ConvNet(3, 10)
if cuda:
neuralnet.cuda() # run on GPU
sgd_back = torch.optim.Adam(neuralnet.parameters(), lr=FLAGS.learning_rate)
#lists with losses and accuracies
train_losses = []
train_accs = []
test_losses = []
test_accs = []
graph_x = []
# for i in range(50):
for i in range(FLAGS.max_steps):
neuralnet.train()
x, y = cifar10['train'].next_batch(FLAGS.batch_size)
if cuda:
x = torch.from_numpy(x).cuda()
y = torch.from_numpy(y).cuda()
else:
x = torch.from_numpy(x)
y = torch.from_numpy(y)
# predict on the train data and calculate the gradients
out = neuralnet.forward(x) #output after the softmax
train_loss = loss_function(out, y.argmax(dim=1))
sgd_back.zero_grad()
train_loss.backward()
sgd_back.step()
# save the losses for every eval_freqth loop
if i % FLAGS.eval_freq == 0 or i == (FLAGS.max_steps - 1):
# if i % 1 == 0 or i == (FLAGS.max_steps - 1):
neuralnet.eval()
with torch.no_grad():
test_x, test_y = cifar10['test'].next_batch(1000)
if cuda:
torch.cuda.empty_cache()
test_x = torch.from_numpy(test_x).cuda()
test_y = torch.from_numpy(test_y).cuda()
else:
test_x = torch.from_numpy(test_x)
test_y = torch.from_numpy(test_y)
train_out = neuralnet.forward(x) # output after the softmax
test_out = neuralnet.forward(test_x)
if cuda:
torch.cuda.empty_cache()
train_loss = loss_function(train_out, y.argmax(dim=1))
test_loss = loss_function(test_out, test_y.argmax(dim=1))
train_acc = accuracy(out, y)
test_acc = accuracy(test_out, test_y)
train_losses.append(train_loss)
train_accs.append(train_acc)
test_losses.append(test_loss)
test_accs.append(test_acc)
graph_x.append(i)
print("iteration:", i)
print("Test accuracy:", test_accs[-1])
print("Test loss:", test_losses[-1])
plt.figure()
plt.subplot(1, 2, 1)
plt.plot(graph_x, train_losses, label="train loss")
plt.plot(graph_x, test_losses, label="test loss")
plt.title('Losses')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(graph_x, train_accs, label="train acc")
plt.plot(graph_x, test_accs, label="test acc")
plt.title('Accuracies')
plt.legend()
print("Final test accuracy:", test_accs[-1])
print("Final test loss:", test_losses[-1])
plt.savefig("conv_acc_and_loss.png")
with open('conv_acc_and_loss.txt', 'w') as f:
f.write("test_losses \n")
f.write(str(test_losses) + "\n \n")
f.write("test accs \n")
f.write(str(test_accs))
def train():
"""
Performs training and evaluation of ConvNet model.
TODO:
Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
########################
# PUT YOUR CODE HERE #
#######################
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
print()
cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT)
train = cifar10['train']
input_channels = 3
output_class = 10
net = ConvNet(input_channels, output_class)
net.to(device)
if OPTIMIZER_DEFAULT == 'ADAM':
optimizer = torch.optim.Adam(net.parameters(),lr=FLAGS.learning_rate)
CrossEntropyLoss = nn.CrossEntropyLoss()
loss_iter = []
loss_mean = []
acc_iter = []
loss_sum = 0.0
for iter_n in np.arange(0,FLAGS.max_steps):
x, labels = train.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
labels = np.argwhere(labels>0)
labels = torch.from_numpy(labels[:,1]).to(device)
optimizer.zero_grad()
net.train()
predictions = net.forward(x)
loss = CrossEntropyLoss(predictions,labels.long())
loss_iter.append(loss.item())
loss_sum += loss.item()
loss_mean.append(np.mean(loss_iter[:-50:-1]))
loss.backward()
optimizer.step()
print("Iter: {}, training Loss: {}".format(iter_n, "%.3f"%loss.item()))
if (iter_n+1) % int(FLAGS.eval_freq) == 0:
print("....Testing.....\n")
test = cifar10['test']
acc = []
with torch.no_grad():
for _ in np.arange(0,(test.num_examples//FLAGS.batch_size)):
x, t = test.next_batch(FLAGS.batch_size)
x = torch.tensor(x).to(device)
t = torch.tensor(t).to(device)
net.eval()
y = net.forward(x)
acc.append(accuracy(y,t))
acc_iter.append(np.mean(acc))
print("Testing accuracy: {}".format("%.3f"%np.mean(acc)))
print()
plt.plot(loss_iter,'r-',alpha=0.1, label="Batch loss")
plt.plot(loss_mean,'b-', label="Average loss")
plt.legend()
plt.xlabel("Iterations")
plt.ylabel("Loss")
plt.title("Training Loss")
plt.grid(True)
plt.show()
plt.close()
plt.plot(acc_iter,'g-')
plt.xlabel("Iterations")
plt.ylabel("Accuracy")
plt.grid(True)
plt.title("Test Accuracy")
plt.show()
plt.close()
print()
print("COMPLETED")
def train():
"""
Performs training and evaluation of ConvNet model.
"""
### DO NOT CHANGE SEEDS!
# Set the random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)
if torch.cuda.is_available():
torch.cuda.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
# print("Device", device)
data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir)
train = data['train']
test = data['test']
# print(train.images[0].shape)
# n_inputs = train.images[0].flatten().shape[0]
n_channels = train.images[0].shape[0]
n_classes = train.labels[0].shape[0]
model = ConvNet(n_channels, n_classes)
loss_mod = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=FLAGS.learning_rate)
model.to(device)
loss_history = []
acc_history = []
for step in range(FLAGS.max_steps): #FLAGS.max_steps
model.train()
x, y = train.next_batch(FLAGS.batch_size)
x = torch.from_numpy(x).to(device)
y = torch.from_numpy(np.argmax(y, axis=1)).to(
device) # converts onehot to dense
# out = model.forward(x)
out = model(x)
loss = loss_mod(out, y)
loss_history.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step == 0 or (step + 1) % FLAGS.eval_freq == 0:
model.eval()
with torch.no_grad():
x, y = torch.from_numpy(test.images), torch.from_numpy(
test.labels)
acc = 0
test_step = int(x.shape[0] / 20)
for i in range(0, x.shape[0], test_step):
batch_x = x[i:i + test_step].to(device)
batch_y = y[i:i + test_step].to(device)
test_out = model.forward(batch_x)
acc += accuracy(test_out, batch_y) / 20
print('Accuracy:', acc)
acc_history.append(acc)
print('Final loss:', loss_history[-1])
print('Final acc:', acc_history[-1])
plt.plot(loss_history)
plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq),
acc_history) # range(0, FLAGS.max_steps, FLAGS.eval_freq)
plt.legend(['loss', 'accuracy'])
# plt.show()
plt.savefig('/home/lgpu0376/code/output_dir/loss_acc_graph.png')
# plt.savefig('loss_acc_graph.png')
with open('/home/lgpu0376/code/output_dir/output.out', 'w+') as f:
f.write(f'Final loss: {loss_history[-1]}')
f.write(f'\nFinal acc: {acc_history[-1]}')
