def test(model_name):
# Parameters
output_period = 100
batch_size = 100
maxk = 5
print("Using: " + model_name)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_18()
model.load_state_dict(torch.load('models/' + model_name))
model = model.to(device)
_, test_loader = dataset.get_val_test_loaders(batch_size)
num_test_batches = len(test_loader)
filenames = get_test_filenames(test_loader)
full_predictions = []
num_test_batches = len(test_loader)
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
if batch_num % output_period == 0:
print('Progress: %.2f' % (batch_num * 1.0 / num_test_batches))
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
pred = outputs.topk(maxk, 1)[1]
corresponding_files = filenames[(batch_num - 1) *
batch_size:batch_num * batch_size]
full_predictions += list(zip(corresponding_files, pred))
write_full_predictions_to_file(full_predictions)
def generate_test_label():
'''
Create Test Labels
'''
batch_size = 1
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_18()
model.load_state_dict(torch.load('./models/model.10', map_location=device))
model = model.to(device)
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
num_test_batches = len(val_loader)
#print(val_loader.dataset.classes)
top_5_correct = 0
top_1_correct = 0
total = 0
model.eval()
F = open("adam_results_3_correct.txt", "w+")
count = 0
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
#print(inputs)
with torch.no_grad():
output = model(inputs)
output = output.to('cpu')[0, :]
output_sorted = output.numpy().argsort()
#print(output[output_sorted[0]])
#print(output[output_sorted[-1]])
top_5 = output_sorted[-5:]
#print(top_5)
mapped = []
for i in top_5:
mapped.append(val_loader.dataset.classes[i])
#print(mapped)
line = test_loader.dataset.samples[count][0]
#line = 'test/' + line[-12:]
line = line + ' ' + str(mapped[4]) + ' ' + str(mapped[3]) + ' ' + str(
mapped[2]) + ' ' + str(mapped[1]) + ' ' + str(mapped[0])
line += '\n'
#print(line)
F.write(line)
count += 1
if count % 100 == 0:
print(count)
F.close()
def run(model_file):
# Parameters
num_epochs = 10
output_period = 20
batch_size = 70
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_18()
model.load_state_dict(torch.load(model_file))
model = model.to(device)
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
num_test_batches = len(test_loader)
folder_list = val_loader.dataset.classes
with open(model_file + "_test_out.txt", "w") as outptfile:
model.eval()
correctInValEpoch = 0
top5InValEpoch = 0
epoch_val_samples = 0
with torch.no_grad():
for batch_num, (inputs, labels) in enumerate(val_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
acc1, acc5 = accuracy(outputs, labels, topk=(1,5))
n = outputs.size(0)
correctInValEpoch += acc1[0]*n
top5InValEpoch += acc5[0]*n
epoch_val_samples += n
print("top 5 accuracy: ",100-top5InValEpoch.item()/(epoch_val_samples))
outs = []
with torch.no_grad():
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs).topk(5, 1, True, True)[1]
outputs = outputs.cpu()
def toFolderNum(x):
return str(folder_list[x])
toFolderNumVec = np.vectorize(toFolderNum)
classifications = toFolderNumVec(outputs.numpy())
#print(classifications)
outs.extend(classifications)
for name, preds in zip(sorted(os.listdir("data/test/999")), outs):
outptfile.write("test/{} {}".format(name, " ".join(preds)))
outptfile.write("\n")
gc.collect()
def main():
categories = load_categories()
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
batch_size = 100
# load model and set to evaluation mode
network = 'OurBest'
# network = 'AlexNet'
model = load_model(network)
model.to(device)
model.eval()
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
###################################################
# Making result file
###################################################
f = open("2pm/result.txt", "w")
f_ = open("2pm/result_more.txt", "w")
count = 1
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
prediction = model(inputs)
prediction = prediction.to('cpu')
_, ind = torch.topk(prediction, 5)
for i in xrange(ind.shape[0]):
line = 'test/' + '%08d' % count + '.jpg ' + str(
ind[i][0].item()) + ' ' + str(ind[i][1].item()) + ' ' + str(
ind[i][2].item()) + ' ' + str(
ind[i][3].item()) + ' ' + str(ind[i][4].item()) + '\n'
line_details = 'test/' + '%08d' % count + '.jpg ' + str(
categories[ind[i][0].item()]) + ' ' + str(
categories[ind[i][1].item()]) + ' ' + str(
categories[ind[i][2].item()]) + ' ' + str(
categories[ind[i][3].item()]) + ' ' + str(
categories[ind[i][4].item()]) + '\n'
f.write(line)
f_.write(line_details)
count += 1
f.close()
f_.close()
def run_test():
batch_size = 1
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_50()
model.load_state_dict(
torch.load('./models/model_res50adam01.10', map_location=device))
model = model.to(device)
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
num_test_batches = len(val_loader)
top_5_correct = 0
top_1_correct = 0
total = 0
model.eval()
'''
Evaluate on validation set
'''
for batch_num, (inputs, labels) in enumerate(val_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
#print(inputs)
with torch.no_grad():
output = model(inputs)
output = output.to('cpu')[0, :]
output_sorted = output.numpy().argsort()
#print(output[output_sorted[0]])
#print(output[output_sorted[-1]])
top_5 = output_sorted[-5:]
for i in top_5:
if i == int(labels):
top_5_correct += 1
if int(torch.argmax(output)) == int(labels):
top_1_correct += 1
total += 1
if total % 100 == 0:
print(total)
if total > 1000:
print("Top 5 correct: ", float(top_5_correct) / total)
print("Top 1 correct: ", float(top_1_correct) / total)
return
def run(num_epochs, out_period, batch_size, model):
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = model.to(device)
epoch = 8 #default epoch to load
if len(sys.argv) > 1: #first sysarg is name of script
epoch = sys.argv[1] #take number
# load model
print("loading models/model.%s" % epoch)
model.load_state_dict(torch.load("models/model.%s" % epoch))
model.eval()
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
# Opens file to write results to, will overwrite existing files
out_file = open("results.txt", "w")
total = 0
for (inputs, labels) in tqdm(test_loader):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
top5 = torch.topk(outputs, 5)[1]
# path = "test/" + '{0:08d}'.format(i) + ".jpg"
for i in range(len(inputs)):
filename = test_loader.dataset.samples[total][0][-12:-1]
# formats string in the structure of "test/00000132.jpg 1 3 5 6 9"
path_top5 = "test/" + filename + "g" #lol, SFB ETU RFC
for j in top5[i]:
path_top5 = path_top5 + " " + str(j.item())
out_file.write(path_top5 + "\n")
# print(labels[i], "TOP5:", top5[i])
# print(path_top5)
total += 1
gc.collect()
#remove final newline
out_file.seek(out_file.tell() - 2)
out_file.truncate()
def run():
# Parameters
num_epochs = 10
output_period = 100
batch_size = 100
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_18()
model = model.to(device)
val_loader, _ = dataset.get_val_test_loaders(batch_size)
epoch = 1
# run validation on different models (USER INPUT)
model.load_state_dict(torch.load('models/trialTwo/model.10'))
model.eval()
# Calculate classification error and Top-5 Error
# on validation dataset here
top5Correct = 0
tot = 0
for batch_num, (inputs, labels) in enumerate(val_loader, 1):
inputs = inputs.to(device)
outputs = model(inputs)
prediction = outputs.to('cpu')
for i in range(batch_size):
curr = prediction[i].unsqueeze(1)
_, cls5 = torch.topk(curr, 5, dim=0)
cls5List = list(cls5.numpy())
if labels[i] in cls5:
top5Correct += 1
tot += 1
print('top 5 percent error for validation set:',
1 - (top5Correct / (tot * 1.0)))
gc.collect()
import gc
import sys
import pickle
from torch.autograd import Variable
import torch
import torch.nn as nn
torch.backends.cudnn.benchmark = True
import matplotlib.pyplot as plt
import numpy as np
import dataset
model_path = './'
# Parameters
num_epochs = 3
output_period = 100
batch_size = 100
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
lol = inputs[99].numpy()
plt.imshow(np.transpose(lol, (1, 2, 0)))
plt.show()
break
# print('Starting evaluation on', model_path)
# run()
# print('Evaluation terminated for', model_path)
def run():
# Parameters
num_epochs = 10
output_period = 100
batch_size = 100
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_18()
model = model.to(device)
val_loader, test_loader = dataset.get_val_test_loaders(batch_size)
epoch = 1
# know what model is before calling this file bc of validation
model.load_state_dict(torch.load('models/trialTwo/model.10'))
model.eval()
# INITializes variables
top5Correct = 0
tot = 0
dictOldNewLabels = {}
f = open('testResults.txt', 'w')
# HANDLES data_loader error
dir = './data/train'
count = 0
# get the order that data_loader sees the classes
subdirs = [x[0] for x in os.walk(dir)]
subdirs.sort(key=lambda x: str(x))
# remaps indices of the data_loader to the real classes
ind = len(dir)
subdirs = subdirs[1:]
for subdir in subdirs:
dictOldNewLabels[str(count)] = subdir[ind + 1:]
count += 1
# CALCULATE classification error and Top-5 Error
filePathNumber = 0
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
outputs = model(inputs)
prediction = outputs.to('cpu')
for i in range(batch_size):
lineArray = []
filePathNumber += 1
filePath = 'test/' + str(filePathNumber).zfill(8) + ".jpg"
lineArray.append(filePath)
curr = prediction[i].unsqueeze(1)
_, cls5 = torch.topk(curr, 5, dim=0)
cls5List = list(cls5.numpy())
for category in cls5List:
actualCategory = dictOldNewLabels[str(category[0])]
lineArray.append(str(actualCategory))
line = " ".join(lineArray) + "\n"
if filePathNumber == 10000:
line = " ".join(lineArray) # don't append the "\n"
f.write(line)
gc.collect()
def run(OPTIMIZATION_OPTION = 0):
# Parameters
num_epochs = 20 # do not have to train 10 every time
output_period = 100
batch_size = 100
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_34()
model = model.to(device)
train_loader, val_loader = dataset.get_data_loaders(batch_size)
num_train_batches = len(train_loader)
criterion = nn.CrossEntropyLoss().to(device)
# TODO: optimizer is currently unoptimized
# there's a lot of room for improvement/different optimizers
lr = 1e-3
lr_step = 1
optimizer = ''
scheduler = ''
if OPTIMIZATION_OPTION == 0: #default
optimizer = optim.SGD(model.parameters(), lr=1e-3) # change lr value in order to change learning rate
elif OPTIMIZATION_OPTION == 3:
# add your optimization here..
# optimizer = ..
lr = 1e-1
optimizer = optim.SGD(model.parameters(), lr=lr, dampening=0.5, weight_decay=1e-4)
elif OPTIMIZATION_OPTION == 1:
lr = 1e-1
optimizer = optim.SGD(list(filter(lambda p: p.requires_grad, model.parameters())), lr=lr, momentum=0.9, dampening=0.5, weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
elif OPTIMIZATION_OPTION == 9:
# starting from low lr and increasing lr at each batch in order to find optimal lr
min_lr = 1e-7
max_lr = 100
lr = min_lr
optimizer = optim.SGD(model.parameters(), lr=min_lr,dampening=0.5, weight_decay=1e-4, momentum=0.9)
lr_step=(max_lr/min_lr)**(1/100)
output_period = 1
elif OPTIMIZATION_OPTION == 8:
num_epochs = 9
# add momentum to option 2
lr = 1e-1
optimizer = optim.SGD(model.parameters(), lr=lr, dampening=0.5, weight_decay=1e-4, momentum=0.9)
elif OPTIMIZATION_OPTION == 5:
lr = 1e-1
optimizer = optim.SGD(list(filter(lambda p: p.requires_grad, model.parameters())), lr=lr, momentum=0.9, weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
elif OPTIMIZATION_OPTION == 6:
lr = 1e-1
optimizer = optim.SGD(list(filter(lambda p: p.requires_grad, model.parameters())), lr=lr, momentum=0.9, weight_decay=1e-5)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
else:
output_period = 100
top1_dic = {}
top5_dic = {}
#localtime = time.localtime()
#timeString = time.strftime("%Y%m%d%H%M%S", localtime)
epoch = 1
while epoch <= num_epochs:
running_loss = 0.0
for param_group in optimizer.param_groups:
print('Current learning rate: ' + str(param_group['lr']))
if scheduler:
scheduler.step()
model.train()
for batch_num, (inputs, labels) in enumerate(train_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
#print(batch_num)
#print(inputs)
#print(labels)
optimizer.zero_grad()
outputs = model(inputs)
#print(outputs)
loss = criterion(outputs, labels)
loss.backward()
topk = 5
#val, index = outputs[0].topk(topk, 0, True, True)
#print(val, index)
optimizer.step()
running_loss += loss.item()
if batch_num % output_period == 0:
print('[%d:%.2f] loss: %.3f' % (
epoch, batch_num*1.0/num_train_batches,
running_loss/output_period
))
running_loss = 0.0
gc.collect()
# update learning rate every batch in order to get the best lr
# ref: Cyclical Learning Rates for Training Neural Networks
if lr_step != 1:
print(lr)
#print()
lr = lr * lr_step
for param_group in optimizer.param_groups:
param_group['lr'] = lr
gc.collect()
# save after every epoch
torch.save(model.state_dict(), "models/model.%d" % epoch)
# TODO: Calculate classification error and Top-5 Error
# on training and validation datasets here
# comparing the labels vector against the output vector?
print("Validating validation datasets ... ")
model.eval()
top1_list_val = []
top5_list_val = []
for batch_num, (inputs, labels) in enumerate(val_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
acc1, acc5 = accuracy(outputs, labels, topk=(1, 5))
top1_list_val.append(acc1[0] / inputs.size(0))
top5_list_val.append(acc5[0] / inputs.size(0))
top1_dic_val[epoch] = np.mean(top1_list_val)
top5_dic_val[epoch] = np.mean(top5_list_val)
print("The current model" + str(epoch) + ":")
print("Top 1 Accuracy: " + str(np.mean(top1_list_val)))
print("Top 5 Accuracy: " + str(np.mean(top5_list_val)))
gc.collect()
if np.mean(top5_list_val) > 0.70:
print("Predicting test datasets cuz it above 70.0% ... ")
model.eval()
_, test_loader = dataset.get_val_test_loaders(batch_size)
localtime = time.localtime()
timeString = time.strftime("%m%d%H%M%S", localtime)
f = open("output%s.txt" % timeString, "w")
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
for i in range(batch_size):
_, index = outputs[i].topk(5, 0, True, True)
f.write("test/%08d.jpg %d %d %d %d %d \n" % ((batch_num-1)*batch_size+(i+1),
index[0], index[1], index[2], index[3], index[4]))
f.close()
gc.collect()
epoch += 1
val_loader2, test_loader= dataset.get_val_test_loaders(batch_size)
localtime = time.localtime()
timeString = time.strftime("%m%d%H%M%S", localtime)
f = open("output%s.txt" % timeString, "w")
#f.write("Woops! I have deleted the content!")
# test data set
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
#print(labels)
outputs = model(inputs)
for i in range(batch_size):
val, index = outputs[i].topk(5, 0, True, True)
f.write("test/%08d.jpg %d %d %d %d %d \n" % ((batch_num-1)*batch_size+(i+1), index[0], index[1], index[2], index[3], index[4]))
f.close()
def run():
# Parameters
num_epochs = 40
output_period = 100
batch_size = 100
# setup the device for running
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = resnet_18()
model = model.to(device)
train_loader, val_loader = dataset.get_data_loaders(batch_size)
num_train_batches = len(train_loader)
criterion = nn.CrossEntropyLoss().to(device)
# TODO: optimizer is currently unoptimized
# there's a lot of room for improvement/different optimizers
# lr=1e-3
# optimizer = optim.SGD(model.parameters(), lr=1e-2, weight_decay=1e-4)
optimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)
top1_dic_val = {}
top5_dic_val = {}
top1_dic_train = {}
top5_dic_train = {}
epoch = 1
while epoch <= num_epochs:
running_loss = 0.0
for param_group in optimizer.param_groups:
print('Current learning rate: ' + str(param_group['lr']))
print('Current weight decay: ' + str(param_group['weight_decay']))
model.train()
for batch_num, (inputs, labels) in enumerate(train_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
if batch_num % output_period == 0:
print('[%d:%.2f] loss: %.3f' % (
epoch, batch_num*1.0/num_train_batches,
running_loss/output_period
))
running_loss = 0.0
gc.collect()
gc.collect()
# save after every epoch
torch.save(model.state_dict(), "models/model.%d" % epoch)
# TODO: Calculate classification error and Top-5 Error
# on training and validation datasets here
print("Validating validation datasets ... ")
model.eval()
top1_list_val = []
top5_list_val = []
for batch_num, (inputs, labels) in enumerate(val_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
acc1, acc5 = accuracy(outputs, labels, topk=(1, 5))
top1_list_val.append(acc1[0] / inputs.size(0))
top5_list_val.append(acc5[0] / inputs.size(0))
top1_dic_val[epoch] = np.mean(top1_list_val)
top5_dic_val[epoch] = np.mean(top5_list_val)
print("The current model" + str(epoch) + ":")
print("Top 1 Accuracy: " + str(np.mean(top1_list_val)))
print("Top 5 Accuracy: " + str(np.mean(top5_list_val)))
gc.collect()
if np.mean(top5_list_val) > 0.70:
print("Predicting test datasets cuz it above 70.0% ... ")
model.eval()
_, test_loader = dataset.get_val_test_loaders(batch_size)
localtime = time.localtime()
timeString = time.strftime("%m%d%H%M%S", localtime)
f = open("output%s.txt" % timeString, "w")
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
for i in range(batch_size):
_, index = outputs[i].topk(5, 0, True, True)
f.write("test/%08d.jpg %d %d %d %d %d \n" % ((batch_num-1)*batch_size+(i+1),
index[0], index[1], index[2], index[3], index[4]))
f.close()
# print("Validating training datasets ... ")
# model.eval()
# top1_list_train = []
# top5_list_train = []
# for batch_num, (inputs, labels) in enumerate(train_loader, 1):
# inputs = inputs.to(device)
# labels = labels.to(device)
# outputs = model(inputs)
# acc1, acc5 = accuracy(outputs, labels, topk=(1, 5))
# top1_list_train.append(acc1[0] / inputs.size(0))
# top5_list_train.append(acc5[0] / inputs.size(0))
# top1_dic_train[epoch] = np.mean(top1_list_train)
# top5_dic_train[epoch] = np.mean(top5_list_train)
# print("The current model" + str(epoch) + ":")
# print("Top 1 Accuracy: " + str(np.mean(top1_list_train)))
# print("Top 5 Accuracy: " + str(np.mean(top5_list_train)))
print("**************************************************")
gc.collect()
epoch += 1
print("Predicting test datasets ... ")
model.eval()
_, test_loader = dataset.get_val_test_loaders(batch_size)
localtime = time.localtime()
timeString = time.strftime("%m%d%H%M%S", localtime)
f = open("output%s.txt" % timeString, "w")
for batch_num, (inputs, labels) in enumerate(test_loader, 1):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
for i in range(batch_size):
_, index = outputs[i].topk(5, 0, True, True)
f.write("test/%08d.jpg %d %d %d %d %d \n" % ((batch_num-1)*batch_size+(i+1),
index[0], index[1], index[2], index[3], index[4]))
f.close()