def eval_model():
# to track the training loss as the model trains
test_losses = []
#to track the accuracy
acc_per_image = []
acc_per_batch = []
#track absolute hits
hit_list = []
#track number of fixations
n_fixations = []
model.eval() # prep model for evaluation
t = iter(test_loader)
for i, example in enumerate(t): #start at index 0
# get the inputs
data = example["image"]
#print("input sum: {}".format(torch.sum(data)))
target = example["fixations"]
target_locs = example["fixation_locs"]
#push data and targets to gpu
if gpu:
if torch.cuda.is_available():
data = data.to('cuda')
target = target.to('cuda')
# forward pass: compute predicted outputs by passing inputs to the model
output = model(data)
#drop channel-dimension (is only 1) so that outputs will be of same size as targets (batch_size,100,100)
output = output.view(-1, target.size()[-2], target.size()[-2])
loss = criterion(output, target)
# calculate the loss
#loss = myLoss(output, target)
# record training loss
test_losses.append(loss.item())
#accuracy
acc_this_batch = 0
for batch_idx in range(output.size()[0]):
output_subset = output[batch_idx]
target_subset = target[batch_idx]
target_locs_subset = target_locs[batch_idx]
acc_this_image, hits, num_fix = accuracy(output_subset,
target_subset,
target_locs_subset, gpu)
acc_per_image.append(acc_this_image)
hit_list.append(hits)
n_fixations.append(num_fix)
acc_this_batch += acc_this_image
#divide by batch size
acc_this_batch /= output.size()[0]
acc_per_batch.append(acc_this_batch)
acc_per_image = np.asarray(acc_per_image)
print("Mean test loss is: {}".format(np.average(test_losses)))
print("Mean accuracy for test set ist: {}".format(np.mean(acc_per_image)))
print("Hits: {}".format(sum(hit_list)))
#drop channel-dimension (is only 1) so that outputs will be of same size as targets (batch_size,100,100)
output = output.view(-1, target.size()[-2], target.size()[-2])
loss = criterion(output, target)
# calculate the loss
#loss = myLoss(output, target)
# record training loss
test_losses.append(loss.item())
#accuracy
acc_this_batch = 0
for batch_idx in range(output.size()[0]):
output_subset = output[batch_idx]
target_subset = target[batch_idx]
target_locs_subset = target_locs[batch_idx]
acc_this_image, hits, num_fix = accuracy(output_subset, target_subset,
target_locs_subset, gpu)
acc_per_image.append(acc_this_image)
hit_list.append(hits)
n_fixations.append(num_fix)
acc_this_batch += acc_this_image
#divide by batch size
acc_this_batch /= output.size()[0]
acc_per_batch.append(acc_this_batch)
acc_per_image = np.asarray(acc_per_image)
print("Mean test loss is: {}".format(np.average(test_losses)))
print("Mean accuracy for test set ist: {}".format(np.mean(acc_per_image)))
# In[9]:
#print value of sigmoid scaling parameter (it's the first one, so stop after one iteration)
def run_script3(batch_size=8, lr=0.001, n_epochs=10, gpu=True):
def create_datasets(batch_size):
#transforms
#downsampling by factor of 10 as the images were resized from (1000,1000) to (100,100),
#so the fixations have to be, too
data_transform = transforms.Compose([ToTensor(),Downsampling(10), Targets2D(100,100, 100), NormalizeTargets()]) #ExpandTargets(100)])
#load split data
figrim_dataset_train = FigrimFillersDataset(json_file='allImages_unfolded_train.json',
root_dir='figrim/fillerData/Fillers',
transform=data_transform)
figrim_dataset_val = FigrimFillersDataset(json_file='allImages_unfolded_val.json',
root_dir='figrim/fillerData/Fillers',
transform=data_transform)
figrim_dataset_test = FigrimFillersDataset(json_file='allImages_unfolded_test.json',
root_dir='figrim/fillerData/Fillers',
transform=data_transform)
#create data loaders
#set number of threads to 8 so that 8 processes will transfer 1 batch to the gpu in parallel
dataset_loader_train = torch.utils.data.DataLoader(figrim_dataset_train, batch_size=batch_size,
shuffle=True, num_workers=8)
dataset_loader_val = torch.utils.data.DataLoader(figrim_dataset_val, batch_size=batch_size,
shuffle=True, num_workers=8)
#no shuffling, as to be able to identify which images were processed well/not so well
dataset_loader_test = torch.utils.data.DataLoader(figrim_dataset_test, batch_size=batch_size,
shuffle=False, num_workers=8)
return dataset_loader_train, dataset_loader_val, dataset_loader_test
# In[26]:
#set extract arguments given on calling the script from the command line
batch_size = batch_size#int(sys.argv[1])
lr = lr#float(sys.argv[2])
n_epochs = n_epochs#int(sys.argv[3])
gpu = gpu#bool(sys.argv[4])
#gpu = True
class MyVGG19(torch.nn.Module):
def __init__(self):
super(MyVGG19, self).__init__()
#leave out final two layers (avgsize-pooling and flattening)
features = list(torchvision.models.vgg19(pretrained = True).features)
self.features = nn.ModuleList(features).eval()
def forward(self, x):
results = []
#go through modules
for ii,model in enumerate(self.features):
#forward propagation
x = model(x)
#take 4th module's activations
if ii in {28,29,31,32,35}:
results.append(x)
# if ii == 15:
# break
#upsample (rescale to same size)
#x = functional.interpolate(x, size=(100,100), mode="bilinear")
#only take intermediate features, not overall output
x = functional.interpolate(results[0], size=(100,100), mode="bilinear")
for i in range(1,len(results)):
#upsample (rescale to same size)
intermediate = functional.interpolate(results[i], size=(100,100), mode="bilinear")
#append to other output along feature channel dimension
x = torch.cat((x,intermediate), 1)
#return combined activations
return x
class CenterBiasSchuett(nn.Module):
def __init__(self, gpu=False):
super(CenterBiasSchuett, self).__init__()
self.sigmax = nn.Parameter(torch.Tensor([1000]), requires_grad=False)
self.sigmay = nn.Parameter(torch.Tensor([1000]), requires_grad=False)
self.gpu = gpu
def forward(self, x):
#eg input dimension of 100: tensor with entries 0-99
gridx = torch.Tensor(range(x.size()[-2]))
gridy = torch.Tensor(range(x.size()[-1]))
if gpu:
if torch.cuda.is_available():
gridx = gridy.to('cuda')
gridy = gridy.to('cuda')
gridx = gridx - torch.mean(gridx)
gridx = gridx ** 2
gridx = gridx / (self.sigmax ** 2)
gridx = gridx.view(gridx.size()[0],1)
gridy = gridy - torch.mean(gridy)
gridy = gridy ** 2
gridy = gridy / (self.sigmay ** 2)
grid = gridx + gridy
CB = torch.exp(-0.5*grid)
CB = CB / torch.sum(CB)
return x * CB
class Smoothing(nn.Module):
def __init__(self, gpu=False):
super(Smoothing, self).__init__()
self.conv = nn.Conv2d(1, 1, 3, bias=False, padding=1)
self.gpu = gpu
self.conv.weight = torch.nn.Parameter(gaussian_map(torch.rand(3,3), 1, 1, self.gpu).view(1,1,3,3), requires_grad=False)
print(self.conv.weight.size())
def forward(self, x):
return self.conv(x)
class TestNet(nn.Module):
def __init__(self, gpu=False):
super(TestNet, self).__init__()
self.vgg = MyVGG19()
#reduce the 576 (512 + 64) channels before upsampling (see forward function) to prevent memory problems
#self.red_ch = nn.Conv2d(512, 256, 1)
#3 input image channels (color-images), 64 output channels, 3x3 square convolution kernel
#padding to keep dimensions of output at 100x100
#self.convfirst = nn.Conv2d(576,1,1)
#self.bnfirst = nn.BatchNorm2d(1)
#self.convsecond = nn.Conv2d(1,1,1)
self.conv_1 = nn.Conv2d(2560,16,1)
self.conv_2 = nn.Conv2d(16,32,1)
self.conv_3 = nn.Conv2d(32,2,1)
self.conv_4 = nn.Conv2d(2,1,1)
self.cb = CenterBiasSchuett()
self.smooth = Smoothing()
self.gpu = gpu
if gpu:
if torch.cuda.is_available():
device = torch.device("cuda")
self.cuda()
def forward(self, x):
#print("input sum at beginning of forward pass: {}".format(torch.sum(x)))
x = self.vgg(x)
#x = functional.relu(self.convfirst(x))
#x = self.bnfirst(x)
#x = self.convsecond(x)
#print("input sum after vgg: {}".format(torch.sum(x)))
x = functional.relu(self.conv_1(x))
#print("input sum after 1. conv and relu: {}".format(torch.sum(x)))
x = functional.relu(self.conv_2(x))
#print("input sum after 2. conv and relu: {}".format(torch.sum(x)))
x = functional.relu(self.conv_3(x))
#print("input sum after 3. conv and relu: {}".format(torch.sum(x)))
x = self.conv_4(x)
#print("input sum after 4. conv: {}".format(torch.sum(x)))
x = self.cb(x)
#flattening
x = x.view(batch_size, -1)
x = functional.softmax(x)
#print("input sum after CB: {}".format(torch.sum(x)))
#x = self.smooth(x)
#print("input sum after Smoothing: {}".format(torch.sum(x)))
#softmax to obtain probability distribution
#x = nn.Softmax(2)(x.view(*x.size()[:2], -1)).view_as(x)
#print("input sum after Softmax: {}".format(torch.sum(x)))
#print("input shape after Softmax: {}".format(x.size()))
return x
#initilaize the NN
model = TestNet(gpu)
#initialize visdom-line-plotter-instances
plotter_train = VisdomLinePlotter(env_name='training', server="http://130.63.188.108", port=9876)
plotter_eval = VisdomLinePlotter(env_name='evaluation', server="http://130.63.188.108", port=9876)
# In[29]:
#optimizer with learning rate
optimizer = optim.Adam(model.parameters(), lr=lr)
#lr-scheduler
#lambda1 = lambda epoch: 0.9 ** epoch
#scheduler = LambdaLR(optimizer, lr_lambda=lambda1)
#if lr is 0.2 in the beginning, after 60 epochs it will decrease to 0.02 with gamma = 0.1
#scheduler = StepLR(optimizer, step_size=15, gamma=0.5)
class Cross_Entropy(nn.Module):
def __init__(self):
super(Cross_Entropy, self).__init__()
def forward(self, output, target):
return -torch.sum(target * torch.log(output))
#Cross Entropy Loss (Combining Softmax and NLLLoss)
#criterion = nn.CrossEntropyLoss(reduction='mean')
criterion = Cross_Entropy()
# In[30]:
def train_model(model, batch_size, patience, n_epochs, gpu, plotter_train, plotter_eval):
# to track the training loss as the model trains
train_losses = []
# to track the validation loss as the model trains
valid_losses = []
# to track the average training loss per epoch as the model trains
avg_train_losses = []
# to track the average validation loss per epoch as the model trains
avg_valid_losses = []
# initialize the early_stopping object
early_stopping = EarlyStopping(patience=patience, verbose=True)
for epoch in range(1, n_epochs + 1):
###################
# train the model #
###################
#schedule LR
#scheduler.step()
model.train() # prep model for training
t = tqdm(iter(train_loader), desc="[Train on Epoch {}/{}]".format(epoch, n_epochs))
for i, example in enumerate(t): #start at index 0
# get the inputs
data = example["image"]
#print("data size: {}".format(data.size()))
target = example["fixations"]
#print("target size: {}".format(target.size()))
# clear the gradients of all optimized variables
optimizer.zero_grad()
#push data and targets to gpu
if gpu:
if torch.cuda.is_available():
data = data.to('cuda')
target = target.to('cuda')
#print(resnet18(data).size())
# forward pass: compute predicted outputs by passing inputs to the model
output = model(data)
#print(output.size())
#drop channel-dimension (is only 1) so that outputs will be of same size as targets (batch_size,100,100)
#infer batch dimension as last batch won't have the full size of eg 128
#output = output.view(-1, target.size()[-1], target.size()[-2])
#print("output size: {}".format(output.size()))
# calculate the loss
#loss = myLoss(output, target)
#flatten targets (normalization happened through transform)
target = target.view(batch_size, -1)
loss = criterion(output, target)
# backward pass: compute gradient of the loss with respect to model parameters
loss.backward()
# perform a single optimization step (parameter update)
optimizer.step()
# record training loss
train_losses.append(loss.item())
#print("On iteration {} loss is {:.3f}".format(i+1, loss.item()))
#for the first epoch, plot loss per iteration to have a quick overview of the early training phase
iteration = i + 1
#plot is always appending the newest value, so just give the last item if the list
if epoch == 1:
plotter_train.plot('loss', 'train', 'Loss per Iteration', iteration, train_losses[-1], batch_size, lr, 'iteration')
######################
# validate the model #
######################
model.eval() # prep model for evaluation
t = tqdm(iter(val_loader), desc="[Valid on Epoch {}/{}]".format(epoch, n_epochs))
for i, example in enumerate(t):
# get the inputs
data = example["image"]
#print("input sum: {}".format(torch.sum(data)))
target = example["fixations"]
#push data and targets to gpu
if gpu:
if torch.cuda.is_available():
data = data.to('cuda')
target = target.to('cuda')
#print("target sum: {}".format(torch.sum(target)))
# forward pass: compute predicted outputs by passing inputs to the model
output = model(data)
#drop channel-dimension (is only 1) so that outputs will be of same size as targets (batch_size,100,100)
#output = output.view(-1, target.size()[-2], target.size()[-2])
#flatten targets (normalization happened through transform)
target = target.view(batch_size, -1)
#print("output sum: {}".format(torch.sum(output)))
# calculate the loss
#loss = myLoss(output, target)
loss = criterion(output, target)
# record validation loss
valid_losses.append(loss.item())
#plotter_val.plot('loss', 'val', 'Loss per Iteration', iteration, valid_losses[-1])
# print training/validation statistics
# calculate average loss over an epoch
train_loss = np.average(train_losses)
valid_loss = np.average(valid_losses)
avg_train_losses.append(train_loss)
avg_valid_losses.append(valid_loss)
epoch_len = str(n_epochs)
print_msg = ('[{}/{}] '.format(epoch, epoch_len) +
'train_loss: {:.15f} '.format(train_loss) +
'valid_loss: {:.15f}'.format(valid_loss))
print(print_msg)
#plot average loss for this epoch
plotter_eval.plot('loss', 'train', 'Loss per Epoch', epoch, train_loss, batch_size, lr, 'epoch')
plotter_eval.plot('loss', 'val', 'Loss per Epoch', epoch, valid_loss, batch_size, lr, 'epoch')
# clear lists to track next epoch
train_losses = []
valid_losses = []
# early_stopping needs the validation loss to check if it has decresed,
# and if it has, it will make a checkpoint of the current model
early_stopping(valid_loss, model, batch_size, lr)
if early_stopping.early_stop:
print("Early stopping")
break
# load the last checkpoint with the best model
name = "checkpoint_batch_size_{}_lr_{}.pt".format(batch_size, lr)
model.load_state_dict(torch.load(name))
return model, avg_train_losses, avg_valid_losses
print('Starting to load results data')
#load df to store results in
results = pd.read_csv("results.csv")
#create temporary df (its contents will be appended to "results.csv")
results_cur = pd.DataFrame(columns = ["batch_size", "n_epochs", "learning_rate", "mean_accuracy_per_image", "mean_test_loss", "mean_validation_loss", "mean_train_loss", "number_of_hits", "number_of_test_images", "number_of_fixations"])
# In[31]:
#call the training/validation loop
batch_size #= 32
n_epochs #= 10
print('Create data sets')
train_loader, val_loader, test_loader = create_datasets(batch_size)
# early stopping patience; how long to wait after last time validation loss improved.
patience = 50
print('Start trainging')
model, train_loss, valid_loss = train_model(model, batch_size, patience, n_epochs, gpu, plotter_train, plotter_eval)
# In[ ]:
# visualize the loss as the network trained
#turn interactive mode off, because plot cannot be displayed in console
plt.ioff()
fig = plt.figure(figsize=(10,8))
plt.plot(range(1,len(train_loss)+1),train_loss, label='Training Loss')
plt.plot(range(1,len(valid_loss)+1),valid_loss,label='Validation Loss')
# find position of lowest validation loss
minposs = valid_loss.index(min(valid_loss))+1
plt.axvline(minposs, linestyle='--', color='r',label='Lowest Validation Loss')
plt.xlabel('epochs')
plt.ylabel('loss')
#plt.ylim(0, 0.5) # consistent scale
plt.xlim(0, len(train_loss)+1) # consistent scale
plt.grid(True)
plt.legend()
plt.title("Training and Validation Loss per Epoch", fontsize=20)
plt.tight_layout()
#plt.show() #no showing, only saving
name = "loss_plot_batch_size_{}_lr_{}.png".format(batch_size, lr)
fig.savefig(name, bbox_inches='tight')
# In[ ]:
#evaluate the model
# to track the training loss as the model trains
test_losses = []
#to track the accuracy
acc_per_image = []
acc_per_batch = []
#track absolute hits
hit_list = []
#track number of fixations
n_fixations = []
######################
# evaluate the model #
######################
model.eval() # prep model for evaluation
t = tqdm(iter(test_loader), desc="Evaluating Model")
for i, example in enumerate(t): #start at index 0
# get the inputs
data = example["image"]
#print("input sum: {}".format(torch.sum(data)))
target = example["fixations"]
target_locs = example["fixation_locs"]
#push data and targets to gpu
if gpu:
if torch.cuda.is_available():
data = data.to('cuda')
target = target.to('cuda')
# forward pass: compute predicted outputs by passing inputs to the model
output = model(data)
#drop channel-dimension (is only 1) so that outputs will be of same size as targets (batch_size,100,100)
output = output.view(-1, target.size()[-2], target.size()[-2])
loss = criterion(output, target)
# calculate the loss
#loss = myLoss(output, target)
# record training loss
test_losses.append(loss.item())
#accuracy
acc_this_batch = 0
for batch_idx in range(output.size()[0]):
output_subset = output[batch_idx]
target_subset = target[batch_idx]
target_locs_subset = target_locs[batch_idx]
acc_this_image, hits, num_fix = accuracy(output_subset, target_subset, target_locs_subset, gpu)
acc_per_image.append(acc_this_image)
hit_list.append(hits)
n_fixations.append(num_fix)
acc_this_batch += acc_this_image
#divide by batch size
acc_this_batch /= output.size()[0]
acc_per_batch.append(acc_this_batch)
acc_per_image = np.asarray(acc_per_image)
print("Mean test loss is: {}".format(np.average(test_losses)))
print("Mean accuracy for test set ist: {}".format(np.mean(acc_per_image)))
# In[9]:
#print value of sigmoid scaling parameter (it's the first one, so stop after one iteration)
#k = 0
#for name, param in model.named_parameters():
# if param.requires_grad:
# print(name, param.data)
# if k == 0:
# break
#print(list(model.parameters()))
#store all the information from this hyperparameter configuration
results_cur.loc[0,"batch_size"] = batch_size
results_cur.loc[0, "n_epochs"] = n_epochs
results_cur.loc[0, "learning_rate"] = lr
results_cur.loc[0, "mean_accuracy_per_image"] = np.mean(acc_per_image)
results_cur.loc[0, "mean_test_loss"] = np.average(test_losses)
results_cur.loc[0, "mean_validation_loss"] = np.average(valid_loss)
results_cur.loc[0, "mean_train_loss"] = np.average(train_loss)
results_cur.loc[0, "number_of_hits"] = sum(hit_list)
results_cur.loc[0, "number_of_test_images"] = len(acc_per_image)
results_cur.loc[0, "number_of_fixations"] = sum(n_fixations)
#and append results_cur to results
results = results.append(results_cur)
#store results
results.to_csv("results.csv", index=False, header=True)