def visualize_reordered(self, path, number, shape, permutations):
data = self.visualize_sample(path, number, shape)
data = data.reshape(-1, shape[0] * shape[1] * shape[2])
concat = deepcopy(data)
image_frame_dim = int(np.floor(np.sqrt(number)))
for i in range(1, self.n_tasks):
_, inverse_permutation = permutations[i].sort()
reordered_data = deepcopy(data.index_select(
1, inverse_permutation))
concat = torch.cat((concat, reordered_data), 0)
if shape[2] == 1:
concat = concat.numpy().reshape(number * self.n_tasks, shape[0],
shape[1], shape[2])
save_images(
concat[:image_frame_dim * image_frame_dim *
self.n_tasks, :, :, :],
[self.n_tasks * image_frame_dim, image_frame_dim], path)
else:
concat = concat.numpy().reshape(number * self.n_tasks, shape[2],
shape[1], shape[0])
make_samples_batche(concat[:self.batch_size], self.batch_size,
path)
def sort_sequence(data, len_data):
_, idx_sort = torch.sort(len_data, dim=0, descending=True)
_, idx_unsort = torch.sort(idx_sort, dim=0)
sorted_data = data.index_select(0, idx_sort)
sorted_len = len_data.index_select(0, idx_sort)
return sorted_data, sorted_len.data.cpu().numpy(), idx_unsort
def sortData(length, data, label, cuda):
length, indices = torch.sort(length, dim=0, descending=True)
if cuda:
data = data.cuda()
indices = indices.cuda()
label = label.cuda()
data = data.index_select(1, indices)#[:, indices, :]
label = label.index_select(0, indices)
if not isSorted(length):
print(length)
raise RuntimeError("Batch not sorted!")
return length, data, label
def test():
print('test:')
RMA.eval() # set the module in evaluation mode
test_loss = 0. # sum of train loss up to current batch
global current_test_iteration
sum_prediction_label = torch.zeros(1, 80) + 1e-6
sum_correct_prediction_label = torch.zeros(1, 80)
sum_ground_truth_label = torch.zeros(1, 80)
for batch_num, (data, target) in enumerate(test_loader):
if target.sum() == 0:
continue
target = target.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
data = data.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
if GPU_IN_USE:
data, target = data.cuda(), target.cuda() # set up GPU Tensor
f_I = extract_features(data)
# ten-crop
# f_I: batchsize*channel*inputSize*inputSize
# tencrop_results: 10*batchsize*channel*cropSize*cropSize
tencrop_results = tencrop(f_I, f_I.size(0), f_I.size(1), f_I.size(2), CROPSIZE_512)
RMA_outputs = torch.zeros(target.size())
RMA_losses = 0
tencrop_results = tencrop_results.cuda()
RMA_outputs = RMA_outputs.cuda()
for i in range(10):
crop_RMA_output, crop_RMA_M = RMA(tencrop_results[i])
RMA_outputs += crop_RMA_output
RMA_losses += loss_function(crop_RMA_output, target, crop_RMA_M, add_constraint=True)
output = RMA_outputs * 0.1
loss = RMA_losses * 0.1
# output, M = RMA(f_I)
# loss = loss_function(output, target, M, add_constraint=True)
test_loss += loss
prediction = torch.topk(F.softmax(output, dim=1), 10, dim=1)
filter = prediction[0].eq(0.1) + prediction[0].gt(0.1)
prediction_index = torch.mul(prediction[1]+1, filter.type(torch.cuda.LongTensor))
extend_eye_mat = torch.cat((torch.zeros(1, 80), torch.eye(80)), 0)
prediction_label = extend_eye_mat[prediction_index.view(-1)].view(-1, 10, 80).sum(dim=1)
correct_prediction_label = (target.cpu().byte() & prediction_label.byte()).type(torch.FloatTensor)
#count the sum of label vector
sum_prediction_label += prediction_label.sum(dim=0)
sum_correct_prediction_label += correct_prediction_label.sum(dim=0)
sum_ground_truth_label += target.cpu().sum(dim=0)
#for i in range(0, target.size(0)):
# print('-----------------')
# print('ground-truth: ', target[i].nonzero().view(-1))
# print('prediction: ', prediction_index[i] - 1)
# print('-----------------')
#
if batch_num % LOSS_OUTPUT_INTERVAL == 0:
# visualization: draw the test loss graph
vis.line(
X=current_test_iteration,
Y=torch.tensor([test_loss.data]) / (batch_num+1),
win=loss_graph_window,
name='test loss',
update=None if current_test_iteration == 1 else 'append',
# update='insert' if current_test_iteration == 1 else 'append',
opts=dict(showlegend=True),
)
print('loss %.3f (batch %d)' % (test_loss / (batch_num+1), batch_num+1))
current_test_iteration += LOSS_OUTPUT_INTERVAL
# evaluation metrics
o_p = torch.div(sum_correct_prediction_label.sum(), sum_prediction_label.sum())
o_r = torch.div(sum_correct_prediction_label.sum(), sum_ground_truth_label.sum())
of1 = torch.div(2 * o_p * o_r, o_p + o_r)
c_p = (torch.div(sum_correct_prediction_label, sum_prediction_label)).sum() / NUM_CATEGORIES
c_r = (torch.div(sum_correct_prediction_label, sum_ground_truth_label)).sum() / NUM_CATEGORIES
cf1 = torch.div(2 * c_p * c_r, c_p + c_r)
return c_p, c_r, cf1, o_p, o_r, of1
def train():
print('train:')
RMA.train() # set the module in training mode
train_loss = 0. # sum of train loss up to current batch
global current_training_iteration
sum_prediction_label = torch.zeros(1, 80) + 1e-6
sum_correct_prediction_label = torch.zeros(1, 80)
sum_ground_truth_label = torch.zeros(1, 80)
for batch_num, (data, target) in enumerate(train_loader):
if target.sum() == 0:
continue
target = target.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
data = data.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
if GPU_IN_USE:
data, target = data.cuda(), target.cuda()
# -----forward-----
optimizer.zero_grad()
f_I = extract_features(data)
output, M = RMA(f_I)
# ---end forward---
# ---calculate loss and backward---
loss = loss_function(output, target, M, add_constraint=True)
loss.backward()
optimizer.step()
# ----------end backward-----------
train_loss += loss
prediction = torch.topk(F.softmax(output, dim=1), 10, dim=1)
filter = prediction[0].eq(0.1) + prediction[0].gt(0.1)
prediction_index = torch.mul(prediction[1]+1, filter.type(torch.cuda.LongTensor))
extend_eye_mat = torch.cat((torch.zeros(1, 80), torch.eye(80)), 0)
prediction_label = extend_eye_mat[prediction_index.view(-1)].view(-1, 10, 80).sum(dim=1)
correct_prediction_label = (target.cpu().byte() & prediction_label.byte()).type(torch.FloatTensor)
#count the sum of label vector
sum_prediction_label += prediction_label.sum(dim=0)
sum_correct_prediction_label += correct_prediction_label.sum(dim=0)
sum_ground_truth_label += target.cpu().sum(dim=0)
#for i in range(0, target.size(0)):
# print('-----------------')
# print('ground-truth: ', target[i].nonzero().view(-1))
# print('prediction: ', prediction[1][i])
# print('-----------------')
if batch_num % LOSS_OUTPUT_INTERVAL == 0:
# visualization: draw the train loss graph
vis.line(
X=current_training_iteration,
Y=torch.tensor([train_loss.data]) / (batch_num+1),
win=loss_graph_window,
name='train loss',
update=None if current_training_iteration == 1 else 'append',
opts=dict(xlabel='iteration', ylabel='loss', showlegend=True)
)
print('loss %.3f (batch %d)' % (train_loss/(batch_num+1), batch_num+1))
current_training_iteration += LOSS_OUTPUT_INTERVAL
# evaluation metrics
o_p = torch.div(sum_correct_prediction_label.sum(), sum_prediction_label.sum())
o_r = torch.div(sum_correct_prediction_label.sum(), sum_ground_truth_label.sum())
of1 = torch.div(2 * o_p * o_r, o_p + o_r)
c_p = (torch.div(sum_correct_prediction_label, sum_prediction_label)).sum() / NUM_CATEGORIES
c_r = (torch.div(sum_correct_prediction_label, sum_ground_truth_label)).sum() / NUM_CATEGORIES
cf1 = torch.div(2 * c_p * c_r, c_p + c_r)
return c_p, c_r, cf1, o_p, o_r, of1
def test():
print('test:')
#RMA.eval() # set the module in evaluation mode
sum_prediction_label = torch.zeros(1, 80) + 1e-6
sum_correct_prediction_label = torch.zeros(1, 80)
sum_ground_truth_label = torch.zeros(1, 80)
for batch_num, (data, target, original_imgs) in enumerate(test_loader):
if target.sum() == 0:
continue
target = target.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
data = data.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
original_imgs = original_imgs.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
#print('original_imgs ', original_imgs.size())
if GPU_IN_USE:
data, target = data.cuda(), target.cuda() # set up GPU Tensor
f_I = extract_features(data)
output, M, scores = RMA(f_I, return_whole_scores=True)
#total_thetas.append(M)
#total_scores.append(scores)
#visualize_attentional_regions(original_imgs, M[1:, :, :, :], scores)
visualize_attentional_regions(original_imgs, M, scores)
prediction = torch.topk(F.softmax(output, dim=1), 10, dim=1)
filter = prediction[0].eq(0.1) + prediction[0].gt(0.1)
prediction_index = torch.mul(prediction[1]+1, filter.type(torch.cuda.LongTensor))
extend_eye_mat = torch.cat((torch.zeros(1, 80), torch.eye(80)), 0)
prediction_label = extend_eye_mat[prediction_index.view(-1)].view(-1, 10, 80).sum(dim=1)
correct_prediction_label = (target.cpu().byte() & prediction_label.byte()).type(torch.FloatTensor)
#count the sum of label vector
sum_prediction_label += prediction_label.sum(dim=0)
sum_correct_prediction_label += correct_prediction_label.sum(dim=0)
sum_ground_truth_label += target.cpu().sum(dim=0)
#for i in range(0, target.size(0)):
# print('-----------------')
# print('ground-truth: ', target[i].nonzero().view(-1))
# print('prediction: ', prediction_index[i] - 1)
# print('-----------------')
if batch_num % OUTPUT_INTERVAL == 0:
print(batch_num)
#print('loss %.3f (batch %d)' % (test_loss / (batch_num+1), batch_num+1))
#evaluation metrics
o_p = torch.div(sum_correct_prediction_label.sum(), sum_prediction_label.sum())
o_r = torch.div(sum_correct_prediction_label.sum(), sum_ground_truth_label.sum())
of1 = torch.div(2 * o_p * o_r, o_p + o_r)
c_p = (torch.div(sum_correct_prediction_label, sum_prediction_label)).sum() / NUM_CATEGORIES
c_r = (torch.div(sum_correct_prediction_label, sum_ground_truth_label)).sum() / NUM_CATEGORIES
cf1 = torch.div(2 * c_p * c_r, c_p + c_r)
print('-------------------------------------------------------------')
print('| CP | CR | CF1 | OP | OR | OF1 |')
print('-------------------------------------------------------------')
print('| %.3f | %.3f | %.3f | %.3f | %.3f | %.3f |' % (c_p, c_r, cf1, o_p, o_r, of1))
print('-------------------------------------------------------------')
def test():
print('test:')
IGCMAN.eval() # set the module in evaluation mode
test_loss = 0. # sum of train loss up to current batch
test_correct = 0
total = 0
sum_prediction_label = torch.zeros(1, NUM_INGREDIENT) + 1e-6
sum_correct_prediction_label = torch.zeros(1, NUM_INGREDIENT)
sum_ground_truth_label = torch.zeros(1, NUM_INGREDIENT)
for batch_num, (data, target, category) in enumerate(test_loader):
if target.sum() == 0:
continue
target = target.index_select(0,
torch.nonzero(target.sum(dim=1)).view(-1))
data = data.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
if GPU_IN_USE:
data, target = data.cuda(), target.cuda() # set up GPU Tensor
category = category.cuda()
# f_I = extract_features(data)
output, M, score_category = IGCMAN(data)
loss = loss_function(output,
target,
M,
score_category,
category,
add_constraint=True)
test_loss += loss
prediction = torch.topk(F.softmax(output, dim=1), 10, dim=1)
filter = prediction[0].eq(0.1) + prediction[0].gt(0.1)
prediction_index = torch.mul(prediction[1] + 1,
filter.type(torch.cuda.LongTensor))
extend_eye_mat = torch.cat(
(torch.zeros(1, NUM_INGREDIENT), torch.eye(NUM_INGREDIENT)), 0)
prediction_label = extend_eye_mat[prediction_index.view(-1)].view(
-1, 10, NUM_INGREDIENT).sum(dim=1)
correct_prediction_label = (target.cpu().byte()
& prediction_label.byte()).type(
torch.FloatTensor)
#count the sum of label vector
sum_prediction_label += prediction_label.sum(dim=0)
sum_correct_prediction_label += correct_prediction_label.sum(dim=0)
sum_ground_truth_label += target.cpu().sum(dim=0)
# # calculate accuracy
_, test_predict = torch.max(score_category, 1)
total += BATCH_SIZE
test_correct += torch.sum(test_predict == category.data)
test_acc = float(test_correct) / total
if batch_num % LOSS_OUTPUT_INTERVAL == 0:
print('Test loss %.3f (batch %d) accuracy %0.3f' %
(test_loss / (batch_num + 1), batch_num + 1, test_acc))
# evaluation metrics
o_p = torch.div(sum_correct_prediction_label.sum(),
sum_prediction_label.sum())
o_r = torch.div(sum_correct_prediction_label.sum(),
sum_ground_truth_label.sum())
of1 = torch.div(2 * o_p * o_r, o_p + o_r)
c_p = (torch.div(sum_correct_prediction_label,
sum_prediction_label)).sum() / NUM_INGREDIENT
c_r = (torch.div(sum_correct_prediction_label,
sum_ground_truth_label)).sum() / NUM_INGREDIENT
cf1 = torch.div(2 * c_p * c_r, c_p + c_r)
return c_p, c_r, cf1, o_p, o_r, of1
def train():
print('train:')
IGCMAN.train() # set the module in training mode
train_loss = 0. # sum of train loss up to current batch
train_correct = 0
total = 0
sum_prediction_label = torch.zeros(1, NUM_INGREDIENT) + 1e-6
sum_correct_prediction_label = torch.zeros(1, NUM_INGREDIENT)
sum_ground_truth_label = torch.zeros(1, NUM_INGREDIENT)
for batch_num, (data, target, category) in enumerate(train_loader):
if target.sum() == 0:
continue
target = target.index_select(0,
torch.nonzero(target.sum(dim=1)).view(-1))
data = data.index_select(0, torch.nonzero(target.sum(dim=1)).view(-1))
if GPU_IN_USE:
data, target = data.cuda(), target.cuda()
category = category.cuda()
data = torch.autograd.Variable(data)
target = torch.autograd.Variable(target)
category = torch.autograd.Variable(category)
# -----forward-----
optimizer.zero_grad()
output, M, score_category = IGCMAN(data)
# ---end forward---
# ---calculate loss and backward---
loss = loss_function(output,
target,
M,
score_category,
category,
add_constraint=True)
loss.backward()
optimizer.step()
# ----------end backward-----------
train_loss += loss
prediction = torch.topk(
F.softmax(output, dim=1), 10,
dim=1) # return the max value and the index tuple
filter = prediction[0].eq(0.1) + prediction[0].gt(0.1)
prediction_index = torch.mul(prediction[1] + 1,
filter.type(torch.cuda.LongTensor))
extend_eye_mat = torch.cat(
(torch.zeros(1, NUM_INGREDIENT), torch.eye(NUM_INGREDIENT)), 0)
prediction_label = extend_eye_mat[prediction_index.view(-1)].view(
-1, 10, NUM_INGREDIENT).sum(dim=1)
correct_prediction_label = (target.cpu().byte()
& prediction_label.byte()).type(
torch.FloatTensor)
#count the sum of label vector
sum_prediction_label += prediction_label.sum(dim=0)
sum_correct_prediction_label += correct_prediction_label.sum(dim=0)
sum_ground_truth_label += target.cpu().sum(dim=0)
# # calculate accuracy
_, train_predict = torch.max(score_category, 1)
total += BATCH_SIZE
train_correct += torch.sum(train_predict == category.data)
train_acc = float(train_correct) / total
if batch_num % LOSS_OUTPUT_INTERVAL == 0:
print(
'train loss %.3f (batch %d) accuracy %0.3f' %
(train_loss / (batch_num + 1), batch_num + 1), train_acc)
# evaluation metrics
o_p = torch.div(sum_correct_prediction_label.sum(),
sum_prediction_label.sum())
o_r = torch.div(sum_correct_prediction_label.sum(),
sum_ground_truth_label.sum())
of1 = torch.div(2 * o_p * o_r, o_p + o_r)
c_p = (torch.div(sum_correct_prediction_label,
sum_prediction_label)).sum() / NUM_INGREDIENT
c_r = (torch.div(sum_correct_prediction_label,
sum_ground_truth_label)).sum() / NUM_INGREDIENT
cf1 = torch.div(2 * c_p * c_r, c_p + c_r)
return c_p, c_r, cf1, o_p, o_r, of1