def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acces = AverageMeter()
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, input_mask, target,
meta) in enumerate(val_loader):
# if RELABEL and i == 2 : break
# measure data loading time
data_time.update(time.time() - end)
input, input_mask, target = input.to(device), input_mask.to(
device), target.to(device, non_blocking=True)
input_depth = input_depth.to(device)
batch_size = input.shape[0]
loss = 0
last_state = None
acc_list = []
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
input_mask_now = input_mask[:, j]
target_now = target[:, j]
if j == 0:
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now),
1))
else:
if LSTM_STATE == 'stateness':
output, output_state = model(
torch.cat(
(input_now, input_depth_now, input_mask_now),
1))
elif LSTM_STATE == 'stateful':
output, output_state = model(torch.cat(
(input_now, input_depth_now, input_mask_now), 1),
input_state=last_state)
last_state = output_state
# print(output.shape)
round_output = torch.round(output).float()
loss += criterion(output, target_now)
temp_acc = float(
(round_output == target_now).sum()) / batch_size
acc_list.append(temp_acc)
round_output = round_output.cpu()
# print(round_output)
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-5:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
gt_label_str = None
pred_label_str = None
if target_now[0][0] == 0:
gt_label_str = "GT : False"
elif target_now[0][0] == 1:
gt_label_str = "GT : True"
if round_output[0][0] == 0:
pred_label_str = "Pred : False"
elif round_output[0][0] == 1:
pred_label_str = "Pred : True"
output_str = gt_label_str + '. ' + pred_label_str
# if target_now[0][0] != round_output[0][0] :
# print(raw_rgb_frame_path)
if not gt_win:
plt.plot()
plt.title(output_str)
gt_win = plt.imshow(gt_mask_rgb)
else:
plt.title(output_str)
gt_win.set_data(gt_mask_rgb)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
acces.update(sum(acc_list) / len(acc_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
return losses.avg, acces.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ioues = AverageMeter()
# predictions
predictions = torch.Tensor(val_loader.dataset.__len__(), num_classes, 2)
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, target, meta) in enumerate(val_loader):
if RELABEL and i == 1: break
# measure data loading time
data_time.update(time.time() - end)
input = input.to(device, non_blocking=True)
input_depth = input_depth.to(device, non_blocking=True)
target = target.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
iou_list = []
# store first two stack feature # 2 = first and second stack, 6 = video length
video_feature_cache = torch.zeros(batch_size, 6, 2, 256,
output_res, output_res)
# first compute
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
_, out_tsm_feature = model(
torch.cat((input_now, input_depth_now),
1)) # [B, 4, 256, 256]
for k in range(2):
video_feature_cache[:, j, k] = out_tsm_feature[k]
# TSM module
b, t, _, c, h, w = video_feature_cache.size()
fold_div = 8
fold = c // fold_div
new_tsm_feature = torch.zeros(batch_size, 6, 2, 256, output_res,
output_res)
for j in range(2):
x = video_feature_cache[:, :, j]
temp = torch.zeros(batch_size, 6, 256, output_res, output_res)
temp[:, :-1, :fold] = x[:, 1:, :fold] # shift left
temp[:, 1:, fold:2 * fold] = x[:, :-1,
fold:2 * fold] # shift right
temp[:, :, 2 * fold:] = x[:, :, 2 * fold:] # not shift
new_tsm_feature[:, :, j] = temp
new_tsm_feature = new_tsm_feature.to(device)
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
output, _ = model(torch.cat((input_now, input_depth_now), 1),
True, new_tsm_feature[:, j])
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now)
output = output[-1]
else: # single output
pass
'''
testing now
'''
output = output.cpu()
target_now = target_now.cpu()
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
area_head = '/home/s5078345/Affordance-Detection-on-Video/faster-rcnn.pytorch/data_affordance_bbox'
area_tail = ('/').join(raw_mask_path.split('/')[6:10])
area_to_detect_data_path = os.path.join(
area_head, area_tail, relabel_mask_name[:-4] + '.txt')
area_to_detect_list = []
with open(area_to_detect_data_path) as f:
for line in f:
inner_list = [
int(elt.strip()) for elt in line.split(' ')
]
# in alternative, if you need to use the file content as numbers
# inner_list = [int(elt.strip()) for elt in line.split(',')]
area_to_detect_list.append(inner_list)
if len(area_to_detect_list) == 0:
area_to_detect_list = None
area_to_detect = area_to_detect_list
if area_to_detect is not None:
out_resized_area = torch.zeros((1, 1, 64, 64))
gt_resized_area = torch.zeros((1, 1, 64, 64))
for _i in range(len(area_to_detect)):
x_min, y_min, x_max, y_max = area_to_detect[_i]
x_min = math.floor(x_min / 640 * 64)
y_min = math.floor(y_min / 480 * 64)
x_max = math.ceil(x_max / 640 * 64)
y_max = math.ceil(y_max / 480 * 64)
# clip pred
out_resized_area[0, 0, y_min:y_max,
x_min:x_max] = output[0, 0,
y_min:y_max,
x_min:x_max]
# clip GT
gt_resized_area[0, 0, y_min:y_max,
x_min:x_max] = target_now[0, 0,
y_min:y_max,
x_min:x_max]
output = out_resized_area
target_now = gt_resized_area
else:
pass
temp_iou = intersectionOverUnion(output.cpu(),
target_now.cpu(),
idx) # have not tested
iou_list.append(temp_iou)
score_map = output[-1].cpu() if type(
output) == list else output.cpu()
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
# print(relabel_mask_dir)
# print(relabel_mask_name)
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# print(input_now.shape)
# print(score_map) # [1, 1, 64, 64]
# 2020.7.1
# load info about hide area
# from faster rcnn
area_head = '/home/s5078345/Affordance-Detection-on-Video/faster-rcnn.pytorch/data_affordance_bbox'
area_tail = ('/').join(raw_mask_path.split('/')[6:10])
area_to_detect_data_path = os.path.join(
area_head, area_tail, relabel_mask_name[:-4] + '.txt')
# print(area_to_use_data_path)
area_to_detect_list = []
with open(area_to_detect_data_path) as f:
for line in f:
inner_list = [
int(elt.strip()) for elt in line.split(' ')
]
# in alternative, if you need to use the file content as numbers
# inner_list = [int(elt.strip()) for elt in line.split(',')]
area_to_detect_list.append(inner_list)
if len(area_to_detect_list) == 0:
area_to_detect_list = None
# print(area_to_detect_list)
pred_batch_img, pred_mask = relabel_heatmap(
input_now,
score_map,
'pred',
area_to_detect=area_to_detect_list
) # return an Image object
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text('MASK_RGB_GT')
gt_win = plt.imshow(gt_mask_rgb)
ax2 = plt.subplot(122)
ax2.title.set_text('Mask_RGB_PRED')
pred_win = plt.imshow(pred_batch_img)
else:
gt_win.set_data(gt_mask_rgb)
pred_win.set_data(pred_batch_img)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
pred_mask.save(os.path.join(relabel_mask_dir, index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# acces.update(acc[0], input.size(0))
ioues.update(sum(iou_list) / len(iou_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
return losses.avg, ioues.avg, predictions
def train(train_loader, model, criterion, optimizer, debug=False, flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
for i, (input, input_depth, input_mask, target,
meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_mask, target = input.to(device), input_mask.to(
device), target.to(device, non_blocking=True)
input_depth = input_depth.to(device)
# target_weight = meta['target_weight'].to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
last_state = None
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
input_mask_now = input_mask[:, j]
target_now = target[:, j]
if j == 0:
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now), 1))
else:
if LSTM_STATE == 'stateness':
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now),
1))
elif LSTM_STATE == 'stateful':
output, output_state = model(torch.cat(
(input_now, input_depth_now, input_mask_now), 1),
input_state=last_state)
last_state = output_state
loss += criterion(output, target_now)
# print(loss.item())
# # measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
)
bar.next()
bar.finish()
return losses.avg
def validate(val_loader,
model,
criterions,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
total_losses = AverageMeter()
heatmap_losses = AverageMeter()
mask_losses = AverageMeter()
label_losses = AverageMeter()
heatmap_ioues = AverageMeter()
mask_ioues = AverageMeter()
label_acces = AverageMeter()
# iou > 50% and step 2 labels are both right -> correcct
# if label is false (and pred is false too) -> correct
final_acces = AverageMeter()
# for statistic
gt_trues = AverageMeter() # positive
gt_falses = AverageMeter() # negative
pred_trues = AverageMeter() # true == true and iou > 50%
pred_falses = AverageMeter()
pred_trues_first = AverageMeter() # true == true
# Loss
criterion_iou, criterion_bce, criterion_focal = criterions
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, target_heatmap, target_mask, target_label,
meta) in enumerate(val_loader):
# if RELABEL and i == 10 : break
# measure data loading time
data_time.update(time.time() - end)
input, input_depth = input.to(device), input_depth.to(device)
target_heatmap, target_mask, target_label = target_heatmap.to(device, non_blocking=True), target_mask.to(device, non_blocking=True), \
target_label.to(device, non_blocking=True)
batch_size = input.shape[0]
total_loss, heatmap_loss, mask_loss, label_loss = 0, 0, 0, 0
last_state = None
last_tsm_buffer = None
heatmap_iou_list, mask_iou_list, label_acc_list = [], [], []
final_acc_list = []
# for statistic
gt_true_list, gt_false_list, pred_true_list, pred_false_list, pred_true_first_list = [], [], [], [], []
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j] # [B, 1, 256, 256]
target_heatmap_now = target_heatmap[:, j] # [B, 1, 64, 64]
target_mask_now = target_mask[:, j] # [B, 1, 64, 64]
target_label_now = target_label[:, j] # [B, 1]
# print(target_label_now)
if j == 0:
output_heatmap, output_mask, output_label, output_state, output_tsm = model(
torch.cat((input_now, input_depth_now), 1))
else:
output_heatmap, output_mask, output_label, output_state, output_tsm = model(torch.cat((input_now, input_depth_now), 1), \
input_state = last_state, tsm_input = last_tsm_buffer)
last_state = output_state
last_tsm_buffer = output_tsm
# Loss computation
for o_heatmap in output_heatmap:
temp = criterion_iou(
o_heatmap, target_heatmap_now) * 0.05 + criterion_bce(
o_heatmap, target_heatmap_now) * 0.05
total_loss += temp
heatmap_loss += temp
for o_mask in output_mask:
# temp = criterion_iou(o_mask, target_mask_now) + criterion_bce(o_mask, target_mask_now)
# temp = criterion_focal(o_mask, target_mask_now) # exp 1
temp = criterion_focal(
o_mask, target_mask_now) + criterion_iou(
o_mask, target_mask_now) # exp 2
total_loss += temp
mask_loss += temp
temp = criterion_bce(output_label, target_label_now)
total_loss += temp
label_loss += temp
# choose last one as prediction
output_heatmap = output_heatmap[-1]
output_mask = output_mask[-1]
# evaluation metric
heatmap_iou = intersectionOverUnion(output_heatmap.cpu(),
target_heatmap_now.cpu(),
idx) # experiemnt
heatmap_iou_list.append(heatmap_iou)
mask_iou = intersectionOverUnion(output_mask.cpu(),
target_mask_now.cpu(),
idx,
return_list=True)
mask_iou_list.append((sum(mask_iou) / len(mask_iou))[0])
round_output_label = torch.round(output_label).float()
label_acc = float((round_output_label
== target_label_now).sum()) / batch_size
label_acc_list.append(label_acc)
score_map_mask = output_mask.cpu()
# score_map_mask = output_heatmap.cpu()
# print((sum(mask_iou) / len(mask_iou))[0])
#########################
# final evuation accuracy
import numpy as np
temp_1 = (round_output_label == 1) & (
target_label_now == 1) # positve label correct
temp_acc_1 = temp_1.cpu().numpy()
temp_2 = (round_output_label == 0) & (
target_label_now == 0) # negative label correct
temp_acc_2 = temp_2.cpu().numpy()
final_pred_1 = np.logical_and(
temp_acc_1,
mask_iou > 0.5) # positve label correct + iou > 50%
final_pred_2 = temp_acc_2 # negative label correct
final_acc = np.logical_or(final_pred_1, final_pred_2)
final_acc_list.append(np.sum(final_acc) / batch_size)
# for statistic
temp_1 = (target_label_now == 1).cpu().numpy()
temp_2 = (target_label_now == 0).cpu().numpy()
gt_true_list.append(np.sum(temp_1) / batch_size)
gt_false_list.append(np.sum(temp_2) / batch_size)
pred_true_list.append(np.sum(final_pred_1) / batch_size)
pred_false_list.append(np.sum(final_pred_2) / batch_size)
pred_true_first_list.append(np.sum(temp_acc_1) / batch_size)
###############################
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
# print(relabel_mask_dir)
# print(relabel_mask_name)
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# print(input_now.shape)
# print(score_map.shape)
pred_batch_img, pred_mask = relabel_heatmap(
input_now.cpu(), score_map_mask,
'pred') # return an Image object
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
gt_label_str = None
pred_label_str = None
from PIL import Image
pred_Image = Image.fromarray(pred_batch_img)
if target_label_now[0][0] == 0:
gt_label_str = "GT : False"
elif target_label_now[0][0] == 1:
gt_label_str = "GT : True"
# print(gt_label_str)
if round_output_label[0][0] == 0:
pred_label_str = "Pred : False"
elif round_output_label[0][0] == 1:
pred_label_str = "Pred : True"
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text(gt_label_str)
gt_win = plt.imshow(gt_mask_rgb)
ax2 = plt.subplot(122)
ax2.title.set_text(pred_label_str)
pred_win = plt.imshow(pred_batch_img)
else:
gt_win.set_data(gt_mask_rgb)
pred_win.set_data(pred_batch_img)
ax1.title.set_text(gt_label_str)
ax2.title.set_text(pred_label_str)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
pred_mask.save(os.path.join(relabel_mask_dir, index_name))
pred_Image.save(
os.path.join(relabel_mask_dir, 'image_' + index_name))
# print(os.path.join(relabel_mask_dir, index_name))
# record final acc
final_acces.update(
sum(final_acc_list) / len(final_acc_list), input.size(0))
# for statistic
gt_trues.update(
sum(gt_true_list) / len(gt_true_list), input.size(0))
gt_falses.update(
sum(gt_false_list) / len(gt_false_list), input.size(0))
pred_trues.update(
sum(pred_true_list) / len(pred_true_list), input.size(0))
pred_falses.update(
sum(pred_false_list) / len(pred_false_list), input.size(0))
pred_trues_first.update(
sum(pred_true_first_list) / len(pred_true_first_list),
input.size(0))
# record loss
total_losses.update(total_loss.item(), input.size(0))
heatmap_losses.update(heatmap_loss.item(), input.size(0))
mask_losses.update(mask_loss.item(), input.size(0))
label_losses.update(label_loss.item(), input.size(0))
# record metric
heatmap_ioues.update(
sum(heatmap_iou_list) / len(heatmap_iou_list), input.size(0))
mask_ioues.update(
sum(mask_iou_list) / len(mask_iou_list), input.size(0))
label_acces.update(
sum(label_acc_list) / len(label_acc_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=total_losses.avg)
bar.next()
bar.finish()
print(heatmap_losses.avg, heatmap_ioues.avg, \
mask_losses.avg, mask_ioues.avg, \
label_losses.avg, label_acces.avg, \
final_acces.avg)
return heatmap_losses.avg, heatmap_ioues.avg, \
mask_losses.avg, mask_ioues.avg, \
label_losses.avg, label_acces.avg, \
final_acces.avg
def train(train_loader, model, criterion, optimizer, debug=False, flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
for i, (input, input_depth, target, meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_depth, target = input.to(device), input_depth.to(
device), target.to(device, non_blocking=True)
# target_weight = meta['target_weight'].to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
# store first two stack feature # 2 = first and second stack, 6 = video length
video_feature_cache = torch.zeros(batch_size, 6, 2, 256, output_res,
output_res)
with torch.no_grad():
# first compute
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
_, out_tsm_feature = model(
torch.cat((input_now, input_depth_now),
1)) # [B, 4, 256, 256]
for k in range(2):
video_feature_cache[:, j, k] = out_tsm_feature[k]
# TSM module
b, t, _, c, h, w = video_feature_cache.size()
fold_div = 8
fold = c // fold_div
new_tsm_feature = torch.zeros(batch_size, 6, 2, 256, output_res,
output_res)
for j in range(2):
x = video_feature_cache[:, :, j]
temp = torch.zeros(batch_size, 6, 256, output_res, output_res)
temp[:, :-1, :fold] = x[:, 1:, :fold] # shift left
temp[:, 1:, fold:2 * fold] = x[:, :-1,
fold:2 * fold] # shift right
temp[:, :, 2 * fold:] = x[:, :, 2 * fold:] # not shift
new_tsm_feature[:, :, j] = temp
new_tsm_feature = new_tsm_feature.to(device)
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
output, _ = model(torch.cat((input_now, input_depth_now), 1), True,
new_tsm_feature[:, j])
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now)
output = output[-1]
else: # single output
pass
# loss = criterion(output, target, target_weight)
# # measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
)
bar.next()
bar.finish()
return losses.avg
def train(train_loader, model, criterions, optimizer, debug=False, flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
total_losses = AverageMeter()
heatmap_losses = AverageMeter()
mask_losses = AverageMeter()
label_losses = AverageMeter()
# Loss
criterion_iou, criterion_bce, criterion_focal = criterions
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
for i, (input, input_depth, target_heatmap, target_mask, target_label,
meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_depth = input.to(device), input_depth.to(device)
target_heatmap, target_mask, target_label = target_heatmap.to(device, non_blocking=True), target_mask.to(device, non_blocking=True), \
target_label.to(device, non_blocking=True)
batch_size = input.shape[0]
total_loss, heatmap_loss, mask_loss, label_loss = 0, 0, 0, 0
last_state = None
last_tsm_buffer = None
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j] # [B, 1, 256, 256]
target_heatmap_now = target_heatmap[:, j] # [B, 1, 64, 64]
target_mask_now = target_mask[:, j] # [B, 1, 64, 64]
target_label_now = target_label[:, j] # [B, 1]
if j == 0:
output_heatmap, output_mask, output_label, output_state, output_tsm = model(
torch.cat((input_now, input_depth_now), 1))
else:
output_heatmap, output_mask, output_label, output_state, output_tsm = model(torch.cat((input_now, input_depth_now), 1), \
input_state = last_state, tsm_input = last_tsm_buffer)
last_state = output_state
last_tsm_buffer = output_tsm
# Loss computation
for o_heatmap in output_heatmap:
temp = criterion_iou(
o_heatmap, target_heatmap_now) * 0.05 + criterion_bce(
o_heatmap, target_heatmap_now) * 0.05 # test now
total_loss += temp
heatmap_loss += temp
for o_mask in output_mask:
# temp = criterion_iou(o_mask, target_mask_now) + criterion_bce(o_mask, target_mask_now)
# temp = criterion_focal(o_mask, target_mask_now) # exp 1
temp = criterion_focal(o_mask,
target_mask_now) + criterion_iou(
o_mask, target_mask_now) # exp 2
total_loss += temp
mask_loss += temp
temp = criterion_bce(output_label, target_label_now)
total_loss += temp
label_loss += temp
# measure accuracy and record loss
total_losses.update(total_loss.item(), input.size(0))
heatmap_losses.update(heatmap_loss.item(), input.size(0))
mask_losses.update(mask_loss.item(), input.size(0))
label_losses.update(label_loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=total_losses.avg,
)
bar.next()
bar.finish()
return heatmap_losses.avg, mask_losses.avg, label_losses.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ioues = AverageMeter()
# predictions
predictions = torch.Tensor(val_loader.dataset.__len__(), num_classes, 2)
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, target, meta) in enumerate(val_loader):
# if RELABEL and i == 10 : break
# measure data loading time
data_time.update(time.time() - end)
input = input.to(device, non_blocking=True)
input_depth = input_depth.to(device, non_blocking=True)
target = target.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
last_state = None
iou_list = []
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
if j == 0:
output, last_state = model(
torch.cat((input_now, input_depth_now), 1))
else:
output, _ = model(torch.cat((input_now, input_depth_now),
1),
input_last_state=last_state)
# print(output.shape)
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now)
output = output[-1]
else: # single output
pass
temp_iou = intersectionOverUnion(output.cpu(),
target_now.cpu(),
idx) # have not tested
iou_list.append(temp_iou)
score_map = output[-1].cpu() if type(
output) == list else output.cpu()
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-5:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
# print(relabel_mask_dir)
# print(relabel_mask_name)
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# pred_batch_img, pred_mask = relabel_heatmap(input_now, score_map, 'pred') # old
_, pred_mask = relabel_heatmap(input_now, score_map,
'pred')
# preprocess
temp = input_now[0].cpu().numpy() * 255
temp_input = np.zeros(
(temp.shape[1], temp.shape[2], temp.shape[0]))
for _i in range(3):
temp_input[:, :, _i] = temp[_i, :, :]
temp_input = np.asarray(temp_input, np.uint8)
temp_output = score_map.cpu().numpy() * 255
temp_output = np.asarray(temp_output, np.uint8)
temp_output = np.reshape(temp_output, (64, 64))
pred_batch_img = eval_heatmap(
temp_input, temp_output) # return correct mask + image
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text('MASK_RGB_GT')
gt_win = plt.imshow(gt_mask_rgb)
ax2 = plt.subplot(122)
ax2.title.set_text('Mask_RGB_PRED')
pred_win = plt.imshow(pred_batch_img)
else:
gt_win.set_data(gt_mask_rgb)
pred_win.set_data(pred_batch_img)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
pred_mask.save(os.path.join(relabel_mask_dir, index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# acces.update(acc[0], input.size(0))
ioues.update(sum(iou_list) / len(iou_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
return losses.avg, ioues.avg, predictions
def train_coco(train_loader,
model,
criterion,
optimizer,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
loss_ratio = 1
# coco
for i, (input, input_depth, target, meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_depth, target = input.to(device), input_depth.to(
device), target.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
# first compute
# 2020.6.5 remove TSM module now
for j in range(1):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
output, _ = model(torch.cat((input_now, input_depth_now),
1)) # [B, 4, 256, 256]
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now) * loss_ratio
output = output[-1]
else: # single output
pass
# loss = criterion(output, target, target_weight)
# # measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
)
bar.next()
bar.finish()
return losses.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
import numpy as np
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acces = AverageMeter()
ioues = AverageMeter()
# for statistic
gt_trues = AverageMeter()
gt_falses = AverageMeter()
pred_trues = AverageMeter() # true == true and iou > 50%
pred_falses = AverageMeter()
pred_trues_first = AverageMeter() # true == true
# iou > 50% and step 2 labels are both right -> correcct
# if label is false (and pred is false too) -> correct
final_acces = AverageMeter()
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, input_mask, target, meta,
gt_mask) in enumerate(val_loader):
# if i == 10 : break
# measure data loading time
data_time.update(time.time() - end)
input, input_mask, target = input.to(device), input_mask.to(
device), target.to(device, non_blocking=True)
input_depth = input_depth.to(device)
batch_size = input.shape[0]
loss = 0
last_state = None
acc_list = []
iou_list = []
final_acc_list = []
# for statistic
gt_true_list = []
gt_false_list = []
pred_true_list = []
pred_false_list = []
pred_true_first_list = []
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
input_mask_now = input_mask[:, j]
gt_mask_now = gt_mask[:, j]
target_now = target[:, j]
if j == 0:
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now),
1))
else:
output, output_state = model(torch.cat(
(input_now, input_depth_now, input_mask_now), 1),
input_state=last_state)
# print(output.shape)
last_state = output_state
#############################
'''
testing now
'''
input_mask_now = input_mask_now.cpu()
gt_mask_now = gt_mask_now.cpu()
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
area_head = '/home/s5078345/Affordance-Detection-on-Video/faster-rcnn.pytorch/data_affordance_bbox'
area_tail = ('/').join(raw_mask_path.split('/')[3:-1])
# print(raw_mask_path)
# print(area_tail)
area_to_detect_data_path = os.path.join(
area_head, area_tail, relabel_mask_name[:-4] + '.txt')
area_to_detect_list = []
with open(area_to_detect_data_path) as f:
for line in f:
inner_list = [
int(elt.strip()) for elt in line.split(' ')
]
# in alternative, if you need to use the file content as numbers
# inner_list = [int(elt.strip()) for elt in line.split(',')]
area_to_detect_list.append(inner_list)
if len(area_to_detect_list) == 0:
area_to_detect_list = None
area_to_detect = area_to_detect_list
if area_to_detect is not None:
out_resized_area = torch.zeros((1, 1, 64, 64))
gt_resized_area = torch.zeros((1, 1, 64, 64))
for _i in range(len(area_to_detect)):
x_min, y_min, x_max, y_max = area_to_detect[_i]
x_min = math.floor(x_min / 640 * 64)
y_min = math.floor(y_min / 480 * 64)
x_max = math.ceil(x_max / 640 * 64)
y_max = math.ceil(y_max / 480 * 64)
# clip pred
out_resized_area[0, 0, y_min:y_max,
x_min:x_max] = input_mask_now[
0, 0, y_min:y_max, x_min:x_max]
# clip GT
gt_resized_area[0, 0, y_min:y_max,
x_min:x_max] = gt_mask_now[0, 0,
y_min:y_max,
x_min:x_max]
input_mask_now = out_resized_area
gt_mask_now = gt_resized_area
####################################################
# compute loss
round_output = torch.round(output).float()
loss += criterion(output, target_now)
temp_acc = float(
(round_output == target_now).sum()) / batch_size
temp_1 = (round_output == 1) & (target_now == 1)
temp_acc_1 = temp_1.cpu().numpy()
temp_2 = (round_output == 0) & (target_now == 0)
temp_acc_2 = temp_2.cpu().numpy()
temp_iou = intersectionOverUnion(gt_mask_now,
input_mask_now.cpu(),
idx,
return_list=True)
final_pred_1 = np.logical_and(temp_acc_1, temp_iou > 0.5)
final_pred_2 = temp_acc_2
final_pred = np.logical_or(final_pred_1, final_pred_2)
acc_list.append(temp_acc)
final_acc_list.append(np.sum(final_pred) / batch_size)
round_output = round_output.cpu()
# for statistic
temp_1 = (target_now == 1).cpu().numpy()
temp_2 = (target_now == 0).cpu().numpy()
gt_true_list.append(np.sum(temp_1) / batch_size)
gt_false_list.append(np.sum(temp_2) / batch_size)
pred_true_list.append(np.sum(final_pred_1) / batch_size)
pred_false_list.append(np.sum(final_pred_2) / batch_size)
pred_true_first_list.append(np.sum(temp_acc_1) / batch_size)
if RELABEL:
'''
left image : GT
image : gt_mask_rgb
label : target_now
right image : predict result
image : raw_frames (pred false) or ./checkpoint_0428/pred_vis (pred true)
label : round_output
'''
from PIL import Image
import numpy as np
import copy
# save in same checkpoint
img_index = meta['image_index_list'][j][0]
raw_mask_path = meta['mask_path_list'][j][0]
gt_mask_path = meta['gt_mask_path_list'][j][0]
temp_head = ('/').join(gt_mask_path.split('/')[:-8])
temp_tail = ('/').join(gt_mask_path.split('/')[-5:])
temp = os.path.join(temp_head,
'code/train_two_steps/eval_bbox_clip',
'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(
relabel_mask_dir) # new dir name for pred_vis
# raw frame
raw_rgb_frame_path = os.path.join(
os.path.dirname(os.path.dirname(gt_mask_path)),
'raw_frames',
gt_mask_path.split('/')[-1][:-4] + '.png')
raw_frame = np.array(Image.open(raw_rgb_frame_path))
# gt_mask_rgb
gt_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(gt_mask_path)),
'mask_rgb',
gt_mask_path.split('/')[-1])
if os.path.exists(gt_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(gt_mask_rgb_path))
else:
gt_mask_rgb = copy.deepcopy(raw_frame)
# pred mask
pred_mask_path = os.path.join(
os.path.dirname(raw_mask_path), relabel_mask_name)
pred_mask = np.array(Image.open(pred_mask_path))
pred_mask_rgb = eval_heatmap(raw_frame,
pred_mask) # generate rgb
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
gt_label_str = None
pred_label_str = None
gt_output = gt_mask_rgb
pred_output = None
if target_now[0][0] == 0:
gt_label_str = "GT : False"
elif target_now[0][0] == 1:
gt_label_str = "GT : True"
if round_output[0][0] == 0:
pred_label_str = "Pred : False"
pred_output = raw_frame
elif round_output[0][0] == 1:
pred_output = pred_mask_rgb
if target_now[0][0] == 0:
pred_label_str = "Pred : True"
elif target_now[0][0] == 1 and temp_iou > 0.5:
pred_label_str = "Pred : True (IoU : O)"
elif target_now[0][0] == 1 and temp_iou <= 0.5:
pred_label_str = "Pred : True (IoU : X)"
# output_str = gt_label_str + '. ' + pred_label_str
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text(gt_label_str)
gt_win = plt.imshow(gt_output)
ax2 = plt.subplot(122)
ax2.title.set_text(pred_label_str)
pred_win = plt.imshow(pred_output)
else:
gt_win.set_data(gt_output)
pred_win.set_data(pred_output)
ax1.title.set_text(gt_label_str)
ax2.title.set_text(pred_label_str)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
acces.update(sum(acc_list) / len(acc_list), input.size(0))
final_acces.update(
sum(final_acc_list) / len(final_acc_list), input.size(0))
# for statistic
gt_trues.update(
sum(gt_true_list) / len(gt_true_list), input.size(0))
gt_falses.update(
sum(gt_false_list) / len(gt_false_list), input.size(0))
pred_trues.update(
sum(pred_true_list) / len(pred_true_list), input.size(0))
pred_falses.update(
sum(pred_false_list) / len(pred_false_list), input.size(0))
pred_trues_first.update(
sum(pred_true_first_list) / len(pred_true_first_list),
input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
# for statistic
print("GT true : %.3f" % (gt_trues.avg))
print("GT false : %.3f" % (gt_falses.avg))
print("Pred true : %.3f" % (pred_trues.avg))
print("Pred false : %.3f" % (pred_falses.avg))
print("====")
print("Pred true (no considering IoU) : %.3f" % (pred_trues_first.avg))
print("IoU > 50 percent accuracy : %.3f" %
(pred_trues.avg / pred_trues_first.avg))
print("===")
print("True part : %.3f acc, False part : %.3f acc" %
(pred_trues.avg / gt_trues.avg, pred_falses.avg / gt_falses.avg))
print("Predict true label correct : %.3f" %
(pred_trues_first.avg / gt_trues.avg))
return losses.avg, acces.avg, final_acces.avg
def validate_v2(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ioues = AverageMeter()
# predictions
predictions = torch.Tensor(val_loader.dataset.__len__(), num_classes, 2)
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, target, meta, video_input_eval, _,
video_target_eval,
area_to_detect_list) in enumerate(val_loader):
# if i == 1 : break
# if i != 5 : continue
# if i == 6 : break
# print()
# measure data loading time
data_time.update(time.time() - end)
input = input.to(device, non_blocking=True)
input_depth = input_depth.to(device, non_blocking=True)
target = target.to(device, non_blocking=True)
video_target_eval = video_target_eval.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
iou_list = []
for j in range(6):
# load whole image
input_whole = video_input_eval[:, j]
target_whole = video_target_eval[:, j] #[1, 32, 32]
output_whole = torch.zeros((1, 32, 32))
# output_whole = output_whole.to(device, non_blocking=True)
for obj_i in range(3):
if len(area_to_detect_list[j]) <= obj_i:
continue
input_now = input[:, j, obj_i] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j, obj_i]
target_now = target[:, j, obj_i]
output, _ = model(
torch.cat((input_now, input_depth_now), 1))
output = output[-1]
score_map_part = output.cpu()
score_map_gt = target_now.cpu()
output = output[-1].cpu().numpy()
# map back to area_detect
_x_min = float(area_to_detect_list[j][obj_i][0].numpy())
_y_min = float(area_to_detect_list[j][obj_i][1].numpy())
_x_max = float(area_to_detect_list[j][obj_i][2].numpy())
_y_max = float(area_to_detect_list[j][obj_i][3].numpy())
OUT_RES = 32
# order is reversed
# print(_x_min, _y_min, _x_max, _y_max)
x_min = round(_x_min / 640 * OUT_RES)
y_min = round(_y_min / 480 * OUT_RES)
x_max = min(round(_x_max / 640 * OUT_RES), 31)
y_max = min(round(_y_max / 480 * OUT_RES), 31)
x_len = x_max - x_min
y_len = y_max - y_min
# print(x_min, y_min, x_max, y_max)
## resize out now to fixed size
if x_len > 0 and y_len > 0:
resized_out = faster_rcnn_crop(output, x_len, y_len)
# print(resized_out.shape)
# output_whole[0, y_min:y_max, x_min:x_max] = 1
output_whole[0, y_min:y_max, x_min:x_max] = resized_out
# resized_out = faster_rcnn_crop(output, y_len, x_len)
# output_whole[0, x_min:x_max, y_min:y_max] = resized_out
# # part image
# if RELABEL:
# # save in same checkpoint
# raw_mask_path = meta['mask_path_list'][j][0]
# img_index = meta['image_index_list'][j][0]
# temp_head = ('/').join(raw_mask_path.split('/')[:-8])
# temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
# temp = os.path.join(temp_head, 'code/train_two_steps', checkpoint, 'pred_vis', temp_tail)
# relabel_mask_dir, relabel_mask_name = os.path.split(temp)
# relabel_mask_dir = os.path.dirname(relabel_mask_dir)
# raw_mask_rgb_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'first_mask_rgb', relabel_mask_name)
# new_mask_rgb_path = os.path.join(relabel_mask_dir, 'gt_' + relabel_mask_name)
# raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
# relabel_mask_name[:-4] + '.png')
# from PIL import Image
# import numpy as np
# # if os.path.exists(raw_mask_rgb_path):
# # gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
# # else :
# # gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# gt_mask_rgb, _ = relabel_heatmap(input_now, score_map_gt, 'gt') # checked
# pred_batch_img, pred_mask = relabel_heatmap(input_now, score_map_part, 'pred') # return an Image object
# if not isdir(relabel_mask_dir):
# mkdir_p(relabel_mask_dir)
# if not gt_win or not pred_win:
# ax1 = plt.subplot(121)
# ax1.title.set_text('MASK_RGB_GT')
# gt_win = plt.imshow(gt_mask_rgb)
# ax2 = plt.subplot(122)
# ax2.title.set_text('Mask_RGB_PRED')
# pred_win = plt.imshow(pred_batch_img)
# else:
# gt_win.set_data(gt_mask_rgb)
# pred_win.set_data(pred_batch_img)
# plt.plot()
# plt.savefig(os.path.join(relabel_mask_dir, '%05d_part_%d.jpg' % (img_index, obj_i)))
# # pred_mask.save(os.path.join(relabel_mask_dir, index_name))
output_whole = torch.unsqueeze(output_whole, 0)
target_whole = torch.unsqueeze(target_whole, 0)
score_map_whole = output_whole.cpu()
temp_iou = intersectionOverUnion(output_whole,
target_whole.cpu(),
idx) # have not tested
iou_list.append(temp_iou)
# whole image
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# print(input_now.shape)
# print(score_map.shape)
pred_batch_img, pred_mask = relabel_heatmap(
input_whole, score_map_whole,
'pred') # return an Image object
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text('MASK_RGB_GT')
gt_win = plt.imshow(gt_mask_rgb)
ax2 = plt.subplot(122)
ax2.title.set_text('Mask_RGB_PRED')
pred_win = plt.imshow(pred_batch_img)
else:
gt_win.set_data(gt_mask_rgb)
pred_win.set_data(pred_batch_img)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
pred_mask.save(os.path.join(relabel_mask_dir, index_name))
# print(os.path.join(relabel_mask_dir, 'vis_' + index_name))
# acces.update(acc[0], input.size(0))
ioues.update(sum(iou_list) / len(iou_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td)
bar.next()
bar.finish()
return 0, ioues.avg, predictions