def train(train_loader, model, criterion, optimizer, debug=False, flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
for i, (input, input_depth, input_mask, target,
meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_mask, target = input.to(device), input_mask.to(
device), target.to(device, non_blocking=True)
input_depth = input_depth.to(device)
# target_weight = meta['target_weight'].to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
last_state = None
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
input_mask_now = input_mask[:, j]
target_now = target[:, j]
if j == 0:
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now), 1))
else:
if LSTM_STATE == 'stateness':
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now),
1))
elif LSTM_STATE == 'stateful':
output, output_state = model(torch.cat(
(input_now, input_depth_now, input_mask_now), 1),
input_state=last_state)
last_state = output_state
loss += criterion(output, target_now)
# print(loss.item())
# # measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
)
bar.next()
bar.finish()
return losses.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acces = AverageMeter()
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, input_mask, target,
meta) in enumerate(val_loader):
# if RELABEL and i == 2 : break
# measure data loading time
data_time.update(time.time() - end)
input, input_mask, target = input.to(device), input_mask.to(
device), target.to(device, non_blocking=True)
input_depth = input_depth.to(device)
batch_size = input.shape[0]
loss = 0
last_state = None
acc_list = []
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
input_mask_now = input_mask[:, j]
target_now = target[:, j]
if j == 0:
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now),
1))
else:
if LSTM_STATE == 'stateness':
output, output_state = model(
torch.cat(
(input_now, input_depth_now, input_mask_now),
1))
elif LSTM_STATE == 'stateful':
output, output_state = model(torch.cat(
(input_now, input_depth_now, input_mask_now), 1),
input_state=last_state)
last_state = output_state
# print(output.shape)
round_output = torch.round(output).float()
loss += criterion(output, target_now)
temp_acc = float(
(round_output == target_now).sum()) / batch_size
acc_list.append(temp_acc)
round_output = round_output.cpu()
# print(round_output)
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-5:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
gt_label_str = None
pred_label_str = None
if target_now[0][0] == 0:
gt_label_str = "GT : False"
elif target_now[0][0] == 1:
gt_label_str = "GT : True"
if round_output[0][0] == 0:
pred_label_str = "Pred : False"
elif round_output[0][0] == 1:
pred_label_str = "Pred : True"
output_str = gt_label_str + '. ' + pred_label_str
# if target_now[0][0] != round_output[0][0] :
# print(raw_rgb_frame_path)
if not gt_win:
plt.plot()
plt.title(output_str)
gt_win = plt.imshow(gt_mask_rgb)
else:
plt.title(output_str)
gt_win.set_data(gt_mask_rgb)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
acces.update(sum(acc_list) / len(acc_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
return losses.avg, acces.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ioues = AverageMeter()
# predictions
predictions = torch.Tensor(val_loader.dataset.__len__(), num_classes, 2)
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, target, meta) in enumerate(val_loader):
if RELABEL and i == 1: break
# measure data loading time
data_time.update(time.time() - end)
input = input.to(device, non_blocking=True)
input_depth = input_depth.to(device, non_blocking=True)
target = target.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
iou_list = []
# store first two stack feature # 2 = first and second stack, 6 = video length
video_feature_cache = torch.zeros(batch_size, 6, 2, 256,
output_res, output_res)
# first compute
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
_, out_tsm_feature = model(
torch.cat((input_now, input_depth_now),
1)) # [B, 4, 256, 256]
for k in range(2):
video_feature_cache[:, j, k] = out_tsm_feature[k]
# TSM module
b, t, _, c, h, w = video_feature_cache.size()
fold_div = 8
fold = c // fold_div
new_tsm_feature = torch.zeros(batch_size, 6, 2, 256, output_res,
output_res)
for j in range(2):
x = video_feature_cache[:, :, j]
temp = torch.zeros(batch_size, 6, 256, output_res, output_res)
temp[:, :-1, :fold] = x[:, 1:, :fold] # shift left
temp[:, 1:, fold:2 * fold] = x[:, :-1,
fold:2 * fold] # shift right
temp[:, :, 2 * fold:] = x[:, :, 2 * fold:] # not shift
new_tsm_feature[:, :, j] = temp
new_tsm_feature = new_tsm_feature.to(device)
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
output, _ = model(torch.cat((input_now, input_depth_now), 1),
True, new_tsm_feature[:, j])
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now)
output = output[-1]
else: # single output
pass
'''
testing now
'''
output = output.cpu()
target_now = target_now.cpu()
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
area_head = '/home/s5078345/Affordance-Detection-on-Video/faster-rcnn.pytorch/data_affordance_bbox'
area_tail = ('/').join(raw_mask_path.split('/')[6:10])
area_to_detect_data_path = os.path.join(
area_head, area_tail, relabel_mask_name[:-4] + '.txt')
area_to_detect_list = []
with open(area_to_detect_data_path) as f:
for line in f:
inner_list = [
int(elt.strip()) for elt in line.split(' ')
]
# in alternative, if you need to use the file content as numbers
# inner_list = [int(elt.strip()) for elt in line.split(',')]
area_to_detect_list.append(inner_list)
if len(area_to_detect_list) == 0:
area_to_detect_list = None
area_to_detect = area_to_detect_list
if area_to_detect is not None:
out_resized_area = torch.zeros((1, 1, 64, 64))
gt_resized_area = torch.zeros((1, 1, 64, 64))
for _i in range(len(area_to_detect)):
x_min, y_min, x_max, y_max = area_to_detect[_i]
x_min = math.floor(x_min / 640 * 64)
y_min = math.floor(y_min / 480 * 64)
x_max = math.ceil(x_max / 640 * 64)
y_max = math.ceil(y_max / 480 * 64)
# clip pred
out_resized_area[0, 0, y_min:y_max,
x_min:x_max] = output[0, 0,
y_min:y_max,
x_min:x_max]
# clip GT
gt_resized_area[0, 0, y_min:y_max,
x_min:x_max] = target_now[0, 0,
y_min:y_max,
x_min:x_max]
output = out_resized_area
target_now = gt_resized_area
else:
pass
temp_iou = intersectionOverUnion(output.cpu(),
target_now.cpu(),
idx) # have not tested
iou_list.append(temp_iou)
score_map = output[-1].cpu() if type(
output) == list else output.cpu()
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
# print(relabel_mask_dir)
# print(relabel_mask_name)
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# print(input_now.shape)
# print(score_map) # [1, 1, 64, 64]
# 2020.7.1
# load info about hide area
# from faster rcnn
area_head = '/home/s5078345/Affordance-Detection-on-Video/faster-rcnn.pytorch/data_affordance_bbox'
area_tail = ('/').join(raw_mask_path.split('/')[6:10])
area_to_detect_data_path = os.path.join(
area_head, area_tail, relabel_mask_name[:-4] + '.txt')
# print(area_to_use_data_path)
area_to_detect_list = []
with open(area_to_detect_data_path) as f:
for line in f:
inner_list = [
int(elt.strip()) for elt in line.split(' ')
]
# in alternative, if you need to use the file content as numbers
# inner_list = [int(elt.strip()) for elt in line.split(',')]
area_to_detect_list.append(inner_list)
if len(area_to_detect_list) == 0:
area_to_detect_list = None
# print(area_to_detect_list)
pred_batch_img, pred_mask = relabel_heatmap(
input_now,
score_map,
'pred',
area_to_detect=area_to_detect_list
) # return an Image object
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text('MASK_RGB_GT')
gt_win = plt.imshow(gt_mask_rgb)
ax2 = plt.subplot(122)
ax2.title.set_text('Mask_RGB_PRED')
pred_win = plt.imshow(pred_batch_img)
else:
gt_win.set_data(gt_mask_rgb)
pred_win.set_data(pred_batch_img)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
pred_mask.save(os.path.join(relabel_mask_dir, index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# acces.update(acc[0], input.size(0))
ioues.update(sum(iou_list) / len(iou_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
return losses.avg, ioues.avg, predictions
def train(train_loader, model, criterion, optimizer, debug=False, flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
for i, (input, input_depth, target, meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_depth, target = input.to(device), input_depth.to(
device), target.to(device, non_blocking=True)
# target_weight = meta['target_weight'].to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
# store first two stack feature # 2 = first and second stack, 6 = video length
video_feature_cache = torch.zeros(batch_size, 6, 2, 256, output_res,
output_res)
with torch.no_grad():
# first compute
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
_, out_tsm_feature = model(
torch.cat((input_now, input_depth_now),
1)) # [B, 4, 256, 256]
for k in range(2):
video_feature_cache[:, j, k] = out_tsm_feature[k]
# TSM module
b, t, _, c, h, w = video_feature_cache.size()
fold_div = 8
fold = c // fold_div
new_tsm_feature = torch.zeros(batch_size, 6, 2, 256, output_res,
output_res)
for j in range(2):
x = video_feature_cache[:, :, j]
temp = torch.zeros(batch_size, 6, 256, output_res, output_res)
temp[:, :-1, :fold] = x[:, 1:, :fold] # shift left
temp[:, 1:, fold:2 * fold] = x[:, :-1,
fold:2 * fold] # shift right
temp[:, :, 2 * fold:] = x[:, :, 2 * fold:] # not shift
new_tsm_feature[:, :, j] = temp
new_tsm_feature = new_tsm_feature.to(device)
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
output, _ = model(torch.cat((input_now, input_depth_now), 1), True,
new_tsm_feature[:, j])
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now)
output = output[-1]
else: # single output
pass
# loss = criterion(output, target, target_weight)
# # measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
)
bar.next()
bar.finish()
return losses.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ioues = AverageMeter()
# predictions
predictions = torch.Tensor(val_loader.dataset.__len__(), num_classes, 2)
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, target, meta) in enumerate(val_loader):
# if RELABEL and i == 10 : break
# measure data loading time
data_time.update(time.time() - end)
input = input.to(device, non_blocking=True)
input_depth = input_depth.to(device, non_blocking=True)
target = target.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
last_state = None
iou_list = []
# compute use TSM feature
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
if j == 0:
output, last_state = model(
torch.cat((input_now, input_depth_now), 1))
else:
output, _ = model(torch.cat((input_now, input_depth_now),
1),
input_last_state=last_state)
# print(output.shape)
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now)
output = output[-1]
else: # single output
pass
temp_iou = intersectionOverUnion(output.cpu(),
target_now.cpu(),
idx) # have not tested
iou_list.append(temp_iou)
score_map = output[-1].cpu() if type(
output) == list else output.cpu()
if RELABEL:
# save in same checkpoint
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-5:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
raw_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(raw_mask_path)),
'first_mask_rgb', relabel_mask_name)
new_mask_rgb_path = os.path.join(relabel_mask_dir,
'gt_' + relabel_mask_name)
raw_rgb_frame_path = os.path.join(os.path.dirname(os.path.dirname(raw_mask_path)), 'raw_frames', \
relabel_mask_name[:-4] + '.png')
# print(relabel_mask_dir)
# print(relabel_mask_name)
from PIL import Image
import numpy as np
if os.path.exists(raw_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(raw_mask_rgb_path))
else:
gt_mask_rgb = np.array(Image.open(raw_rgb_frame_path))
# pred_batch_img, pred_mask = relabel_heatmap(input_now, score_map, 'pred') # old
_, pred_mask = relabel_heatmap(input_now, score_map,
'pred')
# preprocess
temp = input_now[0].cpu().numpy() * 255
temp_input = np.zeros(
(temp.shape[1], temp.shape[2], temp.shape[0]))
for _i in range(3):
temp_input[:, :, _i] = temp[_i, :, :]
temp_input = np.asarray(temp_input, np.uint8)
temp_output = score_map.cpu().numpy() * 255
temp_output = np.asarray(temp_output, np.uint8)
temp_output = np.reshape(temp_output, (64, 64))
pred_batch_img = eval_heatmap(
temp_input, temp_output) # return correct mask + image
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text('MASK_RGB_GT')
gt_win = plt.imshow(gt_mask_rgb)
ax2 = plt.subplot(122)
ax2.title.set_text('Mask_RGB_PRED')
pred_win = plt.imshow(pred_batch_img)
else:
gt_win.set_data(gt_mask_rgb)
pred_win.set_data(pred_batch_img)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
pred_mask.save(os.path.join(relabel_mask_dir, index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# acces.update(acc[0], input.size(0))
ioues.update(sum(iou_list) / len(iou_list), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
return losses.avg, ioues.avg, predictions
def train_coco(train_loader,
model,
criterion,
optimizer,
debug=False,
flip=True):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
gt_win, pred_win = None, None
bar = Bar('Train', max=len(train_loader))
loss_ratio = 1
# coco
for i, (input, input_depth, target, meta) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input, input_depth, target = input.to(device), input_depth.to(
device), target.to(device, non_blocking=True)
batch_size = input.shape[0]
loss = 0
# first compute
# 2020.6.5 remove TSM module now
for j in range(1):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
target_now = target[:, j]
output, _ = model(torch.cat((input_now, input_depth_now),
1)) # [B, 4, 256, 256]
if type(
output
) == list: # multiple output # beacuse of intermediate prediction
for o in output:
loss += criterion(o, target_now) * loss_ratio
output = output[-1]
else: # single output
pass
# loss = criterion(output, target, target_weight)
# # measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
)
bar.next()
bar.finish()
return losses.avg
def validate(val_loader,
model,
criterion,
num_classes,
checkpoint,
debug=False,
flip=True):
import numpy as np
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acces = AverageMeter()
ioues = AverageMeter()
# for statistic
gt_trues = AverageMeter()
gt_falses = AverageMeter()
pred_trues = AverageMeter() # true == true and iou > 50%
pred_falses = AverageMeter()
pred_trues_first = AverageMeter() # true == true
# iou > 50% and step 2 labels are both right -> correcct
# if label is false (and pred is false too) -> correct
final_acces = AverageMeter()
# switch to evaluate mode
model.eval()
gt_win, pred_win = None, None
iou = None
end = time.time()
bar = Bar('Eval ', max=len(val_loader))
with torch.no_grad():
for i, (input, input_depth, input_mask, target, meta,
gt_mask) in enumerate(val_loader):
# if i == 10 : break
# measure data loading time
data_time.update(time.time() - end)
input, input_mask, target = input.to(device), input_mask.to(
device), target.to(device, non_blocking=True)
input_depth = input_depth.to(device)
batch_size = input.shape[0]
loss = 0
last_state = None
acc_list = []
iou_list = []
final_acc_list = []
# for statistic
gt_true_list = []
gt_false_list = []
pred_true_list = []
pred_false_list = []
pred_true_first_list = []
for j in range(6):
input_now = input[:, j] # [B, 3, 256, 256]
input_depth_now = input_depth[:, j]
input_mask_now = input_mask[:, j]
gt_mask_now = gt_mask[:, j]
target_now = target[:, j]
if j == 0:
output, output_state = model(
torch.cat((input_now, input_depth_now, input_mask_now),
1))
else:
output, output_state = model(torch.cat(
(input_now, input_depth_now, input_mask_now), 1),
input_state=last_state)
# print(output.shape)
last_state = output_state
#############################
'''
testing now
'''
input_mask_now = input_mask_now.cpu()
gt_mask_now = gt_mask_now.cpu()
raw_mask_path = meta['mask_path_list'][j][0]
img_index = meta['image_index_list'][j][0]
temp_head = ('/').join(raw_mask_path.split('/')[:-8])
temp_tail = ('/').join(raw_mask_path.split('/')[-6:])
temp = os.path.join(temp_head, 'code/train_two_steps',
checkpoint, 'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(relabel_mask_dir)
area_head = '/home/s5078345/Affordance-Detection-on-Video/faster-rcnn.pytorch/data_affordance_bbox'
area_tail = ('/').join(raw_mask_path.split('/')[3:-1])
# print(raw_mask_path)
# print(area_tail)
area_to_detect_data_path = os.path.join(
area_head, area_tail, relabel_mask_name[:-4] + '.txt')
area_to_detect_list = []
with open(area_to_detect_data_path) as f:
for line in f:
inner_list = [
int(elt.strip()) for elt in line.split(' ')
]
# in alternative, if you need to use the file content as numbers
# inner_list = [int(elt.strip()) for elt in line.split(',')]
area_to_detect_list.append(inner_list)
if len(area_to_detect_list) == 0:
area_to_detect_list = None
area_to_detect = area_to_detect_list
if area_to_detect is not None:
out_resized_area = torch.zeros((1, 1, 64, 64))
gt_resized_area = torch.zeros((1, 1, 64, 64))
for _i in range(len(area_to_detect)):
x_min, y_min, x_max, y_max = area_to_detect[_i]
x_min = math.floor(x_min / 640 * 64)
y_min = math.floor(y_min / 480 * 64)
x_max = math.ceil(x_max / 640 * 64)
y_max = math.ceil(y_max / 480 * 64)
# clip pred
out_resized_area[0, 0, y_min:y_max,
x_min:x_max] = input_mask_now[
0, 0, y_min:y_max, x_min:x_max]
# clip GT
gt_resized_area[0, 0, y_min:y_max,
x_min:x_max] = gt_mask_now[0, 0,
y_min:y_max,
x_min:x_max]
input_mask_now = out_resized_area
gt_mask_now = gt_resized_area
####################################################
# compute loss
round_output = torch.round(output).float()
loss += criterion(output, target_now)
temp_acc = float(
(round_output == target_now).sum()) / batch_size
temp_1 = (round_output == 1) & (target_now == 1)
temp_acc_1 = temp_1.cpu().numpy()
temp_2 = (round_output == 0) & (target_now == 0)
temp_acc_2 = temp_2.cpu().numpy()
temp_iou = intersectionOverUnion(gt_mask_now,
input_mask_now.cpu(),
idx,
return_list=True)
final_pred_1 = np.logical_and(temp_acc_1, temp_iou > 0.5)
final_pred_2 = temp_acc_2
final_pred = np.logical_or(final_pred_1, final_pred_2)
acc_list.append(temp_acc)
final_acc_list.append(np.sum(final_pred) / batch_size)
round_output = round_output.cpu()
# for statistic
temp_1 = (target_now == 1).cpu().numpy()
temp_2 = (target_now == 0).cpu().numpy()
gt_true_list.append(np.sum(temp_1) / batch_size)
gt_false_list.append(np.sum(temp_2) / batch_size)
pred_true_list.append(np.sum(final_pred_1) / batch_size)
pred_false_list.append(np.sum(final_pred_2) / batch_size)
pred_true_first_list.append(np.sum(temp_acc_1) / batch_size)
if RELABEL:
'''
left image : GT
image : gt_mask_rgb
label : target_now
right image : predict result
image : raw_frames (pred false) or ./checkpoint_0428/pred_vis (pred true)
label : round_output
'''
from PIL import Image
import numpy as np
import copy
# save in same checkpoint
img_index = meta['image_index_list'][j][0]
raw_mask_path = meta['mask_path_list'][j][0]
gt_mask_path = meta['gt_mask_path_list'][j][0]
temp_head = ('/').join(gt_mask_path.split('/')[:-8])
temp_tail = ('/').join(gt_mask_path.split('/')[-5:])
temp = os.path.join(temp_head,
'code/train_two_steps/eval_bbox_clip',
'pred_vis', temp_tail)
relabel_mask_dir, relabel_mask_name = os.path.split(temp)
relabel_mask_dir = os.path.dirname(
relabel_mask_dir) # new dir name for pred_vis
# raw frame
raw_rgb_frame_path = os.path.join(
os.path.dirname(os.path.dirname(gt_mask_path)),
'raw_frames',
gt_mask_path.split('/')[-1][:-4] + '.png')
raw_frame = np.array(Image.open(raw_rgb_frame_path))
# gt_mask_rgb
gt_mask_rgb_path = os.path.join(
os.path.dirname(os.path.dirname(gt_mask_path)),
'mask_rgb',
gt_mask_path.split('/')[-1])
if os.path.exists(gt_mask_rgb_path):
gt_mask_rgb = np.array(Image.open(gt_mask_rgb_path))
else:
gt_mask_rgb = copy.deepcopy(raw_frame)
# pred mask
pred_mask_path = os.path.join(
os.path.dirname(raw_mask_path), relabel_mask_name)
pred_mask = np.array(Image.open(pred_mask_path))
pred_mask_rgb = eval_heatmap(raw_frame,
pred_mask) # generate rgb
if not isdir(relabel_mask_dir):
mkdir_p(relabel_mask_dir)
gt_label_str = None
pred_label_str = None
gt_output = gt_mask_rgb
pred_output = None
if target_now[0][0] == 0:
gt_label_str = "GT : False"
elif target_now[0][0] == 1:
gt_label_str = "GT : True"
if round_output[0][0] == 0:
pred_label_str = "Pred : False"
pred_output = raw_frame
elif round_output[0][0] == 1:
pred_output = pred_mask_rgb
if target_now[0][0] == 0:
pred_label_str = "Pred : True"
elif target_now[0][0] == 1 and temp_iou > 0.5:
pred_label_str = "Pred : True (IoU : O)"
elif target_now[0][0] == 1 and temp_iou <= 0.5:
pred_label_str = "Pred : True (IoU : X)"
# output_str = gt_label_str + '. ' + pred_label_str
if not gt_win or not pred_win:
ax1 = plt.subplot(121)
ax1.title.set_text(gt_label_str)
gt_win = plt.imshow(gt_output)
ax2 = plt.subplot(122)
ax2.title.set_text(pred_label_str)
pred_win = plt.imshow(pred_output)
else:
gt_win.set_data(gt_output)
pred_win.set_data(pred_output)
ax1.title.set_text(gt_label_str)
ax2.title.set_text(pred_label_str)
plt.plot()
index_name = "%05d.jpg" % (img_index)
plt.savefig(
os.path.join(relabel_mask_dir, 'vis_' + index_name))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
acces.update(sum(acc_list) / len(acc_list), input.size(0))
final_acces.update(
sum(final_acc_list) / len(final_acc_list), input.size(0))
# for statistic
gt_trues.update(
sum(gt_true_list) / len(gt_true_list), input.size(0))
gt_falses.update(
sum(gt_false_list) / len(gt_false_list), input.size(0))
pred_trues.update(
sum(pred_true_list) / len(pred_true_list), input.size(0))
pred_falses.update(
sum(pred_false_list) / len(pred_false_list), input.size(0))
pred_trues_first.update(
sum(pred_true_first_list) / len(pred_true_first_list),
input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f}'.format(
batch=i + 1,
size=len(val_loader),
data=data_time.val,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg)
bar.next()
bar.finish()
# for statistic
print("GT true : %.3f" % (gt_trues.avg))
print("GT false : %.3f" % (gt_falses.avg))
print("Pred true : %.3f" % (pred_trues.avg))
print("Pred false : %.3f" % (pred_falses.avg))
print("====")
print("Pred true (no considering IoU) : %.3f" % (pred_trues_first.avg))
print("IoU > 50 percent accuracy : %.3f" %
(pred_trues.avg / pred_trues_first.avg))
print("===")
print("True part : %.3f acc, False part : %.3f acc" %
(pred_trues.avg / gt_trues.avg, pred_falses.avg / gt_falses.avg))
print("Predict true label correct : %.3f" %
(pred_trues_first.avg / gt_trues.avg))
return losses.avg, acces.avg, final_acces.avg
