def forward(self, feats, last_rnn_state, pred_poly):
pred_poly = pred_poly.detach().cpu().numpy() #[bs, time]
# we will use numpy functions to get pred_mask and pred_vertex_mask
pred_mask = np.zeros(
(pred_poly.shape[0], 1, self.grid_size, self.grid_size),
dtype=np.uint8)
pred_vertex_mask = np.zeros(
(pred_poly.shape[0], 1, self.grid_size, self.grid_size),
dtype=np.uint8)
# Draw Vertex mask and full polygon mask
for b in range(pred_poly.shape[0]):
masked_poly = utils.get_masked_poly(pred_poly[b], self.grid_size)
xy_poly = utils.class_to_xy(masked_poly, self.grid_size)
utils.get_vertices_mask(xy_poly, pred_vertex_mask[b, 0])
utils.draw_poly(pred_mask[b, 0], xy_poly)
pred_mask = torch.from_numpy(pred_mask).to(device).to(torch.float32)
pred_vertex_mask = torch.from_numpy(pred_vertex_mask).to(device).to(
torch.float32)
inp = torch.cat([
feats, last_rnn_state[0][0], last_rnn_state[1][0], pred_mask,
pred_vertex_mask
],
dim=1)
conv1 = self.conv1(inp)
conv2 = self.conv2(conv1)
conv2 = conv2.view(conv2.size(0), -1)
pred_iou = self.fc(conv2)
return pred_iou.view(-1)
def iou_from_poly(pred, gt, width, height):
"""
Compute IoU from poly. The polygons should
already be in the final output size
pred: list of np arrays of predicted polygons
gt: list of np arrays of gt polygons
grid_size: grid_size that the polygons are in
"""
masks = np.zeros((2, height, width), dtype=np.uint8)
if not isinstance(pred, list):
pred = [pred]
if not isinstance(gt, list):
gt = [gt]
for p in pred:
masks[0] = utils.draw_poly(masks[0], p)
for g in gt:
masks[1] = utils.draw_poly(masks[1], g)
return iou_from_mask(masks[0], masks[1]), masks
def iou_from_gt_mask_and_pred_poly(gt, pred, width, height):
"""
Compute IoU from poly. The polygons should
already be in the final output size
pred: list of np arrays of predicted polygons
gt: list of np arrays of gt polygons
grid_size: grid_size that the polygons are in
"""
masks = np.zeros((height, width), dtype=np.uint8)
if not isinstance(pred, list):
pred = [pred]
for g in pred:
masks = utils.draw_poly(masks, g)
return iou_from_mask(masks, gt), masks
def process_outputs(self, data, output, save=True):
"""
Process outputs to get final outputs for the whole image
Optionally saves the outputs to a folder for evaluation
"""
instances = data['instance']
pred_spline = output['pred_polys'] # (16, 40, 2)
pred_gcn = output['gcn_layer'] # (16, 40, 2)
## overlay pred_spline to 224*224 image
# origin_img = output['origin_img']
# img = torch.squeeze(origin_img)
# img = data['img']
# img = torch.squeeze(img)
# img = img.cpu().numpy().transpose(1, 2, 0) # (224, 224, 3)
# img = (img-img.min())/(img.max()-img.min()) # normalize to 0~1
# img_save = np.uint8(255 * img)
# import os
# # len_th = len(os.listdir(""))
# imsave("" + str(instances[0]['img_path'][-17:-4]) + '.png', img_save, 'PNG')
# len_th = len(os.listdir(""))
#
# instance_id = instances[0]['img_path'][-17:-4]
#
# for i in range(len(output['pred_polys_all'])):
# pred_spline_numpy = torch.squeeze(output['pred_polys_all'][i]).cpu().numpy() # (cp_num,2)
# to_axis = (pred_spline_numpy * 224).astype(np.int32)
# CAM = np.zeros((224, 224))
# for index, item in enumerate(to_axis):
# CAM[item[1] - 2, item[0] - 2] = 1
# CAM[item[1] - 2, item[0] - 1] = 1
# CAM[item[1] - 1, item[0] - 2] = 1
# CAM[item[1] - 1, item[0] - 1] = 1 # top-left
# CAM[item[1] - 2, item[0]] = 1
# CAM[item[1] - 1, item[0]] = 1 # top
# CAM[item[1] - 2, item[0] + 1] = 1
# CAM[item[1] - 2, item[0] + 2] = 1
# CAM[item[1] - 1, item[0] + 2] = 1
# CAM[item[1] - 1, item[0] + 1] = 1 # top-right
# CAM[item[1], item[0] - 2] = 1
# CAM[item[1], item[0] - 1] = 1 # left
# CAM[item[1], item[0] + 2] = 1
# CAM[item[1], item[0] + 1] = 1 # right
# CAM[item[1] + 1, item[0] - 2] = 1
# CAM[item[1] + 2, item[0] - 1] = 1
# CAM[item[1] + 2, item[0] - 2] = 1
# CAM[item[1] + 1, item[0] - 1] = 1 # bottom-left
# CAM[item[1] + 2, item[0]] = 1
# CAM[item[1] + 1, item[0]] = 1 # bottom
# CAM[item[1] + 1, item[0] + 2] = 1
# CAM[item[1] + 2, item[0] + 2] = 1
# CAM[item[1] + 2, item[0] + 1] = 1
# CAM[item[1] + 1, item[0] + 1] = 1
# CAM[item[1], item[0]] = 1
# import cv2
# heatmap = cv2.applyColorMap(np.uint8(255 * CAM), cv2.COLORMAP_JET)
# heatmap = np.float32(heatmap) / 255
# # cam = heatmap + np.float32(torch.squeeze(img).permute(1,2,0).detach().numpy())
# # cam = heatmap + np.float32(torch.squeeze(img).permute(1,2,0).detach().numpy())
# cam = 0.7*heatmap + np.float32(img) * 0.4 #0.5 should be smaller
# cam = cam / np.max(cam)
# import os
#
# cv2.imwrite("" + str(instance_id) + "_" + str(i) + ".jpg", np.uint8(255 * cam))
# #cv2.imwrite("" + str(i) + "gray_pre" + str(len_th) + ".jpg", np.uint8(255 * CAM))
# preds = self.spline.sample_point(pred_spline) # (16, 1280, 2)
preds = pred_spline
preds_numpy = preds.cpu().numpy()
torch.cuda.synchronize()
preds = preds.cpu().numpy()
pred_spline = pred_spline.cpu()
pred_spline = pred_spline.numpy()
## overlay pre_polygons on croped img
# img_save = np.uint8(img)
# pil_img = Image.fromarray(img_save, mode='RGB')
# img_draw = ImageDraw.Draw(pil_img, 'RGBA')
# # img_draw.polygon(polys_pred, outline='red') #pred polygons
# # img_draw.polygon(polys_gt, outline='blue') #GT polygons
# predicted_poly = []
# poly = preds[0]
# poly = poly * 224#data['patch_w'][0] #to (224, 224)
# # poly[:, 0] += data['starting_point'][0][0]
# # poly[:, 1] += data['starting_point'][0][1]
# predicted_poly.append(poly.tolist())
# polys_pre = predicted_poly[0]
# polys_pre_final = [tuple(item) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0]-data['starting_point'][0][0])*224/data['patch_w'][0],(item[1]-data['starting_point'][0][1])*224/data['patch_w'][0])) for item in polys_gt]
#
# # draw circle
# #img_draw = ImageDraw.Draw(img_ob, 'RGBA')
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
#
# ## width be bigger
# polys_pre_final = [tuple((item[0]-1, item[1]-1)) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0]-1,
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0]-1)) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
#
# polys_pre_final = [tuple((item[0] + 1, item[1] + 1)) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0] + 1,
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0] + 1)) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
# ##top
# polys_pre_final = [tuple((item[0], item[1]- 1)) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0],
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0] - 1)) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
# ##bottom
# polys_pre_final = [tuple((item[0], item[1]+ 1)) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0],
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0] + 1)) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
# ## top-right
# polys_pre_final = [tuple((item[0]+1, item[1] - 1)) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0]+1,
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0] - 1)) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
# ## bottom-left
# polys_pre_final = [tuple((item[0] - 1, item[1] + 1)) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0] - 1,
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0] + 1)) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
# ## left
# polys_pre_final = [tuple((item[0] - 1, item[1])) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0] - 1,
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0])) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
# #img_draw.polygon(polys_gt_final, outline=(0, 0, 255))
# ## right
# polys_pre_final = [tuple((item[0] + 1, item[1])) for item in polys_pre]
# polys_gt = instances[0]['components'][0]['poly']
# polys_gt_final = [tuple(((item[0] - data['starting_point'][0][0]) * 224 / data['patch_w'][0] + 1,
# (item[1] - data['starting_point'][0][1]) * 224 / data['patch_w'][0])) for item in
# polys_gt]
# img_draw.polygon(polys_pre_final, outline=(255, 0, 0)) # pred polygons
## bellow code for draw vertexPoly on croped img
# img_draw.polygon(polys_pre_final, outline=(255, 193, 37))
# imsave_ob = np.array(pil_img).astype(np.uint8)
# for index, item in enumerate(polys_pre_final):
# if index % 32 == 0:
# #cv2.circle(imsave_ob, (int(item[0]),int(item[1])),2,(0,255,255),-1)
# rr,cc = draw.circle(item[1],item[0],2)
# draw.set_color(imsave_ob, [rr,cc], [0,255,255])
# pil_img = Image.fromarray(imsave_ob, mode='RGB')
# pil_img.save(''+str(instances[0]['img_path'][-17:-4])+'.png', 'PNG')
# img_ob.save(imgpath[:-56] + '' + imgpath[-17:-4] + '_overlay_pre_gt.png','PNG')
# pil_img.save(imgpath[:-56] + "es" + '/pre_overlay/' + imgpath[-29:-4] + '_overlay_pre_gt.png','PNG')
polys = []
for i, instance in enumerate(instances):
poly = preds[i]
poly = poly * data['patch_w'][i] # to (224 224)
poly[:, 0] += data['starting_point'][i][0]
poly[:, 1] += data['starting_point'][i][1]
pred_sp = pred_spline[i]
pred_sp = pred_sp * data['patch_w'][i]
pred_sp[:, 0] += data['starting_point'][i][0]
pred_sp[:, 1] += data['starting_point'][i][1]
instance['spline_pos'] = pred_sp.tolist()
polys.append(poly)
if save:
img_h, img_w = instance['img_height'], instance['img_width']
predicted_poly = []
pred_mask = np.zeros((img_h, img_w), dtype=np.uint8)
utils.draw_poly(pred_mask, poly.astype(np.int))
predicted_poly.append(poly.tolist())
## overlap pred_poly with original image
img_path = str(instance['img_path'])
# get gt_polys
# polys_gt = data['instance'][0]['components'][0]['poly']
polys_gt = instance['components'][0]['poly']
# get polys_pre
polys_pre = predicted_poly[0]
polys_gt_final = [tuple(item) for item in polys_gt]
polys_pre_final = [tuple(item) for item in polys_pre]
if len(polys_pre_final) == 1:
print('error!! ', img_path[10:])
print('before optimize is: ', output['pred_polys'][i])
## <<overlay gt_pre>>
else:
pass
##for 30 test volumes
#utils.overlap_pre_gt('/home/lxj/work_station/' + img_path[10:], polys_pre_final, polys_gt_final)
## for 5-cross-val experiments
#utils.overlap_pre_gt('/home/lxj/work_station/' + img_path[25:], polys_pre_final, polys_gt_final)
# utils.overlap_pre_gt('/home/lxj/' + img_path[10:], polys_pre_final, polys_gt_final) # only for citiscapse
gt_mask = utils.get_full_mask_from_instance(
self.opts['dataset']['train_val']['min_area'],
instance)
instance['my_predicted_poly'] = predicted_poly
instance_id = instance['img_path'][-17:-4]
# image_id = instance['image_id']
pred_mask_fname = os.path.join(self.output_dir, '{}_pred.png'.format(instance_id))
instance['pred_mask_fname'] = os.path.relpath(pred_mask_fname, self.output_dir)
gt_mask_fname = os.path.join(self.output_dir, '{}_gt.png'.format(instance_id))
instance['gt_mask_fname'] = os.path.relpath(gt_mask_fname, self.output_dir)
instance['n_corrections'] = 0
info_fname = os.path.join(self.output_dir, '{}_info.json'.format(instance_id))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
sio.imsave(pred_mask_fname, pred_mask)
sio.imsave(gt_mask_fname, gt_mask)
# print '==> dumping json'
with open(info_fname, 'w') as f:
json.dump(instance, f, indent=2)
return polys
def process_outputs(self, data, output, save=True):
"""
Process outputs to get final outputs for the whole image
Optionally saves the outputs to a folder for evaluation
"""
#pred_spline = output['pred_polys']
pred_spline = output['pred_polys']
preds = self.spline.sample_point(pred_spline)
torch.cuda.synchronize()
preds = preds.cpu().numpy()
pred_spline = pred_spline.cpu()
pred_spline = pred_spline.numpy()
instances = data['instance']
polys = []
for i, instance in enumerate(instances):
poly = preds[i]
poly = poly * data['patch_w'][i]
poly[:, 0] += data['starting_point'][i][0]
poly[:, 1] += data['starting_point'][i][1]
pred_sp = pred_spline[i]
pred_sp = pred_sp * data['patch_w'][i]
pred_sp[:, 0] += data['starting_point'][i][0]
pred_sp[:, 1] += data['starting_point'][i][1]
instance['spline_pos'] = pred_sp.tolist()
polys.append(poly)
if save:
img_h, img_w = instance['img_height'], instance['img_width']
predicted_poly = []
pred_mask = np.zeros((img_h, img_w), dtype=np.uint8)
utils.draw_poly(pred_mask, poly.astype(np.int))
#utils.get_edge_mask(poly.astype(np.int),pred_mask)
#utils.get_vertices_mask(poly.astype(np.int),pred_mask)
predicted_poly.append(poly.tolist())
gt_mask = utils.get_full_mask_from_instance(
self.opts['dataset']['train_val']['min_area'],
instance)
instance['my_predicted_poly'] = predicted_poly
instance_id = instance['instance_id']
image_id = instance['image_id']
pred_mask_fname = os.path.join(self.output_dir, '{}_pred.png'.format(instance_id))
instance['pred_mask_fname'] = os.path.relpath(pred_mask_fname, self.output_dir)
gt_mask_fname = os.path.join(self.output_dir, '{}_gt.png'.format(instance_id))
instance['gt_mask_fname'] = os.path.relpath(gt_mask_fname, self.output_dir)
instance['n_corrections'] = 0
info_fname = os.path.join(self.output_dir, '{}_info.json'.format(instance_id))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
sio.imsave(pred_mask_fname, pred_mask)
sio.imsave(gt_mask_fname, gt_mask)
# print '==> dumping json'
with open(info_fname, 'w') as f:
json.dump(instance, f, indent=2)
return polys
def train(self, epoch):
print 'Starting training'
self.model.train()
accum = defaultdict(float)
#loss_loc = WHD_losses.WeightedHausdorffDistance(resized_height=self.opts['p_num'],resized_width=self.opts['p_num'],
# return_2_terms=True,
# device=device)
#loss_loc = WHD_losses.AveragedHausdorffLoss()
loss_loc = losses.WeightedHausdorffDistance(resized_height=224,
resized_width=224,
return_2_terms=True,
device=device)
# focalloss = focal.FocalLoss(None,None,None,'mean')
focalloss = focal.FocalLoss()
# To accumulate stats for printing
for step, data in enumerate(self.train_loader):
if len(data['img']) == 1:
continue
if self.opts['get_point_annotation']:
img = data['img'].to(device)
annotation = data['annotation_prior'].to(device).unsqueeze(1)
img = torch.cat([img, annotation], 1)
else:
img = data['img'].to(device)
self.optimizer.zero_grad()
if self.global_step % self.opts['val_freq'] == 0 and not self.opts[
'debug']:
self.validate()
self.save_checkpoint(epoch)
output = self.model.forward(img, data['fwd_poly'])
loss_sum = 0
pred_cps = output['pred_polys'][-1]
pred_polys = self.spline.sample_point(pred_cps)
# print(pred_polys.shape)
# print(output['vertex_logits'].shape)
gt_right_order, poly_mathcing_loss_sum = losses.poly_mathcing_loss(
self.opts['p_num'],
pred_polys,
data['gt_poly'].to(device),
loss_type=self.opts['loss_type'])
# add by dzh contour refine
## Initializing Contour Box
level_set_config_dict = {
'step_ckpts': [50],
'lambda_': 0.0,
'alpha': 1,
'smoothing': 1,
'render_radius': -1,
'is_gt_semantic': True,
'method': 'MLS',
'balloon': 1,
'threshold': 0.99,
'merge_weight': 0.5
}
cbox = ContourBox.LevelSetAlignment(n_workers=1,
fn_post_process_callback=None,
config=level_set_config_dict)
# print('-------------shape--------------------')
output_contour, _ = cbox(
{
'seg': np.expand_dims(data['edge_mask'], 0),
'bdry': None
},
np.expand_dims(
output['edge_logits'].view(
data['edge_mask'].shape).cpu().detach().numpy(), 0))
masks_step = output_contour[0, :, 0, :, :]
#--------add by dzh 7.18
edge_annotation_loss = 0
curr_fp_edge_loss = losses.fp_edge_loss(
torch.from_numpy(masks_step).to(
device
), #self.opts['fp_weight'] * losses.fp_edge_loss(torch.from_numpy(masks_step).to(device),
output['edge_logits']
) #data['edge_mask'] torch.from_numpy(masks_step)
edge_annotation_loss += curr_fp_edge_loss
tt = []
#pred_poly_mask = np.zeros((36, 36), np.float32)
for i in range(pred_polys.shape[0]):
pred_poly_mask = np.zeros((224, 224), dtype=np.float32)
ff = np.floor(pred_polys[i].detach().cpu().numpy() *
36).astype(np.int32)
if not isinstance(ff, list):
ff = [ff]
for p in ff:
pred_poly_mask = utils.draw_poly(pred_poly_mask, p)
#ff=utils.poly01_to_poly0g(pred_polys[i].detach().cpu().numpy(), 35)
# pred_poly_mask = utils.get_vertices_mask_36(ff, pred_poly_mask)
tt.append(pred_poly_mask)
tt1 = np.array(tt, dtype=np.float32)
pred_poly_mask11 = torch.from_numpy(tt1).cuda()
ll1 = pred_poly_mask11
#ll1 = output['vertex_logits'].view(output['vertex_logits'].shape[0],28,28)
jjj = []
for tt in range(ll1.shape[0]):
jjj.append([224, 224])
#jjj = [[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28],[28,28]]
# print(data['poly_mask'].shape)
kk = []
poly_mask_ori = data['poly_mask']
for hh in range(ll1.shape[0]):
zzz = torch.FloatTensor(poly_mask_ori[hh].astype(
np.float32)).cuda()
#zzz = torch.FloatTensor(data['gt_orig_poly'][hh]).cuda()
kk.append(zzz)
#zzz = torch.from_numpy(data['gt_orig_poly'][0])
# print(ll1.shape)
# print(kk.shape)
#ll1,kk
term1, term2 = loss_loc.forward(
ll1, kk,
torch.FloatTensor(np.array(jjj, dtype=np.float32)).cuda())
#fp_vertex_loss = self.opts['fp_weight'] * (term1+term2)
fp_vertex_loss = 0.1 * (term1 + term2) + poly_mathcing_loss_sum
#fp_vertex_loss = poly_mathcing_loss_sum + self.opts['fp_weight']* 0.1 * (term1+term2)
loss_sum += fp_vertex_loss
loss_sum += edge_annotation_loss # + self.opts['fp_weight'] * (term1+term2)
################iou loss function#################
#preds= pred_polys.detach().data.cpu().numpy()
#iou_loss = 0
#orig_poly = data['orig_poly']
#for i in range(preds.shape[0]):
# curr_pred_poly = np.floor(preds[i] * 224).astype(np.int32)
# curr_gt_poly = np.floor(orig_poly[i] * 224).astype(np.int32)
# cur_iou, masks = metrics.iou_from_poly(np.array(curr_pred_poly, dtype=np.int32),
# np.array(curr_gt_poly, dtype=np.int32),
# 224, 224)
# iou_loss += cur_iou
#iou_loss = -iou_loss / preds.shape[0]
#loss_sum += 0.1 * iou_loss
################iou loss function#################
with torch.no_grad():
iou = 0
gt_mask_0 = []
pred_mask_0 = []
orig_poly = data['orig_poly']
preds = pred_polys.detach().data.cpu().numpy()
# iou_filter = []
for i in range(preds.shape[0]):
curr_pred_poly = np.floor(preds[i] * 224).astype(np.int32)
curr_gt_poly = np.floor(orig_poly[i] * 224).astype(
np.int32)
cur_iou, masks = metrics.iou_from_poly(
np.array(curr_pred_poly, dtype=np.int32),
np.array(curr_gt_poly, dtype=np.int32), 224, 224)
gt_mask_0.append(masks[1])
pred_mask_0.append(masks[0])
gt_mask_1 = torch.from_numpy(
np.array(gt_mask_0)).to(device).float()
pred_mask_1 = torch.from_numpy(
np.array(pred_mask_0)).to(device).float()
# mask_loss = focalloss(pred_mask_1, gt_mask_1)
# mask_loss = losses.class_balanced_cross_entropy_loss(pred_mask_1, gt_mask_1)
# pred111=pred_mask_1.view(pred_mask_1.shape[0],1,224,224)
#mask_loss = 100 * focalloss((pred_mask_1/255), (gt_mask_1/255))
mask_loss = torch.sum(
torch.abs(gt_mask_1 / 250 - pred_mask_1 / 250))
loss_sum += torch.mean(mask_loss)
# # iou_filter.append(1 if cur_iou>self.opts['iou_filter'] else 0)
# iou += cur_iou
# iou = iou / preds.shape[0]
# # iou_filter = np.array(iou_filter)
# # iou_filter = torch.from_numpy(iou_filter).to(device).float()
# loss_sum += (-iou)
# if self.opts['iou_filter']>0:
# loss_sum = (loss_sum + (1-iou)) * iou_filter
# loss_sum = torch.mean(loss_sum)
loss_sum.backward()
if 'grad_clip' in self.opts.keys():
nn.utils.clip_grad_norm_(self.model.parameters(),
self.opts['grad_clip'])
self.optimizer.step()
preds = pred_polys.detach().data.cpu().numpy()
with torch.no_grad():
# Get IoU
iou = 0
orig_poly = data['orig_poly']
for i in range(preds.shape[0]):
curr_pred_poly = np.floor(preds[i] * 224).astype(np.int32)
curr_gt_poly = np.floor(orig_poly[i] * 224).astype(
np.int32)
cur_iou, masks = metrics.iou_from_poly(
np.array(curr_pred_poly, dtype=np.int32),
np.array(curr_gt_poly, dtype=np.int32), 224, 224)
iou += cur_iou
iou = iou / preds.shape[0]
accum['loss'] += float(loss_sum.item())
accum['iou'] += iou
accum['length'] += 1
if self.opts['edge_loss']:
accum['edge_annotation_loss'] += float(
edge_annotation_loss.item())
print(
"[%s] Epoch: %d, Step: %d, Polygon Loss: %f, IOU: %f" \
% (str(datetime.now()), epoch, self.global_step, accum['loss'] / accum['length'], accum['iou'] / accum['length']))
if step % self.opts['print_freq'] == 0:
# Mean of accumulated values
for k in accum.keys():
if k == 'length':
continue
accum[k] /= accum['length']
# Add summaries
masks = np.expand_dims(masks, -1).astype(
np.uint8) # Add a channel dimension
#print(masks.shape)
masks = np.tile(masks, [1, 1, 1, 3]) # Make [2, H, W, 3]
img = (data['img'].cpu().numpy()[-1, ...] * 255).astype(
np.uint8)
img = np.transpose(img, [1, 2, 0]) # Make [H, W, 3]
self.writer.add_image('pred_mask', masks[0],
self.global_step)
self.writer.add_image('gt_mask', masks[1],
self.global_step)
self.writer.add_image('image', img, self.global_step)
self.writer.add_image(
'edge_acm_gt',
np.tile(
np.expand_dims(masks_step[preds.shape[0] - 1],
axis=-1).astype(np.uint8),
[1, 1, 3]), self.global_step)
#self.writer.add_image('ori_GT',
pred_edge_mask = np.tile(
np.expand_dims(
output['edge_logits'].cpu().numpy()[preds.shape[0]
- 1] * 255,
axis=-1).astype(np.uint8),
[1, 1, 3]).reshape(28, 28, 3)
#print(pred_edge_mask.shape)
self.writer.add_image('pred_edge', pred_edge_mask,
self.global_step)
for k in accum.keys():
if k == 'length':
continue
self.writer.add_scalar(k, accum[k], self.global_step)
print(
"[%s] Epoch: %d, Step: %d, Polygon Loss: %f, IOU: %f" \
% (str(datetime.now()), epoch, self.global_step, accum['loss'], accum['iou']))
accum = defaultdict(float)
del (output, masks, pred_polys, preds, loss_sum)
self.global_step += 1
def process_outputs(self, data, output, save=True):
"""
Process outputs to get final outputs for the whole image
Optionally saves the outputs to a folder for evaluation
"""
instances = data['instance']
polys = []
for i, instance in enumerate(instances):
# Postprocess polygon
poly = output['pred_polys'][i]
_, poly = utils.get_full_mask_from_xy(poly, self.grid_size,
data['patch_w'][i],
data['starting_point'][i],
instance['img_height'],
instance['img_width'])
polys.append(poly)
if save:
img_h, img_w = instance['img_height'], instance['img_width']
predicted_poly = []
# Paint pred mask
pred_mask = np.zeros((img_h, img_w), dtype=np.uint8)
poly = poly.astype(np.int)
utils.draw_poly(pred_mask, poly)
predicted_poly.append(poly.tolist())
# Paint GT mask
gt_mask = utils.get_full_mask_from_instance(
self.opts['dataset']['train_val']['min_area'], instance)
instance['my_predicted_poly'] = predicted_poly
instance_id = instance['instance_id']
image_id = instance['image_id']
pred_mask_fname = os.path.join(
self.output_dir, '{}_pred.png'.format(instance_id))
instance['pred_mask_fname'] = os.path.relpath(
pred_mask_fname, self.output_dir)
gt_mask_fname = os.path.join(self.output_dir,
'{}_gt.png'.format(instance_id))
instance['gt_mask_fname'] = os.path.relpath(
gt_mask_fname, self.output_dir)
instance['n_corrections'] = 0
info_fname = os.path.join(self.output_dir,
'{}_info.json'.format(instance_id))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
sio.imsave(pred_mask_fname, pred_mask)
sio.imsave(gt_mask_fname, gt_mask)
with open(info_fname, 'w') as f:
json.dump(instance, f, indent=2)
return polys
