def compute_triplet_loss(model,
batch,
random_proposal=False,
similarity_function=""):
n, c, h, w = batch["images"].shape
model.train()
O_dict = model(batch["images"].cuda())
O = O_dict["embedding_mask"]
n, c, h, w = O.shape
points = batch["points"]
batch["maskObjects"] = None
batch['maskClasses'] = None
batch["maskVoid"] = None
pointList = au.mask2pointList(points)["pointList"]
loss = torch.tensor(0.).cuda()
if len(pointList) == 0:
return loss
propDict = au.pointList2propDict(pointList,
batch,
single_point=True,
thresh=0.5)
Embeddings, Labels = propDict2EmbedLabels(O,
propDict,
random_proposal=True)
loss = triplet(Embeddings, torch.LongTensor(Labels))
return loss
def compute_fg_loss():
propDict = au.pointList2propDict(pointList,
batch,
single_point=single_point,
thresh=0.5)
background = propDict["background"]
propDict = propDict["propDict"]
yList = []
xList = []
for p in pointList:
yList += [p["y"]]
xList += [p["x"]]
fg_seeds = O[:, :, yList, xList]
n_seeds = fg_seeds.shape[-1]
prop_mask = np.zeros((h, w))
for i in range(n_seeds):
annList = propDict[i]["annList"]
if len(annList) == 0:
mask = np.zeros(points.squeeze().shape)
mask[propDict[i]["point"]["y"], propDict[i]["point"]["x"]] = 1
else:
if random_proposal:
ann_i = np.random.randint(0, len(annList))
mask = annList[ann_i]["mask"]
else:
mask = annList[0]["mask"]
mask_ind = np.where(mask)
prop_mask[mask != 0] = (i + 1)
f_A = fg_seeds[:, :, [i]]
# Positive Embeddings
n_pixels = mask_ind[0].shape[0]
P_ind = np.random.randint(0, n_pixels, 100)
yList = mask_ind[0][P_ind]
xList = mask_ind[1][P_ind]
fg_P = O[:, :, yList, xList]
ap = -torch.log(similarity_function(f_A, fg_P))
loss += ap.mean()
# Get Negatives
if n_seeds > 1:
N_ind = [j for j in range(n_seeds) if j != i]
f_N = fg_seeds[:, :, N_ind]
an = -torch.log(1. - similarity_function(f_A, f_N))
loss += an.mean()
def predict(self, batch, predict_method="counts"):
self.eval()
img = batch["images"]
padded_size = (int(np.ceil(img.shape[2]/8)*8), int(np.ceil(img.shape[3]/8)*8))
p2d = (0, padded_size[1] - img.shape[3], 0, padded_size[0] - img.shape[2])
img = F.pad(img, p2d)
dheight = int(np.ceil(img.shape[2]/8))
dwidth = int(np.ceil(img.shape[3]/8))
n, c, h, w = img.shape
lcfcn_pointList = au.mask2pointList(batch["points"])["pointList"]
counts = np.zeros(self.n_classes-1)
if len(lcfcn_pointList) == 0:
return {"blobs": np.zeros((h,w), int), "annList":[], "counts":counts}
propDict = au.pointList2propDict(lcfcn_pointList, batch,
proposal_type="sharp",
thresh=0.5)
aff_mat = torch.pow(self.aff.forward(img.cuda(), True), self.beta)
trans_mat = aff_mat / torch.sum(aff_mat, dim=0, keepdim=True)
for _ in range(self.logt):
trans_mat = torch.matmul(trans_mat, trans_mat)
import ipdb; ipdb.set_trace() # breakpoint ac0c04d2 //
for prop in propDict["propDict"]:
mask = prop["annList"][0]["mask"]
mask = torch.FloatTensor(mask)[None]
mask = F.pad(mask, p2d)
mask_arr = F.avg_pool2d(mask, 8, 8)
mask_vec = mask_arr.view(1, -1)
mask_rw = torch.matmul(mask_vec.cuda(), trans_mat)
mask_rw = mask_rw.view(1, dheight, dwidth)
mask_rw = torch.nn.Upsample((img.shape[2], img.shape[3]), mode='bilinear')(mask_rw[None])
import ipdb; ipdb.set_trace() # breakpoint 89e7f819 //
cam_rw = torch.nn.Upsample((img.shape[2], img.shape[3]), mode='bilinear')(cam_rw)
_, cam_rw_pred = torch.max(cam_rw, 1)
res = np.uint8(cam_rw_pred.cpu().data[0])[:h, :w]
if predict_method == "annList":
pass
else:
return img, res
def metric_base(O, batch, pointList=None):
n, c, h, w = O.shape
if pointList is None:
points = batch["points"]
batch["maskObjects"] = None
batch['maskClasses'] = None
batch["maskVoid"] = None
pointList = au.mask2pointList(points)["pointList"]
if len(pointList) == 0:
return None
if "single_point" in batch:
single_point = True
else:
single_point = False
propDict = au.pointList2propDict(pointList,
batch,
single_point=single_point,
thresh=0.5)
background = propDict["background"]
propDict = propDict["propDict"]
yList = []
xList = []
for p in pointList:
yList += [p["y"]]
xList += [p["x"]]
return {
"xList": xList,
"yList": yList,
"background": background,
"propDict": propDict
}
def predict(self, batch, proposal_type="sharp", predict_method="blobs"):
self.sanity_checks(batch)
model = self
self.eval()
n,c,h,w = batch["images"].shape
model.eval()
lcfcn_pointList = batch["lcfcn_pointList"]
counts = np.zeros(model.n_classes-1)
if len(lcfcn_pointList) == 0:
return {"blobs": np.zeros((h,w), int), "annList":[], "counts":counts}
propDict = au.pointList2propDict(lcfcn_pointList, batch, proposal_type=proposal_type,
thresh=0.5)
# Segmenter
O = model(batch["images"].cuda())["embedding_mask"]
seedList = au.propDict2seedList(propDict)
fg_bg_seeds = au.CombineSeeds(seedList)
blobs_categoryDict = model.get_embedding_blobs(O, fg_bg_seeds)
blobs = blobs_categoryDict["blobs"]
categoryDict = blobs_categoryDict["categoryDict"]
if predict_method == "BestDice":
blob_dict = au.blobs2BestDice(blobs, categoryDict,
propDict, batch)
blobs = blob_dict["blobs"]
annList = blob_dict["annList"]
else:
annList = au.blobs2annList(blobs, categoryDict, batch)
return {"blobs":blobs, "annList":annList, "counts":counts}
def compute_metric_loss_mean(O, batch, random_proposal=False):
n, c, h, w = O.shape
similarity_function = au.log_pairwise_mean
points = batch["points"]
batch["maskObjects"] = None
batch['maskClasses'] = None
batch["maskVoid"] = None
pointList = au.mask2pointList(points)["pointList"]
loss = torch.tensor(0.).cuda()
if len(pointList) == 0:
return loss
if "single_point" in batch:
single_point = True
else:
single_point = False
propDict = au.pointList2propDict(pointList,
batch,
single_point=single_point,
thresh=0.5)
# img = ms.pretty_vis(batch["images"], propDict["propDict"][0]["annList"],dpi=100)
# ms.images(img)
# import ipdb; ipdb.set_trace() # breakpoint 6758c0a1 //
background = propDict["background"]
propDict = propDict["propDict"]
yList = []
xList = []
for p in pointList:
yList += [p["y"]]
xList += [p["x"]]
fg_seeds = O[:, :, yList, xList]
n_seeds = fg_seeds.shape[-1]
prop_mask = np.zeros((h, w))
for i in range(n_seeds):
annList = propDict[i]["annList"]
if len(annList) == 0:
mask = np.zeros(points.squeeze().shape)
mask[propDict[i]["point"]["y"], propDict[i]["point"]["x"]] = 1
else:
if random_proposal:
ann_i = np.random.randint(0, len(annList))
mask = annList[ann_i]["mask"]
else:
mask = annList[0]["mask"]
mask_ind = np.where(mask)
prop_mask[mask != 0] = (i + 1)
f_A = fg_seeds[:, :, [i]]
# Positive Embeddings
n_pixels = mask_ind[0].shape[0]
P_ind = np.random.randint(0, n_pixels, 100)
yList = mask_ind[0][P_ind]
xList = mask_ind[1][P_ind]
fg_P = O[:, :, yList, xList]
ap = -torch.log(similarity_function(f_A, fg_P))
loss += ap.mean()
# Get Negatives
if n_seeds > 1:
N_ind = [j for j in range(n_seeds) if j != i]
f_N = fg_seeds[:, :, N_ind]
an = -torch.log(1. - similarity_function(f_A, f_N))
loss += an.mean()
# # Extract background seeds
bg = np.where(background.squeeze())
n_pixels = bg[0].shape[0]
bg_ind = np.random.randint(0, n_pixels, n_seeds)
yList = bg[0][bg_ind]
xList = bg[1][bg_ind]
f_A = O[:, :, yList, xList]
bg_ind = np.random.randint(0, n_pixels, 100)
yList = bg[0][bg_ind]
xList = bg[1][bg_ind]
f_P = O[:, :, yList, xList]
# BG seeds towards BG pixels, BG seeds away from FG seeds
ap = -torch.log(similarity_function(f_A[:, :, None], f_P[:, :, :, None]))
an = -torch.log(1. - similarity_function(f_A[:, :, None], fg_seeds[:, :, :,
None]))
loss += ap.mean()
loss += an.mean()
if batch["dataset"][0] == "cityscapes" or batch["dataset"][0] == "coco2014":
n_max = 6
else:
n_max = 12
if f_A.shape[2] < n_max:
with torch.no_grad():
diff = similarity_function(
O.view(1, c, -1)[:, :, :, None],
torch.cat([fg_seeds, f_A], 2)[:, :, None])
labels = diff.max(2)[1] + 1
labels = labels <= n_seeds
labels = labels.squeeze().reshape(h, w)
bg = labels.cpu().long() * torch.from_numpy(background)
# ms.images(labels.cpu().long()*torch.from_numpy(background))
# Extract false positive pixels
bg_ind = np.where(bg.squeeze())
n_P = bg_ind[0].shape[0]
if n_P != 0:
A_ind = np.random.randint(0, n_P, n_seeds)
f_P = O[:, :, bg_ind[0][A_ind], bg_ind[1][A_ind]]
ap = -torch.log(
similarity_function(f_A[:, :, None], f_P[:, :, :, None]))
an = -torch.log(1. - similarity_function(f_P[:, :, None],
fg_seeds[:, :, :, None]))
# if i < 3:
loss += ap.mean()
loss += an.mean()
# if visualize:
# diff = log_func(O.view(1,64,-1)[:,:,:,None], torch.cat([se, f_A], 2)[:,:,None])
# labels = diff.max(2)[1] + 1
# labels[labels > n_se] = 0
# labels = labels.squeeze().reshape(h,w)
# ms.images(batch["images"], ms.t2n(labels),denorm=1, win="labels")
# ms.images(batch["images"], prop_mask.astype(int), denorm=1, win="true")
# ms.images(batch["images"], background.astype(int), denorm=1, win="bg")
return loss / max(n_seeds, 1)
def wiseaffinity_loss(model, batch):
# model.lcfcn
images = batch["images"].cuda()
n, c, h, w = images.shape
pointList = au.mask2pointList(batch["points"])["pointList"]
loss = torch.tensor(0.).cuda()
loss_bg = torch.tensor(0.).cuda()
loss_fg = torch.tensor(0.).cuda()
if len(pointList) == 0:
return loss
propDict = au.pointList2propDict(pointList,
batch,
single_point=True,
thresh=0.5)
propList = propDict["propDict"]
pred_dict = model.lcfcn.predict(batch, predict_method="pointList")
blobs = pred_dict["blobs"]
probs = pred_dict["probs"]
blobs_components = morph.label(blobs != 0)
image_pad = ms.pad_image(images)
_, _, dheight, dwidth = image_pad.shape
trans_mat = model.aff.forward_trans(image_pad)
# bg_probs = probs[:,[0]]
for i in range(len(propList)):
prop = propList[i]
if not len(prop["annList"]):
continue
proposal_mask = torch.LongTensor(prop["annList"][0]["mask"]).cuda()
# proposal_mask = F.interpolate(proposal_mask, size=(dheight//8, dwidth//8))
y, x = prop["point"]["y"], prop["point"]["x"]
category = blobs[:, y, x][0]
instance = blobs_components[:, y, x][0]
blob_mask_ind = blobs_components == instance
O = torch.FloatTensor(probs[:, [0, category]]).detach()
O[:, 1] = O[:, 1] * torch.FloatTensor(blob_mask_ind.astype(float))
O[:, 1] = O[:, 1].clamp(1e-10)
O_scale = F.interpolate(O, size=(dheight // 8, dwidth // 8))
O_scale = O_scale.view(1, 2, -1).cuda()
O_rw = torch.matmul(O_scale, trans_mat)
O_rw = O_rw.view(1, 2, dheight // 8, dwidth // 8)
O_final = F.interpolate(O_rw, size=(h, w))
S_log = F.log_softmax(O_final, 1)
loss_bg += F.nll_loss(S_log,
proposal_mask[None],
ignore_index=1,
size_average=True)
loss_fg += F.nll_loss(S_log,
proposal_mask[None],
ignore_index=0,
size_average=True)
return (loss_bg + loss_fg) / len(propList)