def update(self, img, detections=None):
if not detections:
detections = self.detector.detect(img)
for d in detections:
cv2.rectangle(
img, (d.bbox.x, d.bbox.y),
(d.bbox.x + d.bbox.width, d.bbox.y + d.bbox.height),
(0, 0, 255), 3)
if len(detections) > 0:
location_weights = np.empty(len(detections), dtype=float)
for idx, det in enumerate(detections):
obs = np.array(
[det.bbox.x, det.bbox.y, det.bbox.width, det.bbox.height])
location_weights[idx] = math.exp(
2.0 * (-1.0 +
utils.intersection_over_union(self.reference, obs)))
psi = np.empty((len(self.states), len(detections)), dtype=float)
for i, state in enumerate(self.states):
for j, det in enumerate(detections):
obs = np.array([
det.bbox.x, det.bbox.y, det.bbox.width, det.bbox.height
])
#print utils.intersection_over_union(state, obs)
psi[i, j] = location_weights[j] * math.exp(
2.0 *
(-1.0 + utils.intersection_over_union(state, obs)))
tau = psi.sum(axis=0)
self.weights = self.weights * (1 - self.DETECTION_RATE) + psi.sum(
axis=1) / float(self.LAMBDA_C * self.PDF_C)
self.resample()
def forward(self, predictions, target):
predictions = predictions.reshape(-1, self.S, self.S,
self.B * 5 + self.C)
# Note: here we only have one targer. We compare both predicted box with the same x,y,w,h from the last five channels in targets
iou_b1 = intersection_over_union(predictions[..., 21:25],
target[..., 21:25])
iou_b2 = intersection_over_union(predictions[..., 26:30],
target[..., 21:25])
ious = torch.cat([iou_b1.unsqueeze(0), iou_b2.unsqueeze(0)], dim=0)
iou_maxes, bestbox_idx = torch.max(
ious, dim=0
) # bestbox_idx is the index to indicate which box is response to predict
exists_obj = target[..., 20].unsqueeze(
3
) #to keep the third dimension. Is there an object is cell i? n*7*7*1
#coord loss
box_predictions = exists_obj * (( #get the best box
bestbox_idx * predictions[..., 26:30] +
(1 - bestbox_idx) * predictions[
..., 21:
25] #bestbox_idx suppose to be 0 or 1, index to indicate whcih box to use
))
box_targets = exists_obj * target[..., 21:25]
box_predictions[..., 2:4] = torch.sign(
box_predictions[..., 2:4]) * torch.sqrt(
torch.abs(box_predictions[..., 2:4] +
1e-6)) # handle possible negative values
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
# (N,S,S,4) -> (N*S*S, 4)
box_loss = self.mse(
torch.flatten(box_predictions, end_dim=-2),
torch.flatten(box_targets, end_dim=-2),
)
#obj loss
pred_box = (bestbox_idx * predictions[..., 25:26] +
(1 - bestbox_idx) * predictions[..., 20:21])
obj_loss = self.mse(torch.flatten(exists_obj * pred_box),
torch.flatten(exists_obj * target[..., 20:21]))
no_obj_loss = self.mse(
torch.flatten((1 - exists_obj) * predictions[..., 25:26],
start_dim=1),
torch.flatten((1 - exists_obj) * target[..., 20:21], start_dim=1))
no_obj_loss += self.mse(
torch.flatten((1 - exists_obj) * predictions[..., 20:21],
start_dim=1),
torch.flatten((1 - exists_obj) * target[..., 20:21], start_dim=1))
#class loss
class_loss = self.mse(
torch.flatten(exists_obj * predictions[..., :20], end_dim=-2),
torch.flatten(exists_obj * target[..., :20], end_dim=-2))
loss = (self.lambda_coord * box_loss + obj_loss +
self.lambda_noobj * no_obj_loss + class_loss)
return loss
def forward(self, predictions, target):
predictions = predictions.reshape(-1, self.S, self.S, self.B,
self.C + self.B * S)
iou_b1 = intersection_over_union(predictions[..., 21:25],
target[..., 21:25])
iou_b2 = intersection_over_union(predictions[..., 26:30],
target[..., 21:25])
ious = torch.cat([iou_b1.unsqueeze(0), iou_b1.unsqueeze(0)], dim=0)
iou_maxes, best_box = torch.max(ious, dim=0)
exists_box = target[..., 20:21].unsqueeze(3) #Iobj_i
# For Box Coordinates
box_predictions = exists_box * (
(best_box * predictions[..., 26:30] +
(1 - best_box) * predictions[..., 21:25]))
box_targets = exists_box * target[..., 21:25]
box_predictions[..., 2:4] = torch.sign(
box_predictions[..., 2:4]) * torch.sqrt(
torch.abs(box_predictions[..., 2:4] + 1e-6))
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
# (N, S, S, 4) -> (N*S*S, 4)
box_loss = self.mse(
torch.flatten(box_predictions, end_dim=-2),
torch.flatten(box_targets, end_dim=-2),
)
# For Object Loss
pred_box = (best_box * predictions[..., 25:26] +
(1 - best_box) * predictions[..., 20:21])
# (N * S * S)
object_loss = self.mse(torch.flatten(exists_box * pred_box),
torch.flatten(exists_box * target[..., 20:21]))
# For No Object Loss
# (N, S, S, 1) -> (N, S * S)
no_object_loss = self.mse(
torch.flatten((1 - exists_box) * predictions[..., 20:21],
start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1))
no_object_loss += self.mse(
torch.flatten((1 - exists_box) * predictions[..., 25:26],
start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1))
# For Class Loss
class_loss = self.mse(
torch.flatten(exists_box * predictions[..., :20], end_dim=2),
torch.flatten(exists_box * target[..., :20], end_dim=-2),
)
loss = (self.lambda_coord * box_loss + object_loss +
self.lambda_noobj * no_object_loss + class_loss)
return loss
def forward(self, predictions, target):
predictions = predictions.reshape(-1, self.S, self.S,
self.C + self.B * 5)
# 计算预测框和真实框的iou
iou_b1 = intersection_over_union(predictions[..., 21:25],
target[..., 21:25])
iou_b2 = intersection_over_union(predictions[..., 26:30],
target[..., 21:25])
ious = torch.cat([iou_b1.unsqueeze(0), iou_b2.unsqueeze(0)], dim=0)
iou_maxes, bestbox = torch.max(ious, dim=0) # 得到最合适的框
exists_box = target[..., 20].unsqueeze(3)
"""
对于每个grid box，最终有两个预测框，所以bestbox的取值只有0和1两种取值
exists_box:取值为0或1，表示这里是否有真实框
bestbox=0: 表示第一个框的预测是正确的，那么留下第一个框的值，
bestbox=1：表示第二个框的预测是正确的，就留下第二个框的值
box_prediction包含了（x, y, width, height)
损失函数.png 红框
"""
box_prediction = exists_box * (
(bestbox * predictions[..., 26:30] +
(1 - bestbox) * predictions[..., 21:25]))
box_targets = exists_box * target[..., 21:25]
# sign 判断数的正负号， 对w和h开根号
box_prediction[..., 2:4] = torch.sign(box_prediction[..., 2:4]) * \
torch.sqrt(torch.abs(box_prediction[..., 2:4]+ 1e-6 ))
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
box_loss = self.mse(torch.flatten(box_prediction, end_dim=-2),
torch.flatten(box_targets, end_dim=-2))
"""损失函数.png 蓝框 第一行"""
pred_box = (bestbox * predictions[..., 25:26] +
(1 - bestbox) * predictions[..., 20:21])
object_loss = self.mse(torch.flatten(exists_box * pred_box),
torch.flatten(exists_box * target[..., 20:21]))
"""
损失函数.png 蓝框 第二行
将没有物体的框筛选出来，和真实框做loss， 这里因为有两个预测框，所以要加两次
"""
no_object_loss = self.mse(
torch.flatten((1 - exists_box) * predictions[..., 20:21],
start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21]))
no_object_loss += self.mse(
torch.flatten((1 - exists_box) * predictions[..., 25:26],
start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1))
"""损失函数.png 绿框"""
class_loss = self.mse(
torch.flatten(
exists_box * predictions[..., :20],
end_dim=-2,
), torch.flatten(exists_box * target[..., 20], end_dim=-2))
loss = (self.lambda_coord * box_loss + object_loss +
self.lambda_noobj * no_object_loss + class_loss)
return loss
def forward(self, predictions, target):
preds = predictions.reshape(-1, self.S, self.S, self.C + self.B * 5)
#these are the two boxes per cell
iou_b1 = intersection_over_union(
preds[..., 21:25],
target[..., 21:25]) #0-19 class probs, 21-24 4 bb vals
iou_b2 = intersection_over_union(preds[..., 26:30],
target[..., 21:25]) #26-29 bb vals
ious = torch.cat([iou_b1.unsqueeze(0), iou_b2.unsqueeze(0)], dim=0)
iou_maxes, bestbox = torch.max(ious, dim=0)
exists_box = target[..., 20].unsqueeze(3) #if box is in object i
##Box Coordinate Loss, coordinates are 2 and 3
box_preds = exists_box * ((bestbox * preds[..., 26:30] +
(1 - bestbox) * preds[..., 21:25]))
print(target[:, 21:25])
box_targets = exists_box * target[..., 21:25]
box_preds[..., 2:4] = torch.sign(box_preds[..., 2:4]) * torch.sqrt(
torch.abs(box_preds[..., 2:4] + 1e-6))
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
box_loss = self.mse(torch.flatten(box_preds, end_dim=-2),
torch.flatten(box_targets, end_dim=-2))
##Object Loss
pred_box = (bestbox * preds[..., 25:26] +
(1 - bestbox) * preds[..., 20:21])
object_loss = self.mse(torch.flatten(exists_box * pred_box),
torch.flatten(exists_box * target[..., 20:21]))
#no object loss, take loss for both
no_object_loss = self.mse(
torch.flatten((1 - exists_box) * preds[..., 20:21], start_dim=1),
torch.flatten((1 - exists_box) * preds[..., 20:21], start_dim=1))
no_object_loss += self.mse(
torch.flatten((1 - exists_box) * preds[..., 25:26], start_dim=1),
torch.flatten((1 - exists_box) * preds[..., 25:26], start_dim=1))
##Class Loss
class_loss = self.mse(
torch.flatten(
exists_box * preds[..., :20],
end_dim=-2,
), torch.flatten(
exists_box * target[..., :20],
end_dim=-2,
))
#total loss
loss = self.lambda_coord * box_loss + object_loss + self.lambda_obj * no_object_loss + class_loss
return loss
def forward(self, predictions, target):
predictions = predictions.reshape(-1, self.S, self.S, self.C +
self.B * 5) # N , S , S , 30
iou_b1 = intersection_over_union(
predictions[..., self.C + 1:self.C + 5],
target[..., self.C + 1:self.C + 5]) # num_examples , iou
iou_b2 = intersection_over_union(
predictions[..., self.C + 6:self.C + 10],
target[..., self.C + 1:self.C + 5]) # num_examples , iou
ious = torch.cat([iou_b1.unsqueeze(0), iou_b2.unsqueeze(0)], dim=0) #
iou_maxes, best_box = torch.max(ious, dim=0)
exists_box = target[..., self.C].unsqueeze(3)
box_targets = exists_box * target[..., self.C + 1:self.C + 5]
box_predictions = exists_box * (
(best_box * predictions[..., self.C + 6:self.C + 10] +
(1 - best_box) * predictions[..., self.C + 1:self.C + 5]))
box_predictions[..., 2:4] = torch.sign(
box_predictions[..., 2:4]) * torch.sqrt(
torch.abs(box_predictions[..., 2:4] + 1e-6)) #
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
box_loss = self.mse(torch.flatten(box_predictions, end_dim=-2),
torch.flatten(box_targets, end_dim=-2))
# object loss (check if its exist)
pred_box = (best_box * predictions[..., self.C + 5:self.C + 6] +
(1 - best_box) * predictions[..., self.C:self.C + 1])
obj_loss = self.mse(
torch.flatten(exists_box * pred_box),
torch.flatten(exists_box * target[..., self.C:self.C + 1]))
# class loss
class_loss = self.mse(
torch.flatten(exists_box * predictions[..., :self.C], end_dim=-2),
torch.flatten(exists_box * target[..., :self.C], end_dim=-2))
#no object loss (N,S*S*1)
noobj_loss = self.mse(
torch.flatten(
(1 - exists_box) * predictions[..., self.C:self.C + 1],
start_dim=1),
torch.flatten((1 - exists_box) * target[..., self.C:self.C + 1],
start_dim=1))
noobj_loss += self.mse(
torch.flatten(
(1 - exists_box) * predictions[..., self.C + 5:self.C + 6],
start_dim=1),
torch.flatten((1 - exists_box) * target[..., self.C:self.C + 1],
start_dim=1))
# some all losses
loss = (obj_loss + class_loss + self.lambda_coord * box_loss +
self.lambda_noobj * noobj_loss)
return loss
def forward(self, preds, target, anchors):
obj = target[..., 0] == 1 # in paper this is Iobj_i
noobj = target[..., 0] == 0 # in paper this is Inoobj_i
#No object loss
no_object_loss = self.bce((preds[..., 0:1][noobj]),
(target[..., 0:1][noobj]))
#object loss
anchors = anchors.reshape(1, 3, 1, 1, 2) #p_w * exp(t_w)
box_preds = torch.cat([
self.sigmoid(preds[..., 1:3]), anchors * torch.exp(preds[..., 3:5])
],
dim=-1)
ious = intersection_over_union(box_preds[obj],
target[..., 1:5][obj]).detach()
object_loss = self.bce((preds[..., 0:1][obj]),
(ious * target[..., 0:1][obj]))
#box coordinate loss
preds[..., 1:3] = self.sigmoid(preds[..., 1:3])
target[...,
3:5] = torch.log(1e-16 + target[..., 3:5] / anchors) #inverse
box_loss = self.mse((preds[..., 1:5][obj]), (target[..., 1:5][obj]))
#class loss
class_loss = self.entropy((preds[..., 5:][obj]),
(target[..., 5][obj].long()))
return self.lambda_box * box_loss + self.lambda_obj * object_loss + self.lambda_noobj * no_object_loss + self.lambda_class * class_loss
def custom_loss(y_true,y_pred): ###y_pred=[no. of images, 5 , 5 , 7] in which 7=>pc,x,y,w,h,c1,c2 5,5 are cells y_true_class=y_true[...,5:] y_pred_class=y_pred[...,5:] y_true_conf=y_true[...,0:1] y_pred_conf=y_pred[...,0:1] obj=y_true[...,0]==1 #true for cells which have objects noobj=y_true[...,0]==0 #true for cells which does not have objects #no obj loss noobj_loss=tf.reduce_mean(K.binary_crossentropy((y_true_conf[noobj]),(y_pred_conf[noobj]),from_logits=True)) #obj loss box_pred=K.concatenate([K.sigmoid(y_pred_xy),(y_pred_wh)],axis=-1) ious=intersection_over_union(box_pred[obj],y_true[...,1:5][obj]) obj_loss=tf.reduce_mean(K.binary_crossentropy((y_true_conf[obj]),(y_pred_conf[obj]),from_logits=True)) #box cordinate loss box_loss=tf.reduce_mean(K.mean(K.square(K.concatenate([y_true[...,1:3],(y_true[...,3:5])])[obj]-K.concatenate([K.sigmoid(y_pred[...,1:3]),(y_pred[...,3:5])])[obj]), axis=-1)) #class loss class_loss=tf.reduce_mean(K.categorical_crossentropy((y_true_class[obj]),(y_pred_class[obj]),from_logits=True)) return ((0.8*noobj_loss)+(1.2*obj_loss)+(1*class_loss)+(5*box_loss))
def validate():
""" Get loss over validation set """
model.eval()
e_loss = 0
e_iou = 0
with torch.no_grad():
for iteration, (img, label) in enumerate(val_data_loader, 1):
img = img.to(device)
label = label.to(device)
label_out = model(img)
batch_loss = criterion(label_out, label.float())
kernel_loss = \
inverse_gaussian_regularization(
model.contour_integration_layer.lateral_e.weight,
model.contour_integration_layer.lateral_i.weight
)
total_loss = batch_loss + lambda1 * kernel_loss
e_loss += total_loss.item()
preds = (torch.sigmoid(label_out) > detect_thres)
e_iou += utils.intersection_over_union(
preds.float(), label.float()).cpu().detach().numpy()
e_loss = e_loss / len(val_data_loader)
e_iou = e_iou / len(val_data_loader)
# print("Val Loss = {:0.4f}, IoU={:0.4f}".format(e_loss, e_iou))
return e_loss, e_iou
def forward(self, predictions, target, anchors):
obj = target[..., 0] == 1
noobj = target[..., 0] == 0
no_object_loss = self.bce(
(predictions[..., 0:1][noobj]),
(target[..., 0:1][noobj]),
)
anchors = anchors.reshape(1, 3, 1, 1, 2)
box_preds = torch.cat([
self.sigmoid(predictions[..., 1:3]),
torch.exp(predictions[..., 3:5]) * anchors
],
dim=-1)
ious = intersection_over_union(box_preds[obj],
target[..., 1:5][obj]).detach()
object_loss = self.mse(self.sigmoid(predictions[..., 0:1][obj]),
ious * target[..., 0:1][obj])
predictions[...,
1:3] = self.sigmoid(predictions[...,
1:3]) # x,y coordinates
target[..., 3:5] = torch.log((1e-16 + target[..., 3:5] / anchors))
box_loss = self.mse(predictions[..., 1:5][obj], target[..., 1:5][obj])
class_loss = self.entropy(
(predictions[..., 5:][obj]),
(target[..., 5][obj].long()),
)
return (self.lambda_box * box_loss + self.lambda_obj * object_loss +
self.lambda_noobj * no_object_loss +
self.lambda_class * class_loss)
def forward(self, predictions, target, anchors):
obj = target[..., 0] == 1
noObj = target[..., 0] == 0
# No object loss
no_object_loss = self.BCE((predictions[..., 0:1][noObj]), (target[..., 0:1][noObj]),)
# Object loss
anchors = anchors.reshape(1, 3, 1, 1, 2)
box_predictions = torch.cat([self.sigmoid(predictions[..., 1:3]), torch.exp(predictions[..., 3:5]) * anchors],dim=-1)
ious = intersection_over_union(box_predictions[obj], target[..., 1:5][obj]).detach()
object_loss = self.mean_square_error(self.sigmoid(predictions[..., 0:1][obj]), ious * target[..., 0:1][obj])
# Box coordinates loss
predictions[..., 1:3] = self.sigmoid(predictions[..., 1:3])
target[..., 3:5] = torch.log((1e-16 + target[..., 3:5] / anchors))
box_loss = self.mean_square_error(predictions[..., 1:5][obj], target[..., 1:5][obj])
# Class loss
class_loss = self.cross_entropy((predictions[..., 5:][obj]), (target[..., 5][obj].long()))
return (self.var_box * box_loss +
self.var_Obj * object_loss +
self.var_noObject * no_object_loss +
self.var_class * class_loss)
def forward(self, predictions, target, anchors):
obj = target[..., 0] == 1
noobj = target[..., 0] == 0
# No object loss
no_object_loss = self.bce(
(predictions[..., 0:1][noobj], (target[..., 0:1][noobj])),
)
# Object Loss
anchors = anchors.reshape(1, 3, 1, 1, 2) # p_w * exp(t_w)
box_preds = torch.cat([self.sigmoid(predictions[..., 1:3]), torch.exp(predictions[..., 3:5] * anchors)], dim=1)
ious = intersection_over_union(box_preds[obj], target[..., 1:5][obj]).detach()
object_loss = self.bce((predictions[..., 0:1][obj]), (ious * target[..., 0:1]))
# Box Coordinate Loss
predictions[..., 1:3] = self.sigmoid(predictions[..., 1:3]) # x, y to between [0,1]
target[..., 3:5] = torch.log(
(1e-6 + target[..., 3:5] / anchors)
)
box_loss = self.mse(predictions[..., 1:5][obj], target[..., 1:5][obj])
# Class Loss
class_loss = self.entropy(
(predictions[..., 5:][obj]), (target[..., 5][obj].long()),
)
return (
self.lambda_obj * box_loss
+ self.lambda_obj * object_loss
+ self.lambda_noobj * no_object_loss
+ self.lambda_class * class_loss
)
def get_performance(model, device_to_use, data_loader):
"""
:param model:
:param device_to_use:
:param data_loader:
:return:
"""
criterion = nn.BCEWithLogitsLoss().to(device_to_use)
detect_thres = 0.5
model.eval()
e_loss = 0
e_iou = 0
with torch.no_grad():
for iteration, (img, label) in enumerate(data_loader, 1):
img = img.to(device_to_use)
label = label.to(device_to_use)
label_out = model(img)
batch_loss = criterion(label_out, label.float())
e_loss += batch_loss.item()
preds = torch.sigmoid(label_out) > detect_thres
e_iou += utils.intersection_over_union(preds.float(), label.float()).cpu().detach().numpy()
e_loss = e_loss / len(data_loader)
e_iou = e_iou / len(data_loader)
return e_iou, e_loss
def forward(self, predictions, target, anchors):
# Check where obj and noobj (we ignore if target == -1)
obj = target[..., 0] == 1 # in paper this is Iobj_i
noobj = target[..., 0] == 0 # in paper this is Inoobj_i
# ======================= #
# FOR NO OBJECT LOSS #
# ======================= #
no_object_loss = self.bce(
(predictions[..., 0:1][noobj]),
(target[..., 0:1][noobj]),
)
# ==================== #
# FOR OBJECT LOSS #
# ==================== #
anchors = anchors.reshape(1, 3, 1, 1, 2)
box_preds = torch.cat([
self.sigmoid(predictions[..., 1:3]),
torch.exp(predictions[..., 3:5]) * anchors
],
dim=-1)
ious = intersection_over_union(box_preds[obj],
target[..., 1:5][obj]).detach()
object_loss = self.bce((predictions[..., 0:1][obj]),
(ious * target[..., 0:1][obj]))
# ======================== #
# FOR BOX COORDINATES #
# ======================== #
predictions[...,
1:3] = self.sigmoid(predictions[...,
1:3]) # x,y coordinates
target[..., 3:5] = torch.log(
(1e-16 + target[..., 3:5] / anchors)) # width, height coordinates
box_loss = self.mse(predictions[..., 1:5][obj], target[..., 1:5][obj])
# ================== #
# FOR CLASS LOSS #
# ================== #
class_loss = self.entropy(
(predictions[..., 5:][obj]),
(target[..., 5][obj].long()),
)
# print("__________________________________")
# print(self.lambda_box * box_loss)
# print(self.lambda_obj * object_loss)
# print(self.lambda_noobj * no_object_loss)
# print(self.lambda_class * class_loss)
# print("\n")
return (self.lambda_box * box_loss + self.lambda_obj * object_loss +
self.lambda_noobj * no_object_loss +
self.lambda_class * class_loss)
def process_image(model,
devise_to_use,
ch_mus,
ch_sigmas,
in_img,
in_img_label=None,
detect_thres=0.5):
"""
Pass image through model and get iou score of the prediction if in_img_label is not None
:param detect_thres:
:param in_img_label:
:param model:
:param devise_to_use:
:param in_img:
:param ch_mus:
:param ch_sigmas:
:return:
"""
# Zero all collected variables
global edge_extract_act
global cont_int_in_act
global cont_int_out_act
edge_extract_act = 0
cont_int_in_act = 0
cont_int_out_act = 0
normalize = transforms.Normalize(mean=ch_mus, std=ch_sigmas)
model_in_img = normalize(in_img)
model_in_img = model_in_img.to(devise_to_use).unsqueeze(0)
# Pass the image through the model
model.eval()
if isinstance(model, new_piech_models.JointPathfinderContourResnet50):
# Output is contour_dataset_out, pathfinder_out
label_out, _ = model(model_in_img)
else:
label_out = model(model_in_img)
iou = None
preds = None
if in_img_label is not None:
in_img_label = in_img_label.to(devise_to_use).unsqueeze(0)
preds = (torch.sigmoid(label_out) > detect_thres)
iou = utils.intersection_over_union(preds.float(),
in_img_label.float())
iou = iou.cpu().detach().numpy()
# Debug show predictions
# z = preds.float().squeeze()
# plt.figure()
# plt.imshow(z)
return iou, preds
def forward(self, obj, predictions, target, anchors):
anchors = anchors.reshape(1, 3, 1, 1, 2)
box_preds = torch.cat([
self.sigmoid(predictions[..., 1:3]),
torch.exp(predictions[..., 3:5]) * anchors
],
dim=-1)
ious = intersection_over_union(box_preds[obj],
target[..., 1:5][obj],
box_format="midpoints").detach()
return self.mse(self.sigmoid(predictions[..., 0:1][obj]),
ious * target[..., 0:1][obj])
def detection_target(self, proposals, gt_boxes, gt_cls_ids, gt_masks):
overlaps = utils.intersection_over_union(proposals, gt_boxes)
positive_count = self.NUM_TARGET * self.RATIO
positive_indices = tf.where(overlaps>=0.5)[0]
negative_indices = tf.where(overlaps<0.5)[0]
positive_indices = tf.random_shuffle(positive_indices)[:positive_count]
negative_count = self.NUM_TARGET - positive_count
negative_indices = tf.random_shuffle(negative_indices)[:negative_count]
positive_proposals = tf.gather(proposals, positive_indices)
positive_gt_boxes = tf.gather(gt_boxes, positive_indices)
positive_gt_deltas = utils.make_deltas(positive_proposals, positive_gt_boxes)
return
def test_iou_center_0(self):
# IF
boxes_predictions = torch.tensor([0.25, 0.25, 0.75, 0.75])
boxes_labels = torch.tensor([0.25, 0.25, 0.75, 0.75])
ious_expected = torch.tensor([1.0], )
# WHEN
ious_actual = intersection_over_union(boxes_predictions, boxes_labels,
"corners")
# THEN
self.assertAlmostEqual(ious_expected.item(),
ious_actual.item(),
places=5)
def test_iou_midpoints_2(self):
"""Checked the iou of two identical, overlapping squares with some offset"""
# IF
boxes_predictions = torch.tensor([0.5, 0.5, 0.5, 0.5])
boxes_labels = torch.tensor([0.25, 0.25, 0.5, 0.5])
ious_expected = torch.tensor([(1 / 16) / (7 / 16)])
# WHEN
ious_actual = intersection_over_union(boxes_predictions, boxes_labels,
"midpoints")
# THEN
self.assertAlmostEqual(ious_expected.item(),
ious_actual.item(),
places=5)
def test_iou_midpoints_0(self):
"""Checked the iou of two coinciding squares"""
# IF
boxes_predictions = torch.tensor([0.5, 0.5, 0.5, 0.5])
boxes_labels = torch.tensor([0.5, 0.5, 0.5, 0.5])
ious_expected = torch.tensor([1.0], )
# WHEN
ious_actual = intersection_over_union(boxes_predictions, boxes_labels,
"midpoints")
# THEN
self.assertAlmostEqual(ious_expected.item(),
ious_actual.item(),
places=5)
def forward(self, predictions, target):
predictions = predictions.reshape(-1, self.S, self.S,(self.B*5) + self.C)
iou_b1 = intersection_over_union(predictions[..., 21:25], target[..., 21:25])
iou_b2= intersection_over_union(predictions[..., 26:30], target[..., 21:25])
ious = torch.cat([iou_b1.unsqueeze(0), iou_b2.unsqueeze(0)], dim=0)
_ , best_box = torch.max(ious, dim=0)
exists_box = target[..., 20].unsqueeze(3) #Iobj_i
#Box loss
box_predictions = exists_box * (best_box * predictions[..., 26:30] + (1 - best_box) * predictions[..., 21:25])
box_targets = exists_box * target[..., 21:25]
box_predictions[..., 2:4] = torch.sign(box_predictions[...,2:4]) * torch.sqrt(torch.abs(box_predictions[...,2:4] + 1e-6))
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
#(N, S, S, 4) -> (N * S * S, 4)
box_loss = self.mse(torch.flatten(box_predictions, end_dim=-2), torch.flatten(box_targets, end_dim=-2))
#Object loss
pred_box = (best_box * predictions[..., 25:26] + (1 - best_box) * predictions[..., 20:21])
#(N * S * S)
object_loss = self.mse(torch.flatten(exists_box * pred_box), torch.flatten(exists_box * target[..., 20:21]))
#no object loss
#(N * S * S, 1) -> (N, S*S)
no_object_loss = self.mse(torch.flatten((1 - exists_box) * predictions[..., 20:21], start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1))
no_object_loss += self.mse(torch.flatten((1 - exists_box) * predictions[..., 25:26], start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1))
#class loss
#(N,S,S,20) -> (N*S*S,20)
class_loss = self.mse(torch.flatten(exists_box * predictions[..., :20], end_dim=-2),
torch.flatten(exists_box * target[..., :20], end_dim=-2))
#overall loss
loss = self.lambda_coord * box_loss + object_loss + self.lambda_noobj * no_object_loss + class_loss
return loss
def train():
""" Train for one Epoch over the train data set """
model.train()
e_loss = 0
e_iou = np.zeros_like(detect_thres)
for iteration, (img, label) in enumerate(train_data_loader, 1):
optimizer.zero_grad() # zero the parameter gradients
img = img.to(device)
label = label.to(device)
label_out = model(img)
bce_loss = criterion(label_out, label.float())
reg_loss = 0
if use_gaussian_reg_on_lateral_kernels:
reg_loss = \
inverse_gaussian_regularization(
model.contour_integration_layer.lateral_e.weight,
model.contour_integration_layer.lateral_i.weight
)
total_loss = bce_loss + lambda1 * reg_loss
# print("Loss: {:0.4f}, bce_loss {:0.4f}, lateral kernels reg_loss {:0.4f}".format(
# total_loss, bce_loss, lambda1 * reg_loss))
total_loss.backward()
optimizer.step()
e_loss += total_loss.item()
sigmoid_label_out = torch.sigmoid(label_out)
for th_idx, thresh in enumerate(detect_thres):
preds = sigmoid_label_out > thresh
e_iou[th_idx] += utils.intersection_over_union(
preds.float(), label.float()).cpu().detach().numpy()
e_loss = e_loss / len(train_data_loader)
e_iou = e_iou / len(train_data_loader)
# iou_arr = ["{:0.2f}".format(item) for item in e_iou]
# print("Train Epoch {} Loss = {:0.4f}, IoU={}".format(epoch, e_loss, iou_arr))
return e_loss, e_iou
def train_contour():
""" Train for one Epoch over the train data set """
model.train()
e_loss = 0
e_iou = 0
for iteration, (img, label) in enumerate(contour_train_data_loader, 1):
optimizer.zero_grad() # zero the parameter gradients
img = img.to(device)
label = label.to(device)
label_out, _ = model(img)
batch_loss = criterion(label_out, label.float())
kernel_loss = \
inverse_gaussian_regularization(
model.contour_integration_layer.lateral_e.weight,
model.contour_integration_layer.lateral_i.weight
)
total_loss = batch_loss + lambda1 * kernel_loss
# print("Total Loss: {:0.4f}, cross_entropy_loss {:0.4f}, kernel_loss {:0.4f}".format(
# total_loss, batch_loss, lambda1 * kernel_loss))
#
# import pdb
# pdb.set_trace()
total_loss.backward()
optimizer.step()
e_loss += total_loss.item()
preds = (torch.sigmoid(label_out) > detect_thres)
e_iou += utils.intersection_over_union(
preds.float(), label.float()).cpu().detach().numpy()
e_loss = e_loss / len(contour_train_data_loader)
e_iou = e_iou / len(contour_train_data_loader)
# print("Train Epoch {} Loss = {:0.4f}, IoU={:0.4f}".format(epoch, e_loss, e_iou))
return e_loss, e_iou
def YOLOloss(predictions, targets, anchors):
"""
calculate loss for one scale
:param predictions: pred shape for one scale (batch, scale, grid, grid, 3, 5 + num_classes)
:param targets: target shape for one scale (batch, scale, grid, grid, 3, 5 + 1)
:param anchors: scaled anchor size based on current scale
:return: box_loss + object_loss + noobject_loss + class_loss
"""
mse = nn.MSELoss()
bce = nn.BCEWithLogitsLoss()
obj = targets[..., 4] == 1
noobj = targets[..., 4] == 0
entropy = nn.CrossEntropyLoss()
# bounding box loss
# use sig(tx), sig(ty) to calculate x,y center loss
predictions[..., 0:2] = torch.sigmoid(predictions[..., 0:2])
# sig(tx), sig(ty), tw, th --> calculate box loss
box_loss = mse(predictions[..., 0:4][obj], targets[..., 0:4][obj])
# object loss
# absolute value --> calculate ious
pred_abs_boxes = rel_to_abs_box(predictions[..., 0:4], anchors)
target_abs_boxes = rel_to_abs_box(targets[..., 0:4], anchors)
ious = intersection_over_union(pred_abs_boxes[obj], target_abs_boxes[obj])
# ious = torch.flatten(ious)
# YOLOv3 predicts an objectness score for each bounding box using logistic regression, so here use sigmoid function
# and based on the iou bt prediction boxes and target boxes to calculate obj loss
object_loss = mse(torch.sigmoid(predictions[..., 4:5][obj]),
ious * targets[..., 4:5][obj])
# noobj loss
noobject_loss = bce(predictions[..., 4:5][noobj], targets[..., 4:5][noobj])
# class_loss = bce(predictions[..., 5:][obj], targets[..., 5:][obj])
class_loss = entropy(
(predictions[..., 5:][obj]),
(targets[..., 5][obj].long()),
)
# here the weights can be changed
return 5 * box_loss + 5 * object_loss + 0.5 * noobject_loss + 5 * class_loss
def detect_objects_response():
global models
model_names = request.args.get('models', "")
model_names = model_names.split(",") if model_names else []
response = {}
execution_mode = None
min_t_iou = 1.0
for model_name in model_names:
model_index = models[model_name]
objects_detected = []
try:
objects, execution_mode, t_iou = model_results[model_index].get(timeout=1)[1:]
for obj in objects:
objects_detected.append({'bbox': [obj.xmin, obj.ymin, obj.xmax - obj.xmin, obj.ymax - obj.ymin],
'class': obj.name, 'score': float(obj.confidence)})
if t_iou < min_t_iou:
min_t_iou = t_iou
except:
pass
response[model_name] = objects_detected
if execution_mode == 'ensemble':
# Ensemble Detection: Use non-maximum suppression to keep only those detected objects with high confidence
objects = list(sorted(list(itertools.chain.from_iterable(response.values())),
key=lambda obj: obj['score'], reverse=True))
skip_ids = []
for i in range(len(objects)):
for j in range(i + 1, len(objects)):
if intersection_over_union(objects[i], objects[j]) > min_t_iou:
skip_ids.append(j)
response = {'all': []}
for i, obj in enumerate(objects):
if i not in skip_ids:
response['all'].append(obj)
if model_names:
record_fps()
response["fps"] = get_fps_stats()
else:
response["fps"] = None
return jsonify(response), 201
def validate():
""" Get loss over validation set """
model.eval()
e_loss = 0
e_iou = np.zeros_like(detect_thres)
with torch.no_grad():
for iteration, (img, label) in enumerate(val_data_loader, 1):
img = img.to(device)
label = label.to(device)
label_out = model(img)
bce_loss = criterion(label_out, label.float())
reg_loss = 0
if use_gaussian_reg_on_lateral_kernels:
reg_loss = \
inverse_gaussian_regularization(
model.contour_integration_layer.lateral_e.weight,
model.contour_integration_layer.lateral_i.weight
)
total_loss = bce_loss + lambda1 * reg_loss
e_loss += total_loss.item()
sigmoid_label_out = torch.sigmoid(label_out)
for th_idx, thresh in enumerate(detect_thres):
preds = sigmoid_label_out > thresh
e_iou[th_idx] += utils.intersection_over_union(
preds.float(), label.float()).cpu().detach().numpy()
e_loss = e_loss / len(val_data_loader)
e_iou = e_iou / len(val_data_loader)
# print("Val Loss = {:0.4f}, IoU={}".format(e_loss, e_iou))
return e_loss, e_iou
def forward(self, predictions, target, device, epoch=0):
self.anchor_boxes = self.anchor_boxes.to(device)
exist_mask = target[..., 4:5]
existing_boxes = exist_mask * predictions
cell_idx = torch.arange(13, device=device)
bx = exist_mask * torch.sigmoid(predictions[
..., 0:1]) + exist_mask * cell_idx.view([1, 1, -1, 1, 1])
by = exist_mask * torch.sigmoid(predictions[
..., 1:2]) + exist_mask * cell_idx.view([1, -1, 1, 1, 1])
bw = (exist_mask * self.anchor_boxes[:, 2].view([1, 1, 1, -1, 1]) *
exist_mask * torch.exp(predictions[..., 2:3]))
bh = (exist_mask * self.anchor_boxes[:, 3].view([1, 1, 1, -1, 1]) *
exist_mask * torch.exp(predictions[..., 3:4]))
ious = intersection_over_union(torch.cat([bx, by, bw, bh], dim=-1),
target[..., :4])
xy_loss = self.mse(torch.cat([bx, by], dim=-1), target[..., :2])
bwbh = torch.cat([bw, bh], dim=-1)
wh_loss = self.mse(
torch.sqrt(torch.abs(bwbh) + 1e-32),
torch.sqrt(torch.abs(target[..., 2:4]) + 1e-32),
)
obj_loss = self.mse(
exist_mask,
exist_mask * ious * torch.sigmoid(existing_boxes[..., 4:5]))
# (ious.max(-1)[0]<0.6).int().unsqueeze(-1)
no_obj_loss = self.mse(
(1 - exist_mask),
(((1 - exist_mask) * (1 - torch.sigmoid(predictions[..., 4:5]))) *
((ious.max(-1)[0] < 0.6).int().unsqueeze(-1))),
)
class_loss = F.nll_loss(
(exist_mask *
F.log_softmax(predictions[..., 5:], dim=-1)).flatten(end_dim=-2),
target[..., 5:].flatten(end_dim=-2).argmax(-1),
)
return 5 * xy_loss + 5 * wh_loss + obj_loss + no_obj_loss + class_loss
def apply_non_max_suppression(single_img_predictions,
detection_threshold=0.5,
overlap_threshold=0.4):
boxes, scores, labels = single_img_predictions
# Filter out non-detections, using the classification probability
# Note that we want to keep the last dimension of boxes, which contains the bounding box cooordinates
score_mask = (scores > detection_threshold)
scores = scores[score_mask]
labels = labels[score_mask]
boxes = boxes[score_mask, :]
## Apply non-max suppression
for iA, (boxA, scoreA) in enumerate(zip(boxes, scores)):
if scoreA > 0.0:
try:
# Search forward for overlapping boxes
suppression_candidates = [(iA, scoreA)]
for iB, (boxB,
scoreB) in enumerate(zip(boxes[iA + 1:, :],
scores[iA + 1:]),
start=iA + 1):
if intersection_over_union(boxA, boxB) > overlap_threshold:
suppression_candidates.append((iB, scoreB))
# Non-max suppression by setting score to -1
if len(suppression_candidates) > 1:
suppression_candidates.sort(key=lambda x: x[-1])
for target, _ in suppression_candidates[:-1]:
scores[target] = -1.0
except IndexError:
# catch iA+1 when it goes out of bound
# Allegedly try-except blocks are ubiquitous for control-flow in Python
pass
return boxes, scores, labels
def forward(self, predictions, target):
# predictions are shaped (BATCH_SIZE, S*S(C+B*5) when inputted
predictions = predictions.reshape(-1, self.S, self.S, self.C + self.B * 5)
# Calculate IoU for the two predicted bounding boxes with target bbox
iou_b1 = intersection_over_union(predictions[..., 21:25], target[..., 21:25])
iou_b2 = intersection_over_union(predictions[..., 26:30], target[..., 21:25])
ious = torch.cat([iou_b1.unsqueeze(0), iou_b2.unsqueeze(0)], dim=0)
# Take the box with highest IoU out of the two prediction
# Note that bestbox will be indices of 0, 1 for which bbox was best
iou_maxes, bestbox = torch.max(ious, dim=0)
exists_box = target[..., 20].unsqueeze(3) # in paper this is Iobj_i
# ======================== #
# FOR BOX COORDINATES #
# ======================== #
# Set boxes with no object in them to 0. We only take out one of the two
# predictions, which is the one with highest Iou calculated previously.
box_predictions = exists_box * (
(
bestbox * predictions[..., 26:30]
+ (1 - bestbox) * predictions[..., 21:25]
)
)
box_targets = exists_box * target[..., 21:25]
# Take sqrt of width, height of boxes to ensure that
box_predictions[..., 2:4] = torch.sign(box_predictions[..., 2:4]) * torch.sqrt(
torch.abs(box_predictions[..., 2:4] + 1e-6)
)
box_targets[..., 2:4] = torch.sqrt(box_targets[..., 2:4])
box_loss = self.mse(
torch.flatten(box_predictions, end_dim=-2),
torch.flatten(box_targets, end_dim=-2),
)
# ==================== #
# FOR OBJECT LOSS #
# ==================== #
# pred_box is the confidence score for the bbox with highest IoU
pred_box = (
bestbox * predictions[..., 25:26] + (1 - bestbox) * predictions[..., 20:21]
)
object_loss = self.mse(
torch.flatten(exists_box * pred_box),
torch.flatten(exists_box * target[..., 20:21]),
)
# ======================= #
# FOR NO OBJECT LOSS #
# ======================= #
#max_no_obj = torch.max(predictions[..., 20:21], predictions[..., 25:26])
#no_object_loss = self.mse(
# torch.flatten((1 - exists_box) * max_no_obj, start_dim=1),
# torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1),
#)
no_object_loss = self.mse(
torch.flatten((1 - exists_box) * predictions[..., 20:21], start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1),
)
no_object_loss += self.mse(
torch.flatten((1 - exists_box) * predictions[..., 25:26], start_dim=1),
torch.flatten((1 - exists_box) * target[..., 20:21], start_dim=1)
)
# ================== #
# FOR CLASS LOSS #
# ================== #
class_loss = self.mse(
torch.flatten(exists_box * predictions[..., :20], end_dim=-2,),
torch.flatten(exists_box * target[..., :20], end_dim=-2,),
)
loss = (
self.lambda_coord * box_loss # first two rows in paper
+ object_loss # third row in paper
+ self.lambda_noobj * no_object_loss # forth row
+ class_loss # fifth row
)
return loss
model_results_dir = os.path.dirname(saved_model)
preds_dir = os.path.join(model_results_dir, 'predictions')
if not os.path.exists(preds_dir):
os.makedirs(preds_dir)
with torch.no_grad():
for iteration, (img, label) in enumerate(val_data_loader, 0):
img = img.to(device)
label = label.to(device)
label_out = net(img)
batch_loss = criterion(label_out, label.float())
preds = (torch.sigmoid(label_out) > detect_thres)
iou = utils.intersection_over_union(
preds.float(), label.float()).cpu().detach().numpy()
e_iou += iou
# Before visualizing Sigmoid the output. This is already done in the loss function
label_out = torch.sigmoid(label_out)
if save_predictions:
filename = list_of_files[iteration].split('/')[-1]
file_path = os.path.dirname(
list_of_files[iteration].split('labels')[-1])
store_path = preds_dir + file_path
if not os.path.exists(store_path):
os.makedirs(store_path)
