def forward(self, x):
"""
Arguments:
x : a tensor which is used as input for the model
Returns:
confidences and locations of predictions made by model during training
or
confidences and boxes of predictions made by model during testing
"""
confidences = []
locations = []
header_index = 0
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.conv13(x)
x = self.conv14(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_1(x)
# x=self.bottleneck_lstm2(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_2(x)
# x=self.bottleneck_lstm3(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_3(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_4(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
confidences = torch.cat(confidences, 1)
locations = torch.cat(locations, 1)
# Store result
self.confidences = confidences
self.locations = locations
if not self.training:
confidences = F.softmax(confidences, dim=2)
boxes = box_utils.convert_locations_to_boxes(
locations, self.priors, self.config.center_variance, self.config.size_variance
)
boxes = box_utils.center_form_to_corner_form(boxes)
return confidences, boxes
else:
return confidences, locations
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
confidences = []
locations = []
start_layer_index = 0
header_index = 0
for end_layer_index in self.source_layer_indexes:
if isinstance(end_layer_index, GraphPath):
path = end_layer_index
end_layer_index = end_layer_index.s0
added_layer = None
elif isinstance(end_layer_index, tuple):
added_layer = end_layer_index[1]
end_layer_index = end_layer_index[0]
path = None
else:
added_layer = None
path = None
for layer in self.base_net[start_layer_index: end_layer_index]:
x = layer(x)
if added_layer:
y = added_layer(x)
else:
y = x
if path:
sub = getattr(self.base_net[end_layer_index], path.name)
for layer in sub[:path.s1]:
x = layer(x)
y = x
for layer in sub[path.s1:]:
x = layer(x)
end_layer_index += 1
start_layer_index = end_layer_index
confidence, location = self.compute_header(header_index, y)
header_index += 1
confidences.append(confidence)
locations.append(location)
for layer in self.base_net[end_layer_index:]:
x = layer(x)
for layer in self.extras:
x = layer(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
confidences = torch.cat(confidences, 1)
locations = torch.cat(locations, 1)
if self.is_test:
confidences = F.softmax(confidences, dim=2)
boxes = box_utils.convert_locations_to_boxes(
locations, self.priors, self.config.center_variance, self.config.size_variance
)
boxes = box_utils.center_form_to_corner_form(boxes)
return confidences, boxes
else:
return confidences, locations
def __init__(self, center_form_priors, center_variance, size_variance,
iou_threshold):
self.center_form_priors = center_form_priors
self.corner_form_priors = box_utils.center_form_to_corner_form(
center_form_priors)
self.center_variance = center_variance
self.size_variance = size_variance
self.iou_threshold = iou_threshold
def _forward_test(self, cls_logits, bbox_pred):
if self.priors is None:
self.priors = PriorBox(self.cfg)().to(bbox_pred.device)
scores = F.softmax(cls_logits, dim=2)
#print(bbox_pred[0])
#print(self.priors[0])
boxes = box_utils.convert_locations_to_boxes(
bbox_pred, self.priors, self.cfg.MODEL.CENTER_VARIANCE,
self.cfg.MODEL.SIZE_VARIANCE)
boxes = box_utils.center_form_to_corner_form(boxes)
detections = (scores, boxes)
detections = self.post_processor(detections)
# for box in detections[0]['boxes']:
# print(box[2] - box[0])
# print(box[3] - box[1])
# print('-----')
return detections, {}
def forward(self, x):
x1 = self.Conv0_11(x)
loc1 = self.feature_map_loc_1(x1).permute((0, 2, 3, 1)).contiguous()
conf1 = self.feature_map_conf_1(x1).permute((0, 2, 3, 1)).contiguous()
x2 = self.Conv12_13(x1)
loc2 = self.feature_map_loc_2(x2).permute((0, 2, 3, 1)).contiguous()
conf2 = self.feature_map_conf_2(x2).permute((0, 2, 3, 1)).contiguous()
x3 = self.Conv14_1_2(x2)
loc3 = self.feature_map_loc_3(x3).permute((0, 2, 3, 1)).contiguous()
conf3 = self.feature_map_conf_3(x3).permute((0, 2, 3, 1)).contiguous()
x4 = self.Conv15_1_2(x3)
loc4 = self.feature_map_loc_4(x4).permute((0, 2, 3, 1)).contiguous()
conf4 = self.feature_map_conf_4(x4).permute((0, 2, 3, 1)).contiguous()
x5 = self.Conv16_1_2(x4)
loc5 = self.feature_map_loc_5(x5).permute((0, 2, 3, 1)).contiguous()
conf5 = self.feature_map_conf_5(x5).permute((0, 2, 3, 1)).contiguous()
x6 = self.Conv17_1_2(x5)
loc6 = self.feature_map_loc_6(x6).permute((0, 2, 3, 1)).contiguous()
conf6 = self.feature_map_conf_6(x6).permute((0, 2, 3, 1)).contiguous()
loc_list = [loc1, loc2, loc3, loc4, loc5, loc6]
conf_list = [conf1, conf2, conf3, conf4, conf5, conf6]
confidences = torch.cat([o.view(o.size(0), -1) for o in conf_list], 1)
locations = torch.cat([o.view(o.size(0), -1) for o in loc_list], 1)
confidences = confidences.view(confidences.size(0), -1,
cfg.MODEL.NUM_CLASSES)
locations = locations.view(locations.size(0), -1, 4)
if not self.training:
#when evaluating, decode the predictions
if self.priorBox is None:
self.priorBox = PriorBox(cfg)().to(locations.device)
confidences = F.softmax(confidences, dim=2)
boxes = convert_locations_to_boxes(locations, self.priorBox,
cfg.MODEL.CENTER_VARIANCE,
cfg.MODEL.SIZE_VARIANCE)
boxes = center_form_to_corner_form(boxes)
return confidences, boxes
else:
return confidences, locations
def forward(self, x):
"""
Arguments:
x : a tensor which is used as input for the model
Returns:
confidences and locations of predictions made by model during training
or
confidences and boxes of predictions made by model during testing
"""
confidences = []
locations = []
header_index = 0
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.conv13(x)
x = self.bottleneck_lstm1(x)
x = x.permute(0, 2, 3, 1)
x = self.fc1(x)
x = nn.Tanh()(x)
x = self.fc2(x)
x = x.permute(0, 3, 1, 2)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_1(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_2(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_3(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
x = self.fmaps_4(x)
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
confidences = torch.cat(confidences, 1)
locations = torch.cat(locations, 1)
if self.is_test: # while testing convert locations to boxes
confidences = F.softmax(confidences, dim=2)
boxes = box_utils.convert_locations_to_boxes(
locations,
self.priors,
self.config.center_variance,
self.config.size_variance,
)
boxes = box_utils.center_form_to_corner_form(boxes)
return confidences, boxes
else:
return confidences, locations
def training_step(self, batch, batch_nb):
if self.hparams.net == 'backbone':
# Compute loss
images, boxes, labels, _ = batch # gt
# Debug
# img_debug, _, _ = self.inverse_val_transform(images[0], None, None)
# img_debug = cv2.resize(img_debug, (160, 160))
# cv2.imshow("img_debug", img_debug)
# key = cv2.waitKey(1)
confidence, locations = self.forward(images)
regression_loss, classification_loss = self.loss_criterion(
confidence, locations, labels, boxes)
loss = regression_loss + classification_loss
# Metrics computation
self.accum_loss += loss.item()
self.train_index += 1
elif self.hparams.net == 'lstm':
# Detach hidden states from graph
self.detach_hidden()
# Compute loss
images, boxes, labels = batch # gt
loss = 0
crop_flag = False
for image, box, label in zip(images, boxes, labels):
image = image.to(self.device)
box = box.to(self.device)
label = label.to(self.device)
if (
random.rand() < self.hparams.crop_prob and crop_flag
): #use cropped image as training sample, also we skip first image crops to make sure that self.featuremap makes sense
faked_fmap = self.feature_map.clone(
) #create a copy of previous fmap in order to merge wit cropped fmap, we detach it from the graph because we dont want the gradients to go back this branch
n = image.shape[0]
for i in range(n):
#calculate crop
current_image = torch.unsqueeze(image[i, :, :, :], 0)
boxes_old = torch.squeeze(last_box[i, :, :])
labels_old = torch.squeeze(last_label[i, :])
boxes_old = box_utils.convert_locations_to_boxes(
torch.unsqueeze(boxes_old,
0), self.pred_dec.priors,
self.pred_dec.config.center_variance,
self.pred_dec.config.size_variance)
boxes_old = box_utils.center_form_to_corner_form(
boxes_old)
boxes_old = torch.squeeze(boxes_old)
boxes_old = boxes_old[labels_old > 0, :]
labels_old = labels_old[labels_old > 0]
boxes_old *= self.config.image_size #scale boxes to image size
if (boxes_old.shape[0] > 0):
#now we have to calculate a bbox that envolves all bboxes
x1 = boxes_old[:, 0].min(dim=0)[0].item()
y1 = boxes_old[:, 1].min(dim=0)[0].item()
x2 = boxes_old[:, 2].max(dim=0)[0].item()
y2 = boxes_old[:, 3].max(dim=0)[0].item()
# Last box is stored
square_side = max(x2 - x1, y2 - y1)
cx = x1 + (x2 - x1) / 2
cy = y1 + (y2 - y1) / 2
w = x2 - x1
h = y2 - y1
s = max(w, h) * (1.0 +
self.hparams.bbox_increase_factor)
focus_box = np.array([
cx - s / 2, cy - s / 2, cx + s / 2, cy + s / 2
])
# Saturate focus box
focus_box[0] = saturate_img_coordinate(
focus_box[0], self.config.image_size)
focus_box[1] = saturate_img_coordinate(
focus_box[1], self.config.image_size)
focus_box[2] = saturate_img_coordinate(
focus_box[2], self.config.image_size)
focus_box[3] = saturate_img_coordinate(
focus_box[3], self.config.image_size)
# Debug
# img_debug, _, _ = self.inverse_val_transform(image[i], None, None)
# cv2.rectangle(img_debug, (int(focus_box[0]), int(focus_box[1])),
# (int(focus_box[2]), int(focus_box[3])), (255, 255, 0), 2)
# cv2.imshow("img_debug", img_debug)
# key = cv2.waitKey(0)
bbox = (focus_box[0], focus_box[1],
focus_box[2] - focus_box[0],
focus_box[3] - focus_box[1]) # (x,y,w,h)
adjustment_dict = adjust_bbox(
bbox, 0, self.config.image_size,
self.config.image_size, self.pred_enc)
# Crop image
input_bbox = adjustment_dict['input_bbox']
if input_bbox[3] <= 16 or input_bbox[2] <= 16:
image_cropped = current_image
self.forward(
image_cropped, full_processing=False
) # we can access the fmap as model.feature_map, but its going to be cropped... beteter to acces later, when its padded
faked_fmap[i, :, :, :] = self.feature_map
else:
image_cropped = current_image[:, :,
int(input_bbox[1]
):
int(input_bbox[1]
+
input_bbox[3]
),
int(input_bbox[0]
):
int(input_bbox[0]
+
input_bbox[2]
)]
#print(image_cropped.shape)
self.forward(
image_cropped, full_processing=False
) # we can access the fmap as model.feature_map, but its going to be cropped... beteter to acces later, when its padded
current_faked_fmap = process_intermmediate_fmap(
torch.unsqueeze(faked_fmap[i, :, :, :], 0),
'prefaked_fmap_img_draw',
cropped_fmap=self.feature_map,
adjustment_dict=adjustment_dict,
plot_image=False)
faked_fmap[i, :, :, :] = torch.unsqueeze(
current_faked_fmap, 0)
else: #if boxes are empty, we have to processs the whole image
self.forward(
image_cropped, full_processing=False
) # we can access the fmap as model.feature_map, but its going to be cropped... beteter to acces later, when its padded
faked_fmap[i, :, :, :] = self.feature_map
confidence, locations = self.forward(
image, full_processing=False, inter_tensor=faked_fmap)
else: #train as usual
# Debug
# img_debug, _, _ = self.inverse_val_transform(image[0], None, None)
# cv2.imshow("img_debug", img_debug)
# key = cv2.waitKey(0)
crop_flag = True
confidence, locations = self.forward(image)
regression_loss, classification_loss = self.loss_criterion(
confidence, locations, label, box)
loss += regression_loss + classification_loss
last_box = box
last_label = label
# Metrics computation
self.accum_loss += loss.item()
self.train_index += 1
tensorboard_logs = {'train_loss': loss, 'epoch': self.current_epoch}
return {'loss': loss, 'log': tensorboard_logs}
def forward(self, x):
"""
Arguments:
x : a tensor which is used as input for the model
Returns:
confidences and locations of predictions made by model during training
or
confidences and boxes of predictions made by model during testing
"""
confidences = []
locations = []
header_index = 0
start_layer_index = 0
print("Basenet")
for end_layer_index in self.source_layer_indexes:
#Bepaal end_layer
if isinstance(end_layer_index, GraphPath):
path = end_layer_index
end_layer_index = end_layer_index.s0
added_layer = None
elif isinstance(end_layer_index, tuple):
added_layer = end_layer_index[1]
end_layer_index = end_layer_index[0]
path = None
else:
added_layer = None
path = None
#Doorloop basenet van start tot end layer
i = 0
for layer in self.base_net[start_layer_index:end_layer_index]:
x = layer(x)
i = i + 1
print('\t' + "Baselaag :", i)
#print("Base_net layer")
#print(x)
#print(layer)
if added_layer:
#print("Added layer")
y = added_layer(x)
#print(layer)
else:
y = x
#Doorloop end_layer indien er een path is
if path:
sub = getattr(self.base_net[end_layer_index], path.name)
for layer in sub[:path.s1]:
print('\t' + "Path - layer")
x = layer(x)
#print(x)
#print(layer)
y = x
for layer in sub[path.s1:]:
x = layer(x)
#print(layer)
#print(x)
#End_layer_index verhogen, start_layer_index vervangen
#print("headers inc")
end_layer_index += 1
start_layer_index = end_layer_index
#Berekenen header en inc header index
#bij gebruik van Bottleneck LSTMs, 1 header in basenet
if any(isinstance(l, BottleneckLSTM) for l in self.extras):
#na 14e laag, niet 19e
if end_layer_index <= 19:
confidence, location = self.compute_header(header_index, y)
print('\t' + "Layer computed header: Basenet", layer)
header_index += 1
confidences.append(confidence)
locations.append(location)
print('\t' + '\t' + "Confidences and locations appended.")
#bij normale mbv2-ssd, 2 headers in basenet
else:
confidence, location = self.compute_header(header_index, y)
print('\t' + "Layer computed header: Basenet", layer)
header_index += 1
confidences.append(confidence)
locations.append(location)
print('\t' + '\t' + "Confidences and locations appended.")
# Doorlopen lagen van basenet, die niet bij in SSD zitten voor predicties (headers)
for layer in self.base_net[end_layer_index:]:
x = layer(x)
print("Laatste basenet laag")
#print(x)
#print(layer)
#Bij het gebruik van bottlenecks
if any(isinstance(layer, BottleneckLSTM) for layer in self.extras):
print("Ja bottlenecks worden gebruikt.")
bottleneck_index = 0
extras_index = 0
for layer in self.extras:
x = layer(x)
#print(x)
#print(layer)
if isinstance(layer, BottleneckLSTM):
print('\t' + "Extra laag met bottleneck")
#berekenen headers na bottleneck
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
print('\t' + '\t' + "Confidences and locations appended.")
bottleneck_index += 1
print("Bottleneck index: ", bottleneck_index)
else:
print('\t' + "Extra laag zonder bottleneck")
extras_index += 1
if (extras_index > bottleneck_index):
confidence, location = self.compute_header(
header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
print('\t' + '\t' +
"Confidences and locations appended.")
extras_index += 1
print("extras_index index: ", extras_index)
continue
confidences = torch.cat(confidences, 1)
locations = torch.cat(locations, 1)
#Mobilenetv2 zonder bottlenecks
else:
print("Bottlenecks worden niet gebruikt in extras.")
for layer in self.extras:
x = layer(x)
print('\t' + "Bottlenecks worden niet gebruikt.")
confidence, location = self.compute_header(header_index, x)
header_index += 1
confidences.append(confidence)
locations.append(location)
print('\t' + '\t' + "Confidences and locations appended.")
#print("Layer computed header: Extras", layer)
confidences = torch.cat(confidences, 1)
locations = torch.cat(locations, 1)
if self.is_test:
confidences = F.softmax(confidences, dim=2)
boxes = box_utils.convert_locations_to_boxes(
locations, self.priors, self.config.center_variance,
self.config.size_variance)
boxes = box_utils.center_form_to_corner_form(boxes)
return confidences, boxes
else:
return confidences, locations
