def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == 3
assert len(top) == 1 or len(top) == 2
pred_detections = np.array(bottom[0].data)
gt_detections = np.array(bottom[1].data)
prior_boxes_data = np.array(bottom[2].data)
batch_gt = self._split_to_batch_gt(gt_detections, GT_RECORD_SIZE,
self._valid_action_ids)
prior_boxes =\
self._parse_prior_boxes(prior_boxes_data,
self._height, self._width, self._num_anchors,
PRIOR_BOXES_RECORD_SIZE)
batch_predictions =\
self._split_to_batch_prediction(pred_detections, PREDICTION_RECORD_SIZE, prior_boxes,
use_priors=self._match_priors)
matched_predictions = self._match_gt_to_predictions(
batch_gt,
batch_predictions,
min_gt_iou=self._min_gt_iou,
min_pr_iou=self._min_pr_iou,
ssd_matching=self._ssd_matching)
matches_blob = self._convert_predictions_to_blob(
matched_predictions, self._record_size)
out_shape = matches_blob.shape
if out_shape[2] == 0:
LOG('!!!No matched detections!!!')
top[0].reshape(out_shape[0], out_shape[1], 1, out_shape[3])
top[0].data[...] = -1.0
if len(top) == 2:
top[1].data[...] = 0.0
else:
top[0].reshape(out_shape[0], out_shape[1], out_shape[2],
out_shape[3])
top[0].data[...] = matches_blob
if len(top) == 2:
num_predictions = np.sum(
[len(l) for l in itervalues(batch_predictions)])
num_matches = np.sum(
[len(l) for l in itervalues(matched_predictions)])
top[1].data[...] = float(num_matches) / float(
num_predictions) if num_predictions > 0 else 0.0
except Exception:
LOG('MatcherLayer exception: {}'.format(traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == 1
assert len(top) == 1 or len(top) == 2
data = np.array(bottom[0].data, dtype=np.float32)
scale = self._get_scale(self._step, self._init_scale,
self._target_scale, self._num_steps,
self._power, self._use_last)
self._last_scale = scale
top[0].data[...] = scale * data
if len(top) == 2:
top[1].data[...] = scale
self._step += 1
except Exception:
LOG('ScheduledScaleLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
num_variables = len(bottom)
for i in range(num_variables):
bottom[i].diff[...] = 0.0
top_diff_value = top[0].diff[0]
for i, loss_scale, var_scale in self._samples:
if propagate_down[i]:
bottom[i].diff[
...] = self._weights[i] * loss_scale * top_diff_value
self.blobs[0].diff[i] += self._weights[i] * (
1.0 - var_scale) * top_diff_value
except Exception:
LOG('AdaptiveWeightingLossLayer backward pass exception: {}'.
format(traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[0]:
raise Exception(
'Cannot propagate down through the matched detections')
centers_data = np.array(bottom[1].data)
centers_diff_data = np.zeros(
bottom[1].data.shape) if propagate_down[1] else None
anchor_diff_data = {}
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 2]:
anchor_diff_data[anchor_id] = np.zeros(
bottom[anchor_id + 2].data.shape)
if len(self._pos_matches) > 0:
factor = top[0].diff[0] / float(self._num_instances)
for i, _ in enumerate(self._pos_matches):
det, center_id = self._pos_matches[i]
loss_weight = self._weights[i]
if propagate_down[det.anchor + 2]:
anchor_diff_data[det.anchor][det.item, :, det.y, det.x]\
+= factor * loss_weight * (centers_data[center_id] - centers_data[det.action])
if centers_diff_data is not None:
embedding = self._embeddings[det.anchor][det.item, :,
det.y, det.x]
centers_diff_data[
det.action] += -factor * loss_weight * embedding
centers_diff_data[
center_id] += factor * loss_weight * embedding
if centers_diff_data is not None:
bottom[1].diff[...] = centers_diff_data
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 2]:
bottom[anchor_id +
2].diff[...] = anchor_diff_data[anchor_id]
except Exception:
LOG('LocalPushLossLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[0]:
raise Exception(
'Cannot propagate down through the matched detections')
centers_data = np.array(bottom[1].data)
centers_diff_data = np.zeros(
bottom[1].data.shape) if propagate_down[1] else None
diff_data = {}
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 2]:
diff_data[anchor_id] = np.zeros(bottom[anchor_id +
2].data.shape)
if len(self._candidates) > 0:
factor = top[0].diff[0] / float(len(self._candidates))
for _, _, anchor_det, ref_det in self._candidates:
anchor_embed = self._embeddings[anchor_det.anchor][
anchor_det.item, :, anchor_det.y, anchor_det.x]
ref_embedding = self._embeddings[ref_det.anchor][
ref_det.item, :, ref_det.y, ref_det.x]
if propagate_down[anchor_det.anchor + 2]:
diff_data[anchor_det.anchor][anchor_det.item, :, anchor_det.y, anchor_det.x] \
+= factor * (ref_embedding - centers_data[anchor_det.action])
if propagate_down[ref_det.anchor + 2]:
diff_data[
ref_det.anchor][ref_det.item, :, ref_det.y,
ref_det.x] += factor * anchor_embed
if centers_diff_data is not None:
centers_diff_data[
anchor_det.action] += -factor * anchor_embed
if centers_diff_data is not None:
bottom[1].diff[...] = centers_diff_data
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 2]:
bottom[anchor_id + 2].diff[...] = diff_data[anchor_id]
except Exception:
LOG('SplitLossLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == 3
assert len(top) == 1
centers = np.array(bottom[0].data, dtype=np.float32)
assert len(centers.shape) == 2
embeddings = np.array(bottom[1].data, dtype=np.float32)
assert len(embeddings.shape) == 2
labels = np.array(bottom[2].data).astype(np.int32)
assert len(labels.shape) == 1
assert centers.shape[1] == embeddings.shape[1]
assert embeddings.shape[0] == labels.shape[0]
input_classes = np.unique(labels)
sum_losses = 0.0
num_instances = 0
valid_class_pairs = []
for class_id in input_classes:
if class_id < 0 or self._valid_class_ids is not None and class_id not in self._valid_class_ids:
continue
mask = labels == class_id
center = centers[class_id].reshape([-1, 1])
class_embeddings = embeddings[mask]
distances = 1.0 - np.matmul(class_embeddings, center)
sum_losses += np.sum(distances)
num_instances += int(np.sum(mask))
valid_class_pairs.append((class_id, mask))
top[0].data[...] = sum_losses / float(
num_instances) if num_instances > 0 else 0.0
self._num_instances = num_instances
self._valid_class_pairs = valid_class_pairs
except Exception:
LOG('PlainCenterLossLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def _print_stat(labels, class_queues, ignore_class_id):
"""Prints statistics of loaded data.
:param labels: List of loaded labels
:param class_queues: Lists of frames with specific class
:param ignore_class_id: ID of ignored class
"""
total_num = np.sum([labels[label] for label in labels if label != ignore_class_id])
if total_num == 0:
LOG('No labels')
return
factor = 100. / float(total_num)
LOG('Labels:')
for label in labels:
if label == ignore_class_id:
LOG(' {}: {:06} - ignored'.format(label, labels[label]))
else:
LOG(' {}: {:06} ({:.2f}%) - {:06} frames'
.format(label, labels[label], factor * float(labels[label]), len(class_queues[label])))
def setup(self, bottom, top):
"""Initializes layer.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
self._load_params(self.param_str)
self._start_prefetch()
except Exception:
LOG('DataLayer exception: {}'.format(traceback.format_exc()))
exit()
def setup(self, bottom, top):
"""Initializes layer.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
self._load_params(self.param_str, num_variables=len(bottom))
num_variables = len(bottom)
self._create_variables(num_variables, self._init)
except Exception:
LOG('AdaptiveWeightingLossLayer setup exception: {}'.format(
traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[0]:
bottom[0].diff[...] = self._last_scale * np.copy(top[0].diff)
except Exception:
LOG('ScheduledScaleLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(top) == 2
images_blob, labels_blob = self._sample_next_batch()
top[0].data[...] = images_blob
top[1].reshape(1, 1, labels_blob.shape[2], 8)
top[1].data[...] = labels_blob
except Exception:
LOG('DataLayer exception: {}'.format(traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[0]:
raise Exception(
'Cannot propagate down through the matched detections')
anchor_diff_data = {}
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 1]:
anchor_diff_data[anchor_id] = np.zeros(
bottom[anchor_id + 1].data.shape)
if len(self._samples) > 0:
diff_data = np.array(top[0].diff)
for out_sample_id, _ in enumerate(self._samples):
alpha, betta, det_i, det_j = self._samples[out_sample_id]
current_diff = diff_data[out_sample_id]
if propagate_down[det_i.anchor + 1]:
anchor_diff_data[det_i.anchor][det_i.item, :, det_i.y, det_i.x] \
+= alpha * current_diff
if propagate_down[det_j.anchor + 1]:
anchor_diff_data[det_j.anchor][det_j.item, :, det_j.y, det_j.x] \
+= betta * current_diff
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 1]:
bottom[anchor_id +
1].diff[...] = anchor_diff_data[anchor_id]
except Exception:
LOG('SamplingExtractorLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
num_variables = len(bottom)
assert num_variables > 0
assert len(top) == 1 or len(top) == 1 + num_variables
samples = []
losses = []
for i in range(num_variables):
loss_value = np.array(bottom[i].data,
dtype=np.float32).reshape([-1])
assert len(loss_value) == 1
loss_value = loss_value[0]
if loss_value > 0.0:
param_value = self.blobs[0].data[i]
loss_factor = np.exp(-param_value)
new_loss_value = param_value + self._scale * loss_factor * loss_value
samples.append((i, self._scale * loss_factor,
self._scale * loss_factor * loss_value))
losses.append(self._weights[i] * new_loss_value)
top[0].data[...] = np.sum(losses) if len(losses) > 0 else 0.0
if len(top) == 1 + num_variables:
for i in range(num_variables):
top[i + 1].data[...] = np.copy(bottom[i].data)
self._samples = samples
except Exception:
LOG('AdaptiveWeightingLossLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[2]:
raise Exception('Cannot propagate down through the labels')
centers = np.array(bottom[0].data, dtype=np.float32)
embeddings = np.array(bottom[1].data, dtype=np.float32)
centers_diff_data = np.zeros(
bottom[0].data.shape) if propagate_down[0] else None
embeddings_diff_data = np.zeros(
bottom[1].data.shape) if propagate_down[1] else None
factor = top[0].diff[0] / float(self._num_instances)
for class_id, mask in self._valid_class_pairs:
if propagate_down[0]:
centers_diff_data[class_id] += -factor * np.sum(
embeddings[mask], axis=0)
if propagate_down[1]:
embeddings_diff_data[mask] += -factor * centers[class_id]
if centers_diff_data is not None:
bottom[0].diff[...] = centers_diff_data
if embeddings_diff_data is not None:
bottom[1].diff[...] = embeddings_diff_data
except Exception:
LOG('PlainCenterLossLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 1
assert len(top) == 1
detections_data = np.array(bottom[0].data)
anchors_data = []
for i in range(self._num_anchors):
anchors_data.append(np.array(bottom[i + 1].data))
all_detections = self._parse_detections(detections_data,
INPUT_RECORD_SIZE,
self._translate_prediction)
all_actions = self._match_detections_with_actions(
all_detections, anchors_data)
if self._use_nms:
all_actions = self._complex_nms(all_actions, self._min_overlap,
self._num_valid_actions)
matches_blob = self._convert_actions_to_blob(
all_actions, OUTPUT_RECORD_SIZE)
out_shape = matches_blob.shape
top[0].reshape(out_shape[0], out_shape[1], out_shape[2],
out_shape[3])
top[0].data[...] = matches_blob
except Exception:
LOG('MatcherLayer exception: {}'.format(traceback.format_exc()))
exit()
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[0]:
raise Exception(
'Cannot propagate down through the matched detections')
diff_data = {}
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 1]:
diff_data[anchor_id] = np.zeros(bottom[anchor_id +
1].data.shape)
factor = top[0].diff[0] / float(len(self._candidates)) if len(
self._candidates) > 0 else 0.0
for _, _, anchor_det, ref_det in self._candidates:
if propagate_down[anchor_det.anchor + 1]:
diff_data[anchor_det.anchor][anchor_det.item, :, anchor_det.y, anchor_det.x] \
+= factor * self._embeddings[ref_det.anchor][ref_det.item, :, ref_det.y, ref_det.x]
if propagate_down[ref_det.anchor + 1]:
diff_data[ref_det.anchor][ref_det.item, :, ref_det.y, ref_det.x] \
+= factor * self._embeddings[anchor_det.anchor][anchor_det.item, :, anchor_det.y, anchor_det.x]
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 1]:
bottom[anchor_id + 1].diff[...] = diff_data[anchor_id]
except Exception:
LOG('PushLossLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 2
assert len(top) == 1 or len(top) == 2 or len(top) == 3
detections_data = np.array(bottom[0].data)
all_detections = self._read_detections(detections_data, MATCHED_RECORD_SIZE,
self._translate_matched_prediction,
self._valid_action_ids, self._min_conf)
self._centers = np.array(bottom[1].data)
self._embeddings = []
for i in range(self._num_anchors):
self._embeddings.append(np.array(bottom[i + 2].data))
if self._adaptive_weights:
total_num = 0
class_counts = {cl_id: 0 for cl_id in self._valid_action_ids}
for det in all_detections:
class_counts[det.action] += 1
total_num += 1
normalizer = 1.0 / float(total_num) if total_num > 0 else 0.0
class_frequencies = {cl_id: normalizer * class_counts[cl_id] for cl_id in self._valid_action_ids}
class_weights = self._estimate_weights(class_frequencies, self._smoothed_frequencies,
self._gamma, self._weight_limits)
else:
class_weights = self._class_weights
max_intra_class_dist = 0.0
losses = []
valid_detections = []
instance_counts = {}
for det in all_detections:
embedding = self._embeddings[det.anchor][det.item, :, det.y, det.x]
if self._use_filtering:
scale = self._get_scale(self._step, self._init_scale, self._target_scale,
self._num_steps, self._power, self._use_last)
scores = np.exp(scale * np.matmul(self._centers, embedding.reshape([-1, 1])))
distribution = scores / np.sum(scores)
regularization_value = -np.log(distribution[det.action]) +\
self._entropy_weight * np.sum(distribution * np.log(distribution))
if regularization_value < 0.0:
continue
center = self._centers[det.action]
dist = 1.0 - np.sum(embedding * center)
max_intra_class_dist = max(max_intra_class_dist, dist)
losses.append(dist)
valid_detections.append(det)
if det.item not in instance_counts.keys():
instance_counts[det.item] = {}
local_instance_counts = instance_counts[det.item]
if det.id not in local_instance_counts.keys():
local_instance_counts[det.id] = 0
local_instance_counts[det.id] += 1
if self._instance_norm:
instance_weights = [class_weights[det.action] / float(instance_counts[det.item][det.id])
for det in valid_detections]
num_instances = np.sum([len(counts) for counts in instance_counts.values()])
else:
instance_weights = [class_weights[det.action] for det in valid_detections]
num_instances = len(valid_detections)
weighted_sum_losses = np.sum([instance_weights[i] * losses[i] for i, _ in enumerate(valid_detections)])
top[0].data[...] = weighted_sum_losses / float(num_instances) if num_instances > 0 else 0.0
if len(top) > 1:
top[1].data[...] = max_intra_class_dist
if len(top) == 3:
top[2].data[...] =\
float(len(valid_detections)) / float(len(all_detections)) if len(all_detections) > 0 else 0.0
self._valid_detections = valid_detections
self._weights = instance_weights
self._num_instances = num_instances
self._step += 1
except Exception:
LOG('CenterLossLayer forward pass exception: {}'.format(traceback.format_exc()))
exit()
def __init__(self, task_path, ignore_occluded, class_names_map,
valid_class_ids, ignore_class_id, use_attribute):
"""Seeks valid images and according annotations files.
:param task_path: Path to file with tasks to load data
:param ignore_occluded: Whether to ignore occluded detections
:param class_names_map: Dictionary to map class names to ID
:param valid_class_ids: List of valid class IDs
:param ignore_class_id: ID of ignored class
"""
self._class_names_map = class_names_map
self._valid_class_ids = valid_class_ids
self._ignore_class_id = ignore_class_id
tasks = self._parse_tasks(task_path)
LOG('Found {} tasks:'.format(len(tasks)))
for task_id, task in enumerate(tasks):
LOG(' {}: {}'.format(task_id, task[0]))
total_frames = 0
total_objects = 0
self._all_frames = []
self._class_queues = {
i: []
for i in self._valid_class_ids if i != self._ignore_class_id
}
glob_class_counts = {
i: 0
for i in self._valid_class_ids if i != self._ignore_class_id
}
for task_id, task in enumerate(tasks):
LOG('Loading task {}...'.format(task[0]))
annotation_path = task[0]
images_dir = task[1]
video_resolution = task[2]
image_paths = self._parse_images(images_dir)
if len(image_paths) == 0:
continue
id_shift = (task_id + 1) * IDS_SHIFT_SCALE
annotation = self._read_annotation(annotation_path,
video_resolution,
ignore_occluded,
self._class_names_map,
self._valid_class_ids, id_shift,
use_attribute)
for frame_id in annotation:
if frame_id not in image_paths:
continue
image_path = image_paths[frame_id]
gt_objects = annotation[frame_id]
if len(gt_objects) == 0:
continue
# Skip images without annotated objectes
if not sum([
gt_o.class_id != self._ignore_class_id
for gt_o in gt_objects
]):
continue
self._all_frames.append(
FrameDesc(path=image_path, objects=gt_objects))
frame_glob_id = len(self._all_frames) - 1
local_class_counts = {
i: 0
for i in self._valid_class_ids
if i != self._ignore_class_id
}
for gt_object in gt_objects:
class_id = gt_object.class_id
if class_id != self._ignore_class_id:
local_class_counts[class_id] += 1
glob_class_counts[class_id] += 1
for class_id in local_class_counts:
if local_class_counts[class_id] > 0:
self._class_queues[class_id].append(frame_glob_id)
total_frames += 1
total_objects += len(gt_objects)
LOG('DataLayer stats: loaded {} frames with {} objects.'.format(
total_frames, total_objects))
self._print_stat(glob_class_counts, self._class_queues,
self._ignore_class_id)
for class_id in self._class_queues:
if class_id == ignore_class_id:
continue
if len(self._class_queues[class_id]) == 0:
raise Exception(
'Cannot find frames with {} class id'.format(class_id))
def backward(self, top, propagate_down, bottom):
"""Carry out backward pass.
:param top: List of top blobs
:param propagate_down: List of indicators to carry out back-propagation for
the specified bottom blob
:param bottom: List of bottom blobs
"""
try:
if propagate_down[0]:
raise Exception(
'Cannot propagate down through the matched detections')
centers_data = np.array(bottom[1].data)
centers_diff_data = np.zeros(
bottom[1].data.shape) if propagate_down[1] else None
anchor_diff_data = {}
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 2]:
anchor_diff_data[anchor_id] = np.zeros(
bottom[anchor_id + 2].data.shape)
if self._valid_num_pairs > 0:
factor = top[0].diff[0] / float(self._valid_num_pairs)
for item_id, _ in enumerate(self._masks):
anchor_masks = self._masks[item_id]
for anchor_id, _ in enumerate(anchor_masks):
embeddings = self._embeddings[anchor_id][item_id]
diff_data = anchor_diff_data[anchor_id][item_id]
embedding_size = embeddings.shape[0]
center_masks = anchor_masks[anchor_id]
for center_id, _ in enumerate(center_masks):
mask = center_masks[center_id]
num_pairs = int(np.sum(mask))
mask = np.tile(np.expand_dims(mask, axis=0),
reps=[embedding_size, 1, 1])
if centers_diff_data is not None:
filtered_embeddings = embeddings[mask].reshape(
[embedding_size, -1])
centers_diff_data[
center_id] += factor * np.sum(
filtered_embeddings, axis=1)
if propagate_down[anchor_id + 2]:
diff_data[mask] += factor * np.tile(
centers_data[center_id].reshape([-1, 1]),
reps=[1, num_pairs]).reshape([-1])
if centers_diff_data is not None:
bottom[1].diff[...] = centers_diff_data
for anchor_id in range(self._num_anchors):
if propagate_down[anchor_id + 2]:
bottom[anchor_id +
2].diff[...] = anchor_diff_data[anchor_id]
except Exception:
LOG('GlobPushLossLayer backward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 2
assert len(top) == 1 or len(top) == 3
detections_data = np.array(bottom[0].data)
batch_detections = self._read_detections(
detections_data, MATCHED_RECORD_SIZE,
self._translate_matched_prediction, self._valid_action_ids,
self._min_conf)
centers_data = np.array(bottom[1].data)
num_centers = centers_data.shape[0]
self._embeddings = []
for i in range(self._num_anchors):
self._embeddings.append(np.array(bottom[i + 2].data))
height, width = self._embeddings[0].shape[2:]
all_masks = []
out_loss = 0.0
total_num_pairs = 0
valid_num_pairs = 0
dist_sum = 0.0
for item_id in batch_detections.keys():
detections = batch_detections[item_id]
outer_mask = self._outer_class_mask(detections, height, width)
total_num_pairs += self._num_anchors * num_centers * int(
np.sum(outer_mask))
anchor_masks = []
for anchor_id in range(self._num_anchors):
anchor_embeddings = self._embeddings[anchor_id][item_id]
center_masks = []
for center_id in range(num_centers):
center_embedding = centers_data[center_id].reshape(
[-1, 1, 1])
distances = 1.0 - np.sum(
anchor_embeddings * center_embedding, axis=0)
losses = self._margin - distances
out_mask = (losses > 0.0) * outer_mask
valid_num_pairs += int(np.sum(out_mask))
out_loss += np.sum(losses[out_mask])
dist_sum += np.sum(distances[out_mask])
center_masks.append(out_mask)
anchor_masks.append(center_masks)
all_masks.append(anchor_masks)
top[0].data[...] = out_loss / float(
valid_num_pairs) if valid_num_pairs > 0 else 0.0
if len(top) == 3:
top[1].data[...] = float(valid_num_pairs) / float(
total_num_pairs) if total_num_pairs > 0 else 0.0
top[2].data[...] = dist_sum / float(
valid_num_pairs) if valid_num_pairs > 0 else 0.0
self._valid_num_pairs = valid_num_pairs
self._masks = all_masks
except Exception:
LOG('GlobPushLossLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 1
assert len(top) == 1 or len(top) == 3
detections_data = np.array(bottom[0].data)
all_detections = self._read_detections(
detections_data, MATCHED_RECORD_SIZE,
self._translate_matched_prediction, self._valid_action_ids,
self._min_conf)
detections = self._filter_detections(all_detections,
self._max_num_samples)
classes = list(detections)
class_pairs = [(classes[i], classes[j])
for i, _ in enumerate(classes)
for j in range(i + 1, len(classes))]
self._embeddings = []
for i in range(self._num_anchors):
self._embeddings.append(np.array(bottom[i + 1].data))
all_candidates = []
total_num_pairs = 0
for class_i, class_j in class_pairs:
detections_i = detections[class_i]
detections_j = detections[class_j]
if len(detections_i) == 0 or len(detections_j) == 0:
continue
for i, _ in enumerate(detections_i):
anchor_det = detections_i[i]
anchor_embed = self._embeddings[anchor_det.anchor][
anchor_det.item, :, anchor_det.y, anchor_det.x]
for j, _ in enumerate(detections_j):
ref_det = detections_j[j]
ref_embed = self._embeddings[ref_det.anchor][
ref_det.item, :, ref_det.y, ref_det.x]
embed_dist = 1.0 - np.sum(anchor_embed * ref_embed)
loss = self._margin - embed_dist
if loss > 0.0:
all_candidates.append(
(loss, embed_dist, anchor_det, ref_det))
total_num_pairs += 1
if len(all_candidates) == 0:
self._candidates = []
top[0].data[...] = 0.0
if len(top) == 3:
top[1].data[...] = 0.0
top[2].data[...] = 0.0
else:
if len(all_candidates) > 2:
threshold_value = np.median(
[tup[0] for tup in all_candidates])
self._candidates = [
tup for tup in all_candidates
if tup[0] > threshold_value
]
else:
self._candidates = all_candidates
loss = np.sum([tup[0] for tup in self._candidates]) / float(
len(self._candidates))
top[0].data[...] = loss
if len(top) == 3:
top[1].data[...] = np.median(
[tup[1] for tup in self._candidates])
top[2].data[...] = float(
len(all_candidates)) / float(total_num_pairs)
except Exception:
LOG('PushLossLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 2
assert len(top) == 1 or len(top) == 4
detections_data = np.array(bottom[0].data)
batch_detections = self._read_detections(
detections_data, MATCHED_RECORD_SIZE,
self._translate_matched_prediction, self._valid_action_ids,
self._min_conf)
centers = np.array(bottom[1].data)
self._embeddings = []
for i in range(self._num_anchors):
self._embeddings.append(np.array(bottom[i + 2].data))
all_candidates = []
total_num_pairs = 0
total_num_overlapped = 0
total_num_incorrect = 0
for item_id in batch_detections.keys():
detections = batch_detections[item_id]
for i, _ in enumerate(detections):
anchor_det = detections[i]
for j in range(i + 1, len(detections)):
ref_det = detections[j]
# exclude same class predictions
if anchor_det.action == ref_det.action:
continue
overlap = self._iou(anchor_det, ref_det)
if overlap < self._min_overlap:
continue
total_num_overlapped += 1
anchor_embed = self._embeddings[anchor_det.anchor][
anchor_det.item, :, anchor_det.y, anchor_det.x]
ref_embed = self._embeddings[ref_det.anchor][
ref_det.item, :, ref_det.y, ref_det.x]
anchor_center_distances =\
(1.0 - np.matmul(centers, anchor_embed.reshape([-1, 1]))).reshape([-1])
ref_center_distances =\
(1.0 - np.matmul(centers, ref_embed.reshape([-1, 1]))).reshape([-1])
anchor_action = np.argmin(anchor_center_distances)
ref_action = np.argmin(ref_center_distances)
# exclude well-separated predictions
if anchor_action != ref_action:
continue
# exclude predictions with both incorrect labels
if anchor_action != anchor_det.action and ref_action != ref_det.action:
continue
total_num_incorrect += 1
embed_dist = 1.0 - np.sum(anchor_embed * ref_embed)
if anchor_action != anchor_det.action:
loss = self._margin + anchor_center_distances[
anchor_det.action] - embed_dist
if loss > 0.0:
all_candidates.append(
(loss, embed_dist, anchor_det, ref_det))
total_num_pairs += 1
if ref_action != ref_det.action:
loss = self._margin + ref_center_distances[
ref_det.action] - embed_dist
if loss > 0.0:
all_candidates.append(
(loss, embed_dist, ref_det, anchor_det))
total_num_pairs += 1
if len(all_candidates) == 0:
self._candidates = []
top[0].data[...] = 0.0
if len(top) == 4:
top[1].data[...] = 0.0
top[2].data[...] = 0.0
top[3].data[...] = 0.0
else:
self._candidates = all_candidates
loss = np.sum([tup[0] for tup in self._candidates]) / float(
len(self._candidates))
top[0].data[...] = loss
if len(top) == 4:
top[1].data[...] = np.median(
[tup[1] for tup in self._candidates])
top[2].data[...] = float(len(
self._candidates)) / float(total_num_pairs)
top[3].data[...] = float(total_num_incorrect) / float(
total_num_overlapped)
except Exception:
LOG('SplitLossLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 1
assert len(top) == 2
embedding_size = bottom[1].data.shape[1]
detections_data = np.array(bottom[0].data)
all_detections = self._read_detections(
detections_data, MATCHED_RECORD_SIZE,
self._translate_matched_prediction, self._valid_action_ids,
self._min_conf)
self._embeddings = []
for i in range(self._num_anchors):
self._embeddings.append(np.array(bottom[i + 1].data))
valid_class_ids = list(all_detections)
valid_class_ids.sort()
self._samples = []
sampled_vectors = []
sampled_labels = []
for class_id in valid_class_ids:
detections = all_detections[class_id]
if len(detections) < 2:
continue
embeddings = np.array([
self._embeddings[det.anchor][det.item, :, det.y, det.x]
for det in detections
])
center = np.mean(embeddings, axis=0, keepdims=True)
center = center / np.sqrt(
np.sum(np.square(center), axis=1, keepdims=True))
distances = (1.0 - np.matmul(embeddings, center.T)).reshape(
[-1])
threshold = np.percentile(distances, 100.0 * self._quantile)\
if self._threshold is None else self._threshold
valid_ids = np.arange(len(distances),
dtype=np.int32)[distances > threshold]
if len(valid_ids) < 2:
continue
sample_ids1 = []
sample_ids2 = []
for _ in range(self._num_steps):
ids_pair = np.random.choice(valid_ids, 2, replace=False)
sample_ids1.append(ids_pair[0])
sample_ids2.append(ids_pair[1])
dist_ratio = np.random.uniform(0.0, 1.0, [self._num_steps])
alpha, betta = self._calc_scales_to_mix_embeddings(
dist_ratio, embeddings[sample_ids1],
embeddings[sample_ids2])
alpha = alpha.reshape([self._num_steps, 1])
betta = betta.reshape([self._num_steps, 1])
sample = alpha * embeddings[sample_ids1] + betta * embeddings[
sample_ids2]
sampled_vectors.append(sample)
sampled_labels.append(
np.full([self._num_steps],
float(class_id),
dtype=np.float32))
self._samples += [
(alpha[i], betta[i], detections[sample_ids1[i]],
detections[sample_ids2[i]])
for i, _ in enumerate(sample_ids1)
]
assert len(self._samples) == len(sampled_vectors) * self._num_steps
if len(self._samples) > 0:
top[0].reshape(len(self._samples), embedding_size)
top[1].reshape(len(self._samples))
top[0].data[...] = np.concatenate(tuple(sampled_vectors),
axis=0)
top[1].data[...] = np.concatenate(tuple(sampled_labels),
axis=0)
else:
LOG('No samples generated!')
top[0].reshape(1, embedding_size)
top[0].data[...] = 0.0
top[1].reshape(1)
top[1].data[...] = -1.0
except Exception:
LOG('SamplingExtractorLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
def forward(self, bottom, top):
"""Carry out forward pass.
:param bottom: List of bottom blobs
:param top: List of top blobs
"""
try:
assert len(bottom) == self._num_anchors + 2
assert len(top) == 1 or len(top) == 4
detections_data = np.array(bottom[0].data)
all_detections = self._read_detections(
detections_data, MATCHED_RECORD_SIZE,
self._translate_matched_prediction, self._valid_action_ids,
self._min_conf)
centers_data = np.array(bottom[1].data)
num_centers = centers_data.shape[0]
self._embeddings = []
for i in range(self._num_anchors):
self._embeddings.append(np.array(bottom[i + 2].data))
if self._adaptive_weights:
total_num = 0
class_counts = {cl_id: 0 for cl_id in self._valid_action_ids}
for det in all_detections:
class_counts[det.action] += 1
total_num += 1
normalizer = 1.0 / float(total_num) if total_num > 0 else 0.0
class_frequencies = {
cl_id: normalizer * class_counts[cl_id]
for cl_id in self._valid_action_ids
}
class_weights = self._estimate_weights(
class_frequencies, self._smoothed_frequencies, self._gamma,
self._weight_limits)
else:
class_weights = self._class_weights
pos_matches = []
total_num_matches = 0
sum_pos_distances = 0.0
sum_neg_distances = 0.0
losses = []
instance_counts = {}
instance_weights = []
for det in all_detections:
det_embedding = self._embeddings[det.anchor][det.item, :,
det.y, det.x]
center_embedding = centers_data[det.action]
pos_distance = 1.0 - np.sum(det_embedding * center_embedding)
for center_id in range(num_centers):
if center_id == det.action:
continue
neg_distance = 1.0 - np.sum(
det_embedding * centers_data[center_id])
loss = self._margin + pos_distance - neg_distance
if loss > 0.0:
losses.append(loss)
sum_pos_distances += pos_distance
sum_neg_distances += neg_distance
pos_matches.append((det, center_id))
instance_weights.append(class_weights[det.action])
if det.item not in instance_counts:
instance_counts[det.item] = {}
local_instance_counts = instance_counts[det.item]
if det.id not in local_instance_counts:
local_instance_counts[det.id] = 0
local_instance_counts[det.id] += 1
total_num_matches += 1
if self._instance_norm:
instance_weights = \
[instance_weights[i] / float(instance_counts[pos_matches[i][0].item][pos_matches[i][0].id])
for i in range(len(pos_matches))]
num_instances = np.sum(
[len(counts) for counts in itervalues(instance_counts)])
else:
instance_weights = [
instance_weights[i] for i, _ in enumerate(pos_matches)
]
num_instances = len(pos_matches)
weighted_sum_losses = np.sum([
instance_weights[i] * losses[i]
for i, _ in enumerate(pos_matches)
])
top[0].data[...] = weighted_sum_losses / float(
num_instances) if num_instances > 0 else 0.0
if len(top) == 4:
top[1].data[...] = float(len(pos_matches)) / float(
total_num_matches) if total_num_matches > 0 else 0.0
top[2].data[...] = float(sum_pos_distances) / float(
len(pos_matches)) if len(pos_matches) > 0 else 0.0
top[3].data[...] = float(sum_neg_distances) / float(
len(pos_matches)) if len(pos_matches) > 0 else 0.0
self._pos_matches = pos_matches
self._weights = instance_weights
self._num_instances = num_instances
except Exception:
LOG('LocalPushLossLayer forward pass exception: {}'.format(
traceback.format_exc()))
exit()
