def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == 3 assert len(top) == 1 or len(top) == 2 pred_detections = np.array(bottom[0].data) gt_detections = np.array(bottom[1].data) prior_boxes_data = np.array(bottom[2].data) batch_gt = self._split_to_batch_gt(gt_detections, GT_RECORD_SIZE, self._valid_action_ids) prior_boxes =\ self._parse_prior_boxes(prior_boxes_data, self._height, self._width, self._num_anchors, PRIOR_BOXES_RECORD_SIZE) batch_predictions =\ self._split_to_batch_prediction(pred_detections, PREDICTION_RECORD_SIZE, prior_boxes, use_priors=self._match_priors) matched_predictions = self._match_gt_to_predictions( batch_gt, batch_predictions, min_gt_iou=self._min_gt_iou, min_pr_iou=self._min_pr_iou, ssd_matching=self._ssd_matching) matches_blob = self._convert_predictions_to_blob( matched_predictions, self._record_size) out_shape = matches_blob.shape if out_shape[2] == 0: LOG('!!!No matched detections!!!') top[0].reshape(out_shape[0], out_shape[1], 1, out_shape[3]) top[0].data[...] = -1.0 if len(top) == 2: top[1].data[...] = 0.0 else: top[0].reshape(out_shape[0], out_shape[1], out_shape[2], out_shape[3]) top[0].data[...] = matches_blob if len(top) == 2: num_predictions = np.sum( [len(l) for l in itervalues(batch_predictions)]) num_matches = np.sum( [len(l) for l in itervalues(matched_predictions)]) top[1].data[...] = float(num_matches) / float( num_predictions) if num_predictions > 0 else 0.0 except Exception: LOG('MatcherLayer exception: {}'.format(traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == 1 assert len(top) == 1 or len(top) == 2 data = np.array(bottom[0].data, dtype=np.float32) scale = self._get_scale(self._step, self._init_scale, self._target_scale, self._num_steps, self._power, self._use_last) self._last_scale = scale top[0].data[...] = scale * data if len(top) == 2: top[1].data[...] = scale self._step += 1 except Exception: LOG('ScheduledScaleLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: num_variables = len(bottom) for i in range(num_variables): bottom[i].diff[...] = 0.0 top_diff_value = top[0].diff[0] for i, loss_scale, var_scale in self._samples: if propagate_down[i]: bottom[i].diff[ ...] = self._weights[i] * loss_scale * top_diff_value self.blobs[0].diff[i] += self._weights[i] * ( 1.0 - var_scale) * top_diff_value except Exception: LOG('AdaptiveWeightingLossLayer backward pass exception: {}'. format(traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[0]: raise Exception( 'Cannot propagate down through the matched detections') centers_data = np.array(bottom[1].data) centers_diff_data = np.zeros( bottom[1].data.shape) if propagate_down[1] else None anchor_diff_data = {} for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 2]: anchor_diff_data[anchor_id] = np.zeros( bottom[anchor_id + 2].data.shape) if len(self._pos_matches) > 0: factor = top[0].diff[0] / float(self._num_instances) for i, _ in enumerate(self._pos_matches): det, center_id = self._pos_matches[i] loss_weight = self._weights[i] if propagate_down[det.anchor + 2]: anchor_diff_data[det.anchor][det.item, :, det.y, det.x]\ += factor * loss_weight * (centers_data[center_id] - centers_data[det.action]) if centers_diff_data is not None: embedding = self._embeddings[det.anchor][det.item, :, det.y, det.x] centers_diff_data[ det.action] += -factor * loss_weight * embedding centers_diff_data[ center_id] += factor * loss_weight * embedding if centers_diff_data is not None: bottom[1].diff[...] = centers_diff_data for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 2]: bottom[anchor_id + 2].diff[...] = anchor_diff_data[anchor_id] except Exception: LOG('LocalPushLossLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[0]: raise Exception( 'Cannot propagate down through the matched detections') centers_data = np.array(bottom[1].data) centers_diff_data = np.zeros( bottom[1].data.shape) if propagate_down[1] else None diff_data = {} for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 2]: diff_data[anchor_id] = np.zeros(bottom[anchor_id + 2].data.shape) if len(self._candidates) > 0: factor = top[0].diff[0] / float(len(self._candidates)) for _, _, anchor_det, ref_det in self._candidates: anchor_embed = self._embeddings[anchor_det.anchor][ anchor_det.item, :, anchor_det.y, anchor_det.x] ref_embedding = self._embeddings[ref_det.anchor][ ref_det.item, :, ref_det.y, ref_det.x] if propagate_down[anchor_det.anchor + 2]: diff_data[anchor_det.anchor][anchor_det.item, :, anchor_det.y, anchor_det.x] \ += factor * (ref_embedding - centers_data[anchor_det.action]) if propagate_down[ref_det.anchor + 2]: diff_data[ ref_det.anchor][ref_det.item, :, ref_det.y, ref_det.x] += factor * anchor_embed if centers_diff_data is not None: centers_diff_data[ anchor_det.action] += -factor * anchor_embed if centers_diff_data is not None: bottom[1].diff[...] = centers_diff_data for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 2]: bottom[anchor_id + 2].diff[...] = diff_data[anchor_id] except Exception: LOG('SplitLossLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == 3 assert len(top) == 1 centers = np.array(bottom[0].data, dtype=np.float32) assert len(centers.shape) == 2 embeddings = np.array(bottom[1].data, dtype=np.float32) assert len(embeddings.shape) == 2 labels = np.array(bottom[2].data).astype(np.int32) assert len(labels.shape) == 1 assert centers.shape[1] == embeddings.shape[1] assert embeddings.shape[0] == labels.shape[0] input_classes = np.unique(labels) sum_losses = 0.0 num_instances = 0 valid_class_pairs = [] for class_id in input_classes: if class_id < 0 or self._valid_class_ids is not None and class_id not in self._valid_class_ids: continue mask = labels == class_id center = centers[class_id].reshape([-1, 1]) class_embeddings = embeddings[mask] distances = 1.0 - np.matmul(class_embeddings, center) sum_losses += np.sum(distances) num_instances += int(np.sum(mask)) valid_class_pairs.append((class_id, mask)) top[0].data[...] = sum_losses / float( num_instances) if num_instances > 0 else 0.0 self._num_instances = num_instances self._valid_class_pairs = valid_class_pairs except Exception: LOG('PlainCenterLossLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def _print_stat(labels, class_queues, ignore_class_id): """Prints statistics of loaded data. :param labels: List of loaded labels :param class_queues: Lists of frames with specific class :param ignore_class_id: ID of ignored class """ total_num = np.sum([labels[label] for label in labels if label != ignore_class_id]) if total_num == 0: LOG('No labels') return factor = 100. / float(total_num) LOG('Labels:') for label in labels: if label == ignore_class_id: LOG(' {}: {:06} - ignored'.format(label, labels[label])) else: LOG(' {}: {:06} ({:.2f}%) - {:06} frames' .format(label, labels[label], factor * float(labels[label]), len(class_queues[label])))
def setup(self, bottom, top): """Initializes layer. :param bottom: List of bottom blobs :param top: List of top blobs """ try: self._load_params(self.param_str) self._start_prefetch() except Exception: LOG('DataLayer exception: {}'.format(traceback.format_exc())) exit()
def setup(self, bottom, top): """Initializes layer. :param bottom: List of bottom blobs :param top: List of top blobs """ try: self._load_params(self.param_str, num_variables=len(bottom)) num_variables = len(bottom) self._create_variables(num_variables, self._init) except Exception: LOG('AdaptiveWeightingLossLayer setup exception: {}'.format( traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[0]: bottom[0].diff[...] = self._last_scale * np.copy(top[0].diff) except Exception: LOG('ScheduledScaleLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(top) == 2 images_blob, labels_blob = self._sample_next_batch() top[0].data[...] = images_blob top[1].reshape(1, 1, labels_blob.shape[2], 8) top[1].data[...] = labels_blob except Exception: LOG('DataLayer exception: {}'.format(traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[0]: raise Exception( 'Cannot propagate down through the matched detections') anchor_diff_data = {} for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 1]: anchor_diff_data[anchor_id] = np.zeros( bottom[anchor_id + 1].data.shape) if len(self._samples) > 0: diff_data = np.array(top[0].diff) for out_sample_id, _ in enumerate(self._samples): alpha, betta, det_i, det_j = self._samples[out_sample_id] current_diff = diff_data[out_sample_id] if propagate_down[det_i.anchor + 1]: anchor_diff_data[det_i.anchor][det_i.item, :, det_i.y, det_i.x] \ += alpha * current_diff if propagate_down[det_j.anchor + 1]: anchor_diff_data[det_j.anchor][det_j.item, :, det_j.y, det_j.x] \ += betta * current_diff for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 1]: bottom[anchor_id + 1].diff[...] = anchor_diff_data[anchor_id] except Exception: LOG('SamplingExtractorLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: num_variables = len(bottom) assert num_variables > 0 assert len(top) == 1 or len(top) == 1 + num_variables samples = [] losses = [] for i in range(num_variables): loss_value = np.array(bottom[i].data, dtype=np.float32).reshape([-1]) assert len(loss_value) == 1 loss_value = loss_value[0] if loss_value > 0.0: param_value = self.blobs[0].data[i] loss_factor = np.exp(-param_value) new_loss_value = param_value + self._scale * loss_factor * loss_value samples.append((i, self._scale * loss_factor, self._scale * loss_factor * loss_value)) losses.append(self._weights[i] * new_loss_value) top[0].data[...] = np.sum(losses) if len(losses) > 0 else 0.0 if len(top) == 1 + num_variables: for i in range(num_variables): top[i + 1].data[...] = np.copy(bottom[i].data) self._samples = samples except Exception: LOG('AdaptiveWeightingLossLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[2]: raise Exception('Cannot propagate down through the labels') centers = np.array(bottom[0].data, dtype=np.float32) embeddings = np.array(bottom[1].data, dtype=np.float32) centers_diff_data = np.zeros( bottom[0].data.shape) if propagate_down[0] else None embeddings_diff_data = np.zeros( bottom[1].data.shape) if propagate_down[1] else None factor = top[0].diff[0] / float(self._num_instances) for class_id, mask in self._valid_class_pairs: if propagate_down[0]: centers_diff_data[class_id] += -factor * np.sum( embeddings[mask], axis=0) if propagate_down[1]: embeddings_diff_data[mask] += -factor * centers[class_id] if centers_diff_data is not None: bottom[0].diff[...] = centers_diff_data if embeddings_diff_data is not None: bottom[1].diff[...] = embeddings_diff_data except Exception: LOG('PlainCenterLossLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 1 assert len(top) == 1 detections_data = np.array(bottom[0].data) anchors_data = [] for i in range(self._num_anchors): anchors_data.append(np.array(bottom[i + 1].data)) all_detections = self._parse_detections(detections_data, INPUT_RECORD_SIZE, self._translate_prediction) all_actions = self._match_detections_with_actions( all_detections, anchors_data) if self._use_nms: all_actions = self._complex_nms(all_actions, self._min_overlap, self._num_valid_actions) matches_blob = self._convert_actions_to_blob( all_actions, OUTPUT_RECORD_SIZE) out_shape = matches_blob.shape top[0].reshape(out_shape[0], out_shape[1], out_shape[2], out_shape[3]) top[0].data[...] = matches_blob except Exception: LOG('MatcherLayer exception: {}'.format(traceback.format_exc())) exit()
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[0]: raise Exception( 'Cannot propagate down through the matched detections') diff_data = {} for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 1]: diff_data[anchor_id] = np.zeros(bottom[anchor_id + 1].data.shape) factor = top[0].diff[0] / float(len(self._candidates)) if len( self._candidates) > 0 else 0.0 for _, _, anchor_det, ref_det in self._candidates: if propagate_down[anchor_det.anchor + 1]: diff_data[anchor_det.anchor][anchor_det.item, :, anchor_det.y, anchor_det.x] \ += factor * self._embeddings[ref_det.anchor][ref_det.item, :, ref_det.y, ref_det.x] if propagate_down[ref_det.anchor + 1]: diff_data[ref_det.anchor][ref_det.item, :, ref_det.y, ref_det.x] \ += factor * self._embeddings[anchor_det.anchor][anchor_det.item, :, anchor_det.y, anchor_det.x] for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 1]: bottom[anchor_id + 1].diff[...] = diff_data[anchor_id] except Exception: LOG('PushLossLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 2 assert len(top) == 1 or len(top) == 2 or len(top) == 3 detections_data = np.array(bottom[0].data) all_detections = self._read_detections(detections_data, MATCHED_RECORD_SIZE, self._translate_matched_prediction, self._valid_action_ids, self._min_conf) self._centers = np.array(bottom[1].data) self._embeddings = [] for i in range(self._num_anchors): self._embeddings.append(np.array(bottom[i + 2].data)) if self._adaptive_weights: total_num = 0 class_counts = {cl_id: 0 for cl_id in self._valid_action_ids} for det in all_detections: class_counts[det.action] += 1 total_num += 1 normalizer = 1.0 / float(total_num) if total_num > 0 else 0.0 class_frequencies = {cl_id: normalizer * class_counts[cl_id] for cl_id in self._valid_action_ids} class_weights = self._estimate_weights(class_frequencies, self._smoothed_frequencies, self._gamma, self._weight_limits) else: class_weights = self._class_weights max_intra_class_dist = 0.0 losses = [] valid_detections = [] instance_counts = {} for det in all_detections: embedding = self._embeddings[det.anchor][det.item, :, det.y, det.x] if self._use_filtering: scale = self._get_scale(self._step, self._init_scale, self._target_scale, self._num_steps, self._power, self._use_last) scores = np.exp(scale * np.matmul(self._centers, embedding.reshape([-1, 1]))) distribution = scores / np.sum(scores) regularization_value = -np.log(distribution[det.action]) +\ self._entropy_weight * np.sum(distribution * np.log(distribution)) if regularization_value < 0.0: continue center = self._centers[det.action] dist = 1.0 - np.sum(embedding * center) max_intra_class_dist = max(max_intra_class_dist, dist) losses.append(dist) valid_detections.append(det) if det.item not in instance_counts.keys(): instance_counts[det.item] = {} local_instance_counts = instance_counts[det.item] if det.id not in local_instance_counts.keys(): local_instance_counts[det.id] = 0 local_instance_counts[det.id] += 1 if self._instance_norm: instance_weights = [class_weights[det.action] / float(instance_counts[det.item][det.id]) for det in valid_detections] num_instances = np.sum([len(counts) for counts in instance_counts.values()]) else: instance_weights = [class_weights[det.action] for det in valid_detections] num_instances = len(valid_detections) weighted_sum_losses = np.sum([instance_weights[i] * losses[i] for i, _ in enumerate(valid_detections)]) top[0].data[...] = weighted_sum_losses / float(num_instances) if num_instances > 0 else 0.0 if len(top) > 1: top[1].data[...] = max_intra_class_dist if len(top) == 3: top[2].data[...] =\ float(len(valid_detections)) / float(len(all_detections)) if len(all_detections) > 0 else 0.0 self._valid_detections = valid_detections self._weights = instance_weights self._num_instances = num_instances self._step += 1 except Exception: LOG('CenterLossLayer forward pass exception: {}'.format(traceback.format_exc())) exit()
def __init__(self, task_path, ignore_occluded, class_names_map, valid_class_ids, ignore_class_id, use_attribute): """Seeks valid images and according annotations files. :param task_path: Path to file with tasks to load data :param ignore_occluded: Whether to ignore occluded detections :param class_names_map: Dictionary to map class names to ID :param valid_class_ids: List of valid class IDs :param ignore_class_id: ID of ignored class """ self._class_names_map = class_names_map self._valid_class_ids = valid_class_ids self._ignore_class_id = ignore_class_id tasks = self._parse_tasks(task_path) LOG('Found {} tasks:'.format(len(tasks))) for task_id, task in enumerate(tasks): LOG(' {}: {}'.format(task_id, task[0])) total_frames = 0 total_objects = 0 self._all_frames = [] self._class_queues = { i: [] for i in self._valid_class_ids if i != self._ignore_class_id } glob_class_counts = { i: 0 for i in self._valid_class_ids if i != self._ignore_class_id } for task_id, task in enumerate(tasks): LOG('Loading task {}...'.format(task[0])) annotation_path = task[0] images_dir = task[1] video_resolution = task[2] image_paths = self._parse_images(images_dir) if len(image_paths) == 0: continue id_shift = (task_id + 1) * IDS_SHIFT_SCALE annotation = self._read_annotation(annotation_path, video_resolution, ignore_occluded, self._class_names_map, self._valid_class_ids, id_shift, use_attribute) for frame_id in annotation: if frame_id not in image_paths: continue image_path = image_paths[frame_id] gt_objects = annotation[frame_id] if len(gt_objects) == 0: continue # Skip images without annotated objectes if not sum([ gt_o.class_id != self._ignore_class_id for gt_o in gt_objects ]): continue self._all_frames.append( FrameDesc(path=image_path, objects=gt_objects)) frame_glob_id = len(self._all_frames) - 1 local_class_counts = { i: 0 for i in self._valid_class_ids if i != self._ignore_class_id } for gt_object in gt_objects: class_id = gt_object.class_id if class_id != self._ignore_class_id: local_class_counts[class_id] += 1 glob_class_counts[class_id] += 1 for class_id in local_class_counts: if local_class_counts[class_id] > 0: self._class_queues[class_id].append(frame_glob_id) total_frames += 1 total_objects += len(gt_objects) LOG('DataLayer stats: loaded {} frames with {} objects.'.format( total_frames, total_objects)) self._print_stat(glob_class_counts, self._class_queues, self._ignore_class_id) for class_id in self._class_queues: if class_id == ignore_class_id: continue if len(self._class_queues[class_id]) == 0: raise Exception( 'Cannot find frames with {} class id'.format(class_id))
def backward(self, top, propagate_down, bottom): """Carry out backward pass. :param top: List of top blobs :param propagate_down: List of indicators to carry out back-propagation for the specified bottom blob :param bottom: List of bottom blobs """ try: if propagate_down[0]: raise Exception( 'Cannot propagate down through the matched detections') centers_data = np.array(bottom[1].data) centers_diff_data = np.zeros( bottom[1].data.shape) if propagate_down[1] else None anchor_diff_data = {} for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 2]: anchor_diff_data[anchor_id] = np.zeros( bottom[anchor_id + 2].data.shape) if self._valid_num_pairs > 0: factor = top[0].diff[0] / float(self._valid_num_pairs) for item_id, _ in enumerate(self._masks): anchor_masks = self._masks[item_id] for anchor_id, _ in enumerate(anchor_masks): embeddings = self._embeddings[anchor_id][item_id] diff_data = anchor_diff_data[anchor_id][item_id] embedding_size = embeddings.shape[0] center_masks = anchor_masks[anchor_id] for center_id, _ in enumerate(center_masks): mask = center_masks[center_id] num_pairs = int(np.sum(mask)) mask = np.tile(np.expand_dims(mask, axis=0), reps=[embedding_size, 1, 1]) if centers_diff_data is not None: filtered_embeddings = embeddings[mask].reshape( [embedding_size, -1]) centers_diff_data[ center_id] += factor * np.sum( filtered_embeddings, axis=1) if propagate_down[anchor_id + 2]: diff_data[mask] += factor * np.tile( centers_data[center_id].reshape([-1, 1]), reps=[1, num_pairs]).reshape([-1]) if centers_diff_data is not None: bottom[1].diff[...] = centers_diff_data for anchor_id in range(self._num_anchors): if propagate_down[anchor_id + 2]: bottom[anchor_id + 2].diff[...] = anchor_diff_data[anchor_id] except Exception: LOG('GlobPushLossLayer backward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 2 assert len(top) == 1 or len(top) == 3 detections_data = np.array(bottom[0].data) batch_detections = self._read_detections( detections_data, MATCHED_RECORD_SIZE, self._translate_matched_prediction, self._valid_action_ids, self._min_conf) centers_data = np.array(bottom[1].data) num_centers = centers_data.shape[0] self._embeddings = [] for i in range(self._num_anchors): self._embeddings.append(np.array(bottom[i + 2].data)) height, width = self._embeddings[0].shape[2:] all_masks = [] out_loss = 0.0 total_num_pairs = 0 valid_num_pairs = 0 dist_sum = 0.0 for item_id in batch_detections.keys(): detections = batch_detections[item_id] outer_mask = self._outer_class_mask(detections, height, width) total_num_pairs += self._num_anchors * num_centers * int( np.sum(outer_mask)) anchor_masks = [] for anchor_id in range(self._num_anchors): anchor_embeddings = self._embeddings[anchor_id][item_id] center_masks = [] for center_id in range(num_centers): center_embedding = centers_data[center_id].reshape( [-1, 1, 1]) distances = 1.0 - np.sum( anchor_embeddings * center_embedding, axis=0) losses = self._margin - distances out_mask = (losses > 0.0) * outer_mask valid_num_pairs += int(np.sum(out_mask)) out_loss += np.sum(losses[out_mask]) dist_sum += np.sum(distances[out_mask]) center_masks.append(out_mask) anchor_masks.append(center_masks) all_masks.append(anchor_masks) top[0].data[...] = out_loss / float( valid_num_pairs) if valid_num_pairs > 0 else 0.0 if len(top) == 3: top[1].data[...] = float(valid_num_pairs) / float( total_num_pairs) if total_num_pairs > 0 else 0.0 top[2].data[...] = dist_sum / float( valid_num_pairs) if valid_num_pairs > 0 else 0.0 self._valid_num_pairs = valid_num_pairs self._masks = all_masks except Exception: LOG('GlobPushLossLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 1 assert len(top) == 1 or len(top) == 3 detections_data = np.array(bottom[0].data) all_detections = self._read_detections( detections_data, MATCHED_RECORD_SIZE, self._translate_matched_prediction, self._valid_action_ids, self._min_conf) detections = self._filter_detections(all_detections, self._max_num_samples) classes = list(detections) class_pairs = [(classes[i], classes[j]) for i, _ in enumerate(classes) for j in range(i + 1, len(classes))] self._embeddings = [] for i in range(self._num_anchors): self._embeddings.append(np.array(bottom[i + 1].data)) all_candidates = [] total_num_pairs = 0 for class_i, class_j in class_pairs: detections_i = detections[class_i] detections_j = detections[class_j] if len(detections_i) == 0 or len(detections_j) == 0: continue for i, _ in enumerate(detections_i): anchor_det = detections_i[i] anchor_embed = self._embeddings[anchor_det.anchor][ anchor_det.item, :, anchor_det.y, anchor_det.x] for j, _ in enumerate(detections_j): ref_det = detections_j[j] ref_embed = self._embeddings[ref_det.anchor][ ref_det.item, :, ref_det.y, ref_det.x] embed_dist = 1.0 - np.sum(anchor_embed * ref_embed) loss = self._margin - embed_dist if loss > 0.0: all_candidates.append( (loss, embed_dist, anchor_det, ref_det)) total_num_pairs += 1 if len(all_candidates) == 0: self._candidates = [] top[0].data[...] = 0.0 if len(top) == 3: top[1].data[...] = 0.0 top[2].data[...] = 0.0 else: if len(all_candidates) > 2: threshold_value = np.median( [tup[0] for tup in all_candidates]) self._candidates = [ tup for tup in all_candidates if tup[0] > threshold_value ] else: self._candidates = all_candidates loss = np.sum([tup[0] for tup in self._candidates]) / float( len(self._candidates)) top[0].data[...] = loss if len(top) == 3: top[1].data[...] = np.median( [tup[1] for tup in self._candidates]) top[2].data[...] = float( len(all_candidates)) / float(total_num_pairs) except Exception: LOG('PushLossLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 2 assert len(top) == 1 or len(top) == 4 detections_data = np.array(bottom[0].data) batch_detections = self._read_detections( detections_data, MATCHED_RECORD_SIZE, self._translate_matched_prediction, self._valid_action_ids, self._min_conf) centers = np.array(bottom[1].data) self._embeddings = [] for i in range(self._num_anchors): self._embeddings.append(np.array(bottom[i + 2].data)) all_candidates = [] total_num_pairs = 0 total_num_overlapped = 0 total_num_incorrect = 0 for item_id in batch_detections.keys(): detections = batch_detections[item_id] for i, _ in enumerate(detections): anchor_det = detections[i] for j in range(i + 1, len(detections)): ref_det = detections[j] # exclude same class predictions if anchor_det.action == ref_det.action: continue overlap = self._iou(anchor_det, ref_det) if overlap < self._min_overlap: continue total_num_overlapped += 1 anchor_embed = self._embeddings[anchor_det.anchor][ anchor_det.item, :, anchor_det.y, anchor_det.x] ref_embed = self._embeddings[ref_det.anchor][ ref_det.item, :, ref_det.y, ref_det.x] anchor_center_distances =\ (1.0 - np.matmul(centers, anchor_embed.reshape([-1, 1]))).reshape([-1]) ref_center_distances =\ (1.0 - np.matmul(centers, ref_embed.reshape([-1, 1]))).reshape([-1]) anchor_action = np.argmin(anchor_center_distances) ref_action = np.argmin(ref_center_distances) # exclude well-separated predictions if anchor_action != ref_action: continue # exclude predictions with both incorrect labels if anchor_action != anchor_det.action and ref_action != ref_det.action: continue total_num_incorrect += 1 embed_dist = 1.0 - np.sum(anchor_embed * ref_embed) if anchor_action != anchor_det.action: loss = self._margin + anchor_center_distances[ anchor_det.action] - embed_dist if loss > 0.0: all_candidates.append( (loss, embed_dist, anchor_det, ref_det)) total_num_pairs += 1 if ref_action != ref_det.action: loss = self._margin + ref_center_distances[ ref_det.action] - embed_dist if loss > 0.0: all_candidates.append( (loss, embed_dist, ref_det, anchor_det)) total_num_pairs += 1 if len(all_candidates) == 0: self._candidates = [] top[0].data[...] = 0.0 if len(top) == 4: top[1].data[...] = 0.0 top[2].data[...] = 0.0 top[3].data[...] = 0.0 else: self._candidates = all_candidates loss = np.sum([tup[0] for tup in self._candidates]) / float( len(self._candidates)) top[0].data[...] = loss if len(top) == 4: top[1].data[...] = np.median( [tup[1] for tup in self._candidates]) top[2].data[...] = float(len( self._candidates)) / float(total_num_pairs) top[3].data[...] = float(total_num_incorrect) / float( total_num_overlapped) except Exception: LOG('SplitLossLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 1 assert len(top) == 2 embedding_size = bottom[1].data.shape[1] detections_data = np.array(bottom[0].data) all_detections = self._read_detections( detections_data, MATCHED_RECORD_SIZE, self._translate_matched_prediction, self._valid_action_ids, self._min_conf) self._embeddings = [] for i in range(self._num_anchors): self._embeddings.append(np.array(bottom[i + 1].data)) valid_class_ids = list(all_detections) valid_class_ids.sort() self._samples = [] sampled_vectors = [] sampled_labels = [] for class_id in valid_class_ids: detections = all_detections[class_id] if len(detections) < 2: continue embeddings = np.array([ self._embeddings[det.anchor][det.item, :, det.y, det.x] for det in detections ]) center = np.mean(embeddings, axis=0, keepdims=True) center = center / np.sqrt( np.sum(np.square(center), axis=1, keepdims=True)) distances = (1.0 - np.matmul(embeddings, center.T)).reshape( [-1]) threshold = np.percentile(distances, 100.0 * self._quantile)\ if self._threshold is None else self._threshold valid_ids = np.arange(len(distances), dtype=np.int32)[distances > threshold] if len(valid_ids) < 2: continue sample_ids1 = [] sample_ids2 = [] for _ in range(self._num_steps): ids_pair = np.random.choice(valid_ids, 2, replace=False) sample_ids1.append(ids_pair[0]) sample_ids2.append(ids_pair[1]) dist_ratio = np.random.uniform(0.0, 1.0, [self._num_steps]) alpha, betta = self._calc_scales_to_mix_embeddings( dist_ratio, embeddings[sample_ids1], embeddings[sample_ids2]) alpha = alpha.reshape([self._num_steps, 1]) betta = betta.reshape([self._num_steps, 1]) sample = alpha * embeddings[sample_ids1] + betta * embeddings[ sample_ids2] sampled_vectors.append(sample) sampled_labels.append( np.full([self._num_steps], float(class_id), dtype=np.float32)) self._samples += [ (alpha[i], betta[i], detections[sample_ids1[i]], detections[sample_ids2[i]]) for i, _ in enumerate(sample_ids1) ] assert len(self._samples) == len(sampled_vectors) * self._num_steps if len(self._samples) > 0: top[0].reshape(len(self._samples), embedding_size) top[1].reshape(len(self._samples)) top[0].data[...] = np.concatenate(tuple(sampled_vectors), axis=0) top[1].data[...] = np.concatenate(tuple(sampled_labels), axis=0) else: LOG('No samples generated!') top[0].reshape(1, embedding_size) top[0].data[...] = 0.0 top[1].reshape(1) top[1].data[...] = -1.0 except Exception: LOG('SamplingExtractorLayer forward pass exception: {}'.format( traceback.format_exc())) exit()
def forward(self, bottom, top): """Carry out forward pass. :param bottom: List of bottom blobs :param top: List of top blobs """ try: assert len(bottom) == self._num_anchors + 2 assert len(top) == 1 or len(top) == 4 detections_data = np.array(bottom[0].data) all_detections = self._read_detections( detections_data, MATCHED_RECORD_SIZE, self._translate_matched_prediction, self._valid_action_ids, self._min_conf) centers_data = np.array(bottom[1].data) num_centers = centers_data.shape[0] self._embeddings = [] for i in range(self._num_anchors): self._embeddings.append(np.array(bottom[i + 2].data)) if self._adaptive_weights: total_num = 0 class_counts = {cl_id: 0 for cl_id in self._valid_action_ids} for det in all_detections: class_counts[det.action] += 1 total_num += 1 normalizer = 1.0 / float(total_num) if total_num > 0 else 0.0 class_frequencies = { cl_id: normalizer * class_counts[cl_id] for cl_id in self._valid_action_ids } class_weights = self._estimate_weights( class_frequencies, self._smoothed_frequencies, self._gamma, self._weight_limits) else: class_weights = self._class_weights pos_matches = [] total_num_matches = 0 sum_pos_distances = 0.0 sum_neg_distances = 0.0 losses = [] instance_counts = {} instance_weights = [] for det in all_detections: det_embedding = self._embeddings[det.anchor][det.item, :, det.y, det.x] center_embedding = centers_data[det.action] pos_distance = 1.0 - np.sum(det_embedding * center_embedding) for center_id in range(num_centers): if center_id == det.action: continue neg_distance = 1.0 - np.sum( det_embedding * centers_data[center_id]) loss = self._margin + pos_distance - neg_distance if loss > 0.0: losses.append(loss) sum_pos_distances += pos_distance sum_neg_distances += neg_distance pos_matches.append((det, center_id)) instance_weights.append(class_weights[det.action]) if det.item not in instance_counts: instance_counts[det.item] = {} local_instance_counts = instance_counts[det.item] if det.id not in local_instance_counts: local_instance_counts[det.id] = 0 local_instance_counts[det.id] += 1 total_num_matches += 1 if self._instance_norm: instance_weights = \ [instance_weights[i] / float(instance_counts[pos_matches[i][0].item][pos_matches[i][0].id]) for i in range(len(pos_matches))] num_instances = np.sum( [len(counts) for counts in itervalues(instance_counts)]) else: instance_weights = [ instance_weights[i] for i, _ in enumerate(pos_matches) ] num_instances = len(pos_matches) weighted_sum_losses = np.sum([ instance_weights[i] * losses[i] for i, _ in enumerate(pos_matches) ]) top[0].data[...] = weighted_sum_losses / float( num_instances) if num_instances > 0 else 0.0 if len(top) == 4: top[1].data[...] = float(len(pos_matches)) / float( total_num_matches) if total_num_matches > 0 else 0.0 top[2].data[...] = float(sum_pos_distances) / float( len(pos_matches)) if len(pos_matches) > 0 else 0.0 top[3].data[...] = float(sum_neg_distances) / float( len(pos_matches)) if len(pos_matches) > 0 else 0.0 self._pos_matches = pos_matches self._weights = instance_weights self._num_instances = num_instances except Exception: LOG('LocalPushLossLayer forward pass exception: {}'.format( traceback.format_exc())) exit()