def compute_tpr_curves(trajectory: List[Region], confidence: List[float], sequence: Sequence, thresholds: List[float], ignore_unknown: bool = True, bounded: bool = True): overlaps = np.array( calculate_overlaps(trajectory, sequence.groundtruth(), (sequence.size) if bounded else None)) confidence = np.array(confidence) n_visible = len([ region for region in sequence.groundtruth() if region.type is not RegionType.SPECIAL ]) precision = len(thresholds) * [float(0)] recall = len(thresholds) * [float(0)] for i, threshold in enumerate(thresholds): subset = confidence >= threshold if np.sum(subset) == 0: precision[i] = 1 recall[i] = 0 else: precision[i] = np.mean(overlaps[subset]) recall[i] = np.sum(overlaps[subset]) / n_visible return precision, recall
def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence, dependencies: List[Grid]) -> Tuple[Any]: results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") overlaps_all = [] success_all = [] for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace success = True if progress < len(overlaps): # tracker has failed during this run overlaps[progress:] = (len(overlaps) - progress) * [float(0)] success = False overlaps_all.append(overlaps) success_all.append(success) return compute_eao_partial(overlaps_all, success_all, self.high), 1
def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if not forward and not backward: raise RuntimeError("Sequence does not contain any anchors") robustness = 0 accuracy = 0 total = 0 for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps( trajectory.regions(), proxy.groundtruth(), (proxy.size) if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace robustness += progress # simplified original equation: len(proxy) * (progress / len(proxy)) accuracy += sum(overlaps[0:progress]) total += len(proxy) ar = (robustness / total, accuracy / robustness if robustness > 0 else 0) return accuracy / robustness if robustness > 0 else 0, robustness / total, ar, robustness, len( sequence)
def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence, dependencies: List[Grid]) -> Tuple[Any]: results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if not forward and not backward: raise RuntimeError("Sequence does not contain any anchors") accuracy = [] success = [] for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps( trajectory.regions(), proxy.groundtruth(), (proxy.size) if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace success.append((i / len(sequence), progress / len(proxy))) accuracy.append( (i / len(sequence), sum(overlaps[0:progress] / len(proxy)))) return success, accuracy
def subcompute(self, experiment: Experiment, tracker: Tracker, sequences: List[Sequence], dependencies: List[Grid]) -> Tuple[Any]: overlaps_all = [] weights_all = [] success_all = [] for sequence in sequences: trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException() for trajectory in trajectories: overlaps = calculate_overlaps( trajectory.regions(), sequence.groundtruth(), (sequence.size) if self.bounded else None) fail_idxs, init_idxs = locate_failures_inits( trajectory.regions()) if len(fail_idxs) > 0: for i in range(len(fail_idxs)): overlaps_all.append( overlaps[init_idxs[i]:fail_idxs[i]]) success_all.append(False) weights_all.append(1) # handle last initialization if len(init_idxs) > len(fail_idxs): # tracker was initilized, but it has not failed until the end of the sequence overlaps_all.append(overlaps[init_idxs[-1]:]) success_all.append(True) weights_all.append(1) else: overlaps_all.append(overlaps) success_all.append(True) weights_all.append(1) return compute_eao_curve(overlaps_all, weights_all, success_all),
def compute_accuracy(trajectory: List[Region], sequence: Sequence, burnin: int = 10, ignore_unknown: bool = True, bounded: bool = True) -> float: overlaps = np.array( calculate_overlaps(trajectory, sequence.groundtruth(), (sequence.size) if bounded else None)) mask = np.ones(len(overlaps), dtype=bool) for i, region in enumerate(trajectory): if is_special(region, Special.UNKNOWN) and ignore_unknown: mask[i] = False elif is_special(region, Special.INITIALIZATION): for j in range(i, min(len(trajectory), i + burnin)): mask[j] = False elif is_special(region, Special.FAILURE): mask[i] = False if any(mask): return np.mean(overlaps[mask]), np.sum(mask) else: return 0, 0
def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") accuracy_ = Counter() tags_count_ = Counter() robustness_ = 0 total_ = 0 for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace for j in range(progress): overlap = overlaps[j] tags = proxy.tags(j) if len(tags) == 0: tags = ['empty'] for t in tags: accuracy_[t] += overlap tags_count_[t] += 1 robustness_ += progress total_ += len(proxy) seq_robustness = robustness_ / total_ seq_accuracy = {} for t in accuracy_: seq_accuracy[t] = accuracy_[t] / tags_count_[t] # calculate weights for each attribute attribute_counter = Counter() for frame_idx in range(len(sequence)): tags = sequence.tags(frame_idx) if len(tags) == 0: tags = ['empty'] for t in tags: attribute_counter[t] += 1 return seq_accuracy, seq_robustness, attribute_counter
def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") tags_count = Counter() tags_not_failed = Counter() for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace for j in range(progress): tags = proxy.tags(j) if len(tags) == 0: tags = ['empty'] for t in tags: tags_count[t] += 1 if progress == len( proxy) or j < progress - self.fail_interval: tags_not_failed[t] += 1 attribute_counter = Counter() for frame_idx in range(len(sequence)): tags = sequence.tags(frame_idx) if len(tags) == 0: tags = ['empty'] for t in tags: attribute_counter[t] += 1 return tags_not_failed, tags_count, attribute_counter