class EAOScoreMultiStart(DependentAnalysis): low = Integer() high = Integer() eaocurve = Include(EAOCurveMultiStart) @property def title(self): return "EAO analysis" def describe(self): return Measure("Expected average overlap", "EAO", minimal=0, maximal=1, direction=Sorting.DESCENDING), def compatible(self, experiment: Experiment): return isinstance(experiment, MultiStartExperiment) def dependencies(self): return self.eaocurve, def join(self, experiment: Experiment, trackers: List[Tracker], sequences: List[Sequence], results: List[Grid]): joined = Grid((len(trackers), )) for i, result in enumerate(results[0]): if result is None: continue joined[i] = (float(np.mean(result[0][self.low:self.high + 1])), ) return joined def axes(self): return Axis.TRACKERS,
class Redetection(Transformer): length = Integer(default=100, val_min=1) initialization = Integer(default=5, val_min=1) padding = Float(default=2, val_min=0) scaling = Float(default=1, val_min=0.1, val_max=10) def __call__(self, sequence: Sequence) -> Sequence: chache_dir = self._cache.directory( self, arg_hash(sequence.name, **self.dump())) if not os.path.isfile(os.path.join(chache_dir, "sequence")): generated = InMemorySequence(sequence.name, sequence.channels()) size = (int(sequence.size[0] * self.scaling), int(sequence.size[1] * self.scaling)) initial_images = dict() redetect_images = dict() for channel in sequence.channels(): rect = sequence.frame(0).groundtruth().convert( RegionType.RECTANGLE) halfsize = int(max(rect.width, rect.height) * self.scaling / 2) x, y = rect.center() image = Image.fromarray(sequence.frame(0).image()) box = (x - halfsize, y - halfsize, x + halfsize, y + halfsize) template = image.crop(box) initial = Image.new(image.mode, size) initial.paste(image, (0, 0)) redetect = Image.new(image.mode, size) redetect.paste( template, (size[0] - template.width, size[1] - template.height)) initial_images[channel] = initial redetect_images[channel] = redetect generated.append(initial_images, sequence.frame(0).groundtruth()) generated.append( redetect_images, sequence.frame(0).groundtruth().move( size[0] - template.width, size[1] - template.height)) write_sequence(chache_dir, generated) source = VOTSequence(chache_dir, name=sequence.name) mapping = [0] * self.initialization + [1] * (self.length - self.initialization) return FrameMapSequence(source, mapping)
class TrackerSorter(Attributee): experiment = String(default=None) analysis = String(default=None) result = Integer(val_min=0, default=0) def __call__(self, experiments, trackers, sequences): if self.experiment is None or self.analysis is None: return range(len(trackers)) experiment = next(filter(lambda x: x.identifier == self.experiment, experiments), None) if experiment is None: raise RuntimeError("Experiment not found") analysis = next(filter(lambda x: x.name == self.analysis, experiment.analyses), None) if analysis is None: raise RuntimeError("Analysis not found") future = analysis.commit(experiment, trackers, sequences) result = future.result() scores = [x[self.result] for x in result] indices = [i[0] for i in sorted(enumerate(scores), reverse=True, key=lambda x: x[1])] return indices
class AverageAccuracyPerSequence(FullySeparableAnalysis): burnin = Integer(default=10, val_min=0) ignore_unknown = Boolean(default=True) bounded = Boolean(default=True) def compatible(self, experiment: Experiment): return isinstance(experiment, MultiRunExperiment) @property def title(self): return "Average accurarcy" def describe(self): return Measure("Accuracy", "AUC", 0, 1, Sorting.DESCENDING), def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): if isinstance(experiment, MultiRunExperiment): trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException() cummulative = 0 for trajectory in trajectories: accuracy, _ = compute_accuracy(trajectory.regions(), sequence, self.burnin, self.ignore_unknown, self.bounded) cummulative = cummulative + accuracy return cummulative / len(trajectories),
class SupervisedExperiment(MultiRunExperiment): skip_initialize = Integer(val_min=1, default=1) skip_tags = List(String(), default=[]) failure_overlap = Float(val_min=0, val_max=1, default=0) def execute(self, tracker: Tracker, sequence: Sequence, force: bool = False, callback: Callable = None): results = self.results(tracker, sequence) with self._get_runtime(tracker, sequence) as runtime: for i in range(1, self.repetitions+1): name = "%s_%03d" % (sequence.name, i) if Trajectory.exists(results, name) and not force: continue if self._can_stop(tracker, sequence): return trajectory = Trajectory(sequence.length) frame = 0 while frame < sequence.length: _, properties, elapsed = runtime.initialize(sequence.frame(frame), self._get_initialization(sequence, frame)) properties["time"] = elapsed trajectory.set(frame, Special(Special.INITIALIZATION), properties) frame = frame + 1 while frame < sequence.length: region, properties, elapsed = runtime.update(sequence.frame(frame)) properties["time"] = elapsed if calculate_overlap(region, sequence.groundtruth(frame), sequence.size) <= self.failure_overlap: trajectory.set(frame, Special(Special.FAILURE), properties) frame = frame + self.skip_initialize if self.skip_tags: while frame < sequence.length: if not [t for t in sequence.tags(frame) if t in self.skip_tags]: break frame = frame + 1 break else: trajectory.set(frame, region, properties) frame = frame + 1 if callback: callback(i / self.repetitions) trajectory.write(results, name)
class AccuracyRobustness(SequenceAveragingAnalysis): sensitivity = Float(default=30, val_min=1) burnin = Integer(default=10, val_min=0) ignore_unknown = Boolean(default=True) bounded = Boolean(default=True) @property def name(self): return "AR analysis" def describe(self): return Measure("Accuracy", "A", minimal=0, maximal=1, direction=Sorting.DESCENDING), \ Measure("Robustness", "R", minimal=0, direction=Sorting.DESCENDING), \ Point("AR plot", dimensions=2, abbreviation="AR", minimal=(0, 0), \ maximal=(1, 1), labels=("Robustness", "Accuracy"), hints=Hints.AXIS_EQUAL), \ None def compatible(self, experiment: Experiment): return isinstance(experiment, SupervisedExperiment) def collapse(self, _: Tracker, sequences: List[Sequence], results: List[tuple]): failures = 0 accuracy = 0 weight_total = 0 for a, f, _, w in results: failures += f * w accuracy += a * w weight_total += w ar = (accuracy / weight_total, math.exp(-(failures / weight_total) * float(self.sensitivity))) return accuracy / weight_total, failures / weight_total, ar, weight_total def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException() accuracy = 0 failures = 0 for trajectory in trajectories: failures += count_failures(trajectory.regions())[0] accuracy += compute_accuracy(trajectory.regions(), sequence, self.burnin, self.ignore_unknown, self.bounded)[0] ar = (accuracy / len(trajectories), math.exp(-(float(failures) / len(trajectories)) * float(self.sensitivity))) return accuracy / len(trajectories), failures / len( trajectories), ar, len(trajectories[0])
class EAOCurve(TrackerSeparableAnalysis): burnin = Integer(default=10, val_min=0) bounded = Boolean(default=True) @property def title(self): return "EAO Curve" def describe(self): return Plot("Expected Average Overlap", "EAO", minimal=0, maximal=1, trait="eao"), def compatible(self, experiment: Experiment): return isinstance(experiment, SupervisedExperiment) def subcompute(self, experiment: Experiment, tracker: Tracker, sequences: List[Sequence]): overlaps_all = [] weights_all = [] success_all = [] for sequence in sequences: trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException() for trajectory in trajectories: overlaps = calculate_overlaps(trajectory.regions(), sequence.groundtruth(), (sequence.size) if self.bounded else None) fail_idxs, init_idxs = locate_failures_inits(trajectory.regions()) if len(fail_idxs) > 0: for i in range(len(fail_idxs)): overlaps_all.append(overlaps[init_idxs[i]:fail_idxs[i]]) success_all.append(False) weights_all.append(1) # handle last initialization if len(init_idxs) > len(fail_idxs): # tracker was initilized, but it has not failed until the end of the sequence overlaps_all.append(overlaps[init_idxs[-1]:]) success_all.append(True) weights_all.append(1) else: overlaps_all.append(overlaps) success_all.append(True) weights_all.append(1) result = Grid((1,1)) result[0, 0] = (compute_eao_curve(overlaps_all, weights_all, success_all),) return result
class EAOScore(DependentAnalysis): eaocurve = Include(EAOCurve) low = Integer() high = Integer() @property def title(self): return "EAO analysis" def describe(self): return Measure("Expected average overlap", "EAO", 0, 1, Sorting.DESCENDING), def compatible(self, experiment: Experiment): return isinstance(experiment, SupervisedExperiment) def dependencies(self): return self.eaocurve, def join(self, experiment: Experiment, trackers: List[Tracker], sequences: List[Sequence], results: List[Grid]): return [(float(np.mean(x[0][self.low:self.high + 1])), ) for x in results[0]] def axes(self): return Axis.TRACKERS,
class MultiRunExperiment(Experiment): repetitions = Integer(val_min=1, default=1) early_stop = Boolean(default=True) def _can_stop(self, tracker: Tracker, sequence: Sequence): if not self.early_stop: return False trajectories = self.gather(tracker, sequence) if len(trajectories) < 3: return False for trajectory in trajectories[1:]: if not trajectory.equals(trajectories[0]): return False return True def scan(self, tracker: Tracker, sequence: Sequence): results = self.results(tracker, sequence) files = [] complete = True for i in range(1, self.repetitions+1): name = "%s_%03d" % (sequence.name, i) if Trajectory.exists(results, name): files.extend(Trajectory.gather(results, name)) elif self._can_stop(tracker, sequence): break else: complete = False break return complete, files, results def gather(self, tracker: Tracker, sequence: Sequence): trajectories = list() results = self.results(tracker, sequence) for i in range(1, self.repetitions+1): name = "%s_%03d" % (sequence.name, i) if Trajectory.exists(results, name): trajectories.append(Trajectory.read(results, name)) return trajectories
class AverageAccuracy(SequenceAveragingAnalysis): burnin = Integer(default=10, val_min=0) ignore_unknown = Boolean(default=True) bounded = Boolean(default=True) def compatible(self, experiment: Experiment): return isinstance(experiment, MultiRunExperiment) @property def name(self): return "Average accurarcy per sequence" def describe(self): return Measure("Accuracy", "AUC", 0, 1, Sorting.DESCENDING), def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): if isinstance(experiment, MultiRunExperiment): trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException() cummulative = 0 for trajectory in trajectories: accuracy, _ = compute_accuracy(trajectory.regions(), sequence, self.burnin, self.ignore_unknown, self.bounded) cummulative = cummulative + accuracy return cummulative / len(trajectories), def collapse(self, _: Tracker, sequences: List[Sequence], results: List[tuple]): accuracy = 0 frames = 0 for sequence, a in zip(sequences, results): accuracy = accuracy + a[0] * len(sequence) frames = frames + len(sequence) return accuracy / frames,
class RealtimeConfig(Attributee): grace = Integer(val_min=0, default=0) fps = Float(val_min=0, default=20)
class AccuracyRobustnessMultiStart(SequenceAveragingAnalysis): burnin = Integer(default=10, val_min=0) grace = Integer(default=10, val_min=0) bounded = Boolean(default=True) threshold = Float(default=0.1, val_min=0, val_max=1) @property def title(self): return "AR analysis" def describe(self): return Measure("Accuracy", "A", minimal=0, maximal=1, direction=Sorting.DESCENDING), \ Measure("Robustness", "R", minimal=0, direction=Sorting.DESCENDING), \ Point("AR plot", dimensions=2, abbreviation="AR", minimal=(0, 0), maximal=(1, 1), labels=("Robustness", "Accuracy"), trait="ar"), \ None, None def compatible(self, experiment: Experiment): return isinstance(experiment, MultiStartExperiment) def collapse(self, tracker: Tracker, sequences: List[Sequence], results: List[tuple]): total_accuracy = 0 total_robustness = 0 weight_accuracy = 0 weight_robustness = 0 for accuracy, robustness, _, accuracy_w, robustness_w in results: total_accuracy += accuracy * accuracy_w total_robustness += robustness * robustness_w weight_accuracy += accuracy_w weight_robustness += robustness_w ar = (total_robustness / weight_robustness, total_accuracy / weight_accuracy) return total_accuracy / weight_accuracy, total_robustness / weight_robustness, ar, weight_accuracy, weight_robustness def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if not forward and not backward: raise RuntimeError("Sequence does not contain any anchors") robustness = 0 accuracy = 0 total = 0 for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps( trajectory.regions(), proxy.groundtruth(), (proxy.size) if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace robustness += progress # simplified original equation: len(proxy) * (progress / len(proxy)) accuracy += sum(overlaps[0:progress]) total += len(proxy) ar = (robustness / total, accuracy / robustness if robustness > 0 else 0) return accuracy / robustness if robustness > 0 else 0, robustness / total, ar, robustness, len( sequence)
class AttributeMultiStart(SequenceAveragingAnalysis): burnin = Integer(default=10, val_min=0) grace = Integer(default=10, val_min=0) bounded = Boolean(default=True) threshold = Float(default=0.1, val_min=0, val_max=1) tags = List(String()) @property def name(self): return "AR per-attribute analysis" def describe(self): accuracy = [ Measure("Accuracy: " + t, "A " + t, minimal=0, maximal=1, direction=Sorting.DESCENDING) for t in self.tags ] robustness = [ Measure("Robutsness" + t, "R " + t, minimal=0, maximal=1, direction=Sorting.DESCENDING) for t in self.tags ] length = [None] * len(self.tags) return tuple( functools.reduce( operator.add, [[a, r, n] for a, r, n in zip(accuracy, robustness, length)])) def compatible(self, experiment: Experiment): return isinstance(experiment, MultiStartExperiment) def collapse(self, tracker: Tracker, sequences: typing.List[Sequence], results: typing.List[tuple]): accuracy = Counter() robustness = Counter() attribute_total = Counter() for seq_acc, seq_rob, seq_attr_count in results: for t in seq_attr_count: accuracy[t] += (seq_acc[t] if t in seq_acc else 0) * seq_attr_count[t] robustness[t] += seq_rob * seq_attr_count[t] attribute_total[t] += seq_attr_count[t] accuracy = [accuracy[t] / attribute_total[t] for t in self.tags] robustness = [robustness[t] / attribute_total[t] for t in self.tags] length = [attribute_total[t] for t in self.tags] return tuple( functools.reduce( operator.add, [[a, r, n] for a, r, n in zip(accuracy, robustness, length)])) def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") accuracy_ = Counter() tags_count_ = Counter() robustness_ = 0 total_ = 0 for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace for j in range(progress): overlap = overlaps[j] tags = proxy.tags(j) if len(tags) == 0: tags = ['empty'] for t in tags: accuracy_[t] += overlap tags_count_[t] += 1 robustness_ += progress total_ += len(proxy) seq_robustness = robustness_ / total_ seq_accuracy = {} for t in accuracy_: seq_accuracy[t] = accuracy_[t] / tags_count_[t] # calculate weights for each attribute attribute_counter = Counter() for frame_idx in range(len(sequence)): tags = sequence.tags(frame_idx) if len(tags) == 0: tags = ['empty'] for t in tags: attribute_counter[t] += 1 return seq_accuracy, seq_robustness, attribute_counter
class AttributeDifficultyLevelMultiStart(SequenceAveragingAnalysis): burnin = Integer(default=10, val_min=0) grace = Integer(default=10, val_min=0) bounded = Boolean(default=True) threshold = Float(default=0.1, val_min=0, val_max=1) fail_interval = Integer(default=30, val_min=1) tags = List(String()) @property def name(self): return "Attribute difficulty" def describe(self): return tuple([ Measure(t, t, minimal=0, maximal=1, direction=Sorting.DESCENDING) for t in self.tags ] + [None] * len(self.tags)) def compatible(self, experiment: Experiment): return isinstance(experiment, MultiStartExperiment) def collapse(self, tracker: Tracker, sequences: typing.List[Sequence], results: typing.List[tuple]): attribute_difficulty = Counter() attribute_counter = Counter() for seq_tags_not_failed, seq_tags_count, seq_attr_count in results: for tag in seq_tags_count: if tag in seq_tags_not_failed: seq_attr_difficulty = seq_tags_not_failed[ tag] / seq_tags_count[tag] else: seq_attr_difficulty = 0 attribute_difficulty[ tag] += seq_attr_difficulty * seq_attr_count[tag] attribute_counter[tag] += seq_attr_count[tag] return tuple([ attribute_difficulty[tag] / attribute_counter[tag] for tag in self.tags ] + [attribute_counter[tag] for tag in self.tags]) def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") tags_count = Counter() tags_not_failed = Counter() for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth( j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace for j in range(progress): tags = proxy.tags(j) if len(tags) == 0: tags = ['empty'] for t in tags: tags_count[t] += 1 if progress == len( proxy) or j < progress - self.fail_interval: tags_not_failed[t] += 1 attribute_counter = Counter() for frame_idx in range(len(sequence)): tags = sequence.tags(frame_idx) if len(tags) == 0: tags = ['empty'] for t in tags: attribute_counter[t] += 1 return tags_not_failed, tags_count, attribute_counter
class EAOCurveMultiStart(SequenceAveragingAnalysis): burnin = Integer(default=10, val_min=0) grace = Integer(default=10, val_min=0) bounded = Boolean(default=True) threshold = Float(default=0.1, val_min=0, val_max=1) low = Integer() high = Integer() @property def title(self): return "EAO Curve" def describe(self): return Plot("Expected average overlap", "EAO", minimal=0, maximal=1, wrt="frames", trait="eao"), def compatible(self, experiment: Experiment): return isinstance(experiment, MultiStartExperiment) def collapse(self, tracker: Tracker, sequences: List[Sequence], results: Grid): eao_curve = self.high * [float(0)] eao_weights = self.high * [float(0)] for (seq_eao_curve, eao_active), seq_w in results: for i, (eao_, active_) in enumerate(zip(seq_eao_curve, eao_active)): eao_curve[i] += eao_ * active_ * seq_w eao_weights[i] += active_ * seq_w return [eao_ / w_ if w_ > 0 else 0 for eao_, w_ in zip(eao_curve, eao_weights)], def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence): results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") overlaps_all = [] success_all = [] for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth(j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace success = True if progress < len(overlaps): # tracker has failed during this run overlaps[progress:] = (len(overlaps) - progress) * [float(0)] success = False overlaps_all.append(overlaps) success_all.append(success) return compute_eao_partial(overlaps_all, success_all, self.high), 1
class InjectConfig(Attributee): # Not implemented yet placeholder = Integer(default=1)
class PrecisionRecallCurve(TrackerSeparableAnalysis): resolution = Integer(default=100) ignore_unknown = Boolean(default=True) bounded = Boolean(default=True) @property def title(self): return "Tracking precision/recall" def describe(self): return Curve("Precision Recall curve", dimensions=2, abbreviation="PR", minimal=(0, 0), maximal=(1, 1), labels=("Recall", "Precision")), None def compatible(self, experiment: Experiment): return isinstance(experiment, UnsupervisedExperiment) def subcompute(self, experiment: Experiment, tracker: Tracker, sequences: List[Sequence]): # calculate thresholds total_scores = 0 for sequence in sequences: trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException( "Missing results for sequence {}".format(sequence.name)) for trajectory in trajectories: total_scores += len(trajectory) # allocate memory for all scores scores_all = total_scores * [float(0)] idx = 0 for sequence in sequences: trajectories = experiment.gather(tracker, sequence) for trajectory in trajectories: conf_ = [ trajectory.properties(i).get('confidence', 0) for i in range(len(trajectory)) ] scores_all[idx:idx + len(conf_)] = conf_ idx += len(conf_) thresholds = determine_thresholds(scores_all, self.resolution) # calculate per-sequence Precision and Recall curves pr_curves = [] re_curves = [] for sequence in sequences: trajectories = experiment.gather(tracker, sequence) if len(trajectories) == 0: raise MissingResultsException() pr = len(thresholds) * [float(0)] re = len(thresholds) * [float(0)] for trajectory in trajectories: conf_ = [ trajectory.properties(i).get('confidence', 0) for i in range(len(trajectory)) ] pr_, re_ = compute_tpr_curves(trajectory.regions(), conf_, sequence, thresholds, self.ignore_unknown, self.bounded) pr = [p1 + p2 for p1, p2 in zip(pr, pr_)] re = [r1 + r2 for r1, r2 in zip(re, re_)] pr = [p1 / len(trajectories) for p1 in pr] re = [r1 / len(trajectories) for r1 in re] pr_curves.append(pr) re_curves.append(re) # calculate a single Precision, Recall and F-score curves for a given tracker # average Pr-Re curves over the sequences pr_curve = len(thresholds) * [float(0)] re_curve = len(thresholds) * [float(0)] for i, _ in enumerate(thresholds): for j, _ in enumerate(pr_curves): pr_curve[i] += pr_curves[j][i] re_curve[i] += re_curves[j][i] curve = [(re / len(pr_curves), pr / len(pr_curves)) for pr, re in zip(pr_curve, re_curve)] return curve, thresholds
class EAOCurveMultiStart2(TrackerSeparableAnalysis): burnin = Integer(default=10, val_min=0) grace = Integer(default=10, val_min=0) bounded = Boolean(default=True) threshold = Float(default=0.1, val_min=0, val_max=1) @property def title(self): return "EAO Curve" def describe(self): return Plot("Expected Average Overlap", "EAO", minimal=0, maximal=1), def compatible(self, experiment: Experiment): return isinstance(experiment, MultiStartExperiment) def subcompute(self, experiment: Experiment, tracker: Tracker, sequences: List[Sequence]): overlaps_all = [] weights_all = [] success_all = [] frames_total = 0 for sequence in sequences: results = experiment.results(tracker, sequence) forward, backward = find_anchors(sequence, experiment.anchor) if len(forward) == 0 and len(backward) == 0: raise RuntimeError("Sequence does not contain any anchors") weights_per_run = [] for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]: name = "%s_%08d" % (sequence.name, i) if not Trajectory.exists(results, name): raise MissingResultsException() if reverse: proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1)))) else: proxy = FrameMapSequence(sequence, list(range(i, sequence.length))) trajectory = Trajectory.read(results, name) overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None) grace = self.grace progress = len(proxy) for j, overlap in enumerate(overlaps): if overlap <= self.threshold and not proxy.groundtruth(j).is_empty(): grace = grace - 1 if grace == 0: progress = j + 1 - self.grace # subtract since we need actual point of the failure break else: grace = self.grace success = True if progress < len(overlaps): # tracker has failed during this run overlaps[progress:] = (len(overlaps) - progress) * [float(0)] success = False overlaps_all.append(overlaps) success_all.append(success) weights_per_run.append(len(proxy)) for w in weights_per_run: weights_all.append((w / sum(weights_per_run)) * len(sequence)) frames_total += len(sequence) weights_all = [w / frames_total for w in weights_all] return compute_eao_curve(overlaps_all, weights_all, success_all),