class EAOScoreMultiStart(DependentAnalysis):
low = Integer()
high = Integer()
eaocurve = Include(EAOCurveMultiStart)
@property
def title(self):
return "EAO analysis"
def describe(self):
return Measure("Expected average overlap", "EAO", minimal=0, maximal=1, direction=Sorting.DESCENDING),
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiStartExperiment)
def dependencies(self):
return self.eaocurve,
def join(self, experiment: Experiment, trackers: List[Tracker], sequences: List[Sequence], results: List[Grid]):
joined = Grid((len(trackers), ))
for i, result in enumerate(results[0]):
if result is None:
continue
joined[i] = (float(np.mean(result[0][self.low:self.high + 1])), )
return joined
def axes(self):
return Axis.TRACKERS,
class Redetection(Transformer):
length = Integer(default=100, val_min=1)
initialization = Integer(default=5, val_min=1)
padding = Float(default=2, val_min=0)
scaling = Float(default=1, val_min=0.1, val_max=10)
def __call__(self, sequence: Sequence) -> Sequence:
chache_dir = self._cache.directory(
self, arg_hash(sequence.name, **self.dump()))
if not os.path.isfile(os.path.join(chache_dir, "sequence")):
generated = InMemorySequence(sequence.name, sequence.channels())
size = (int(sequence.size[0] * self.scaling),
int(sequence.size[1] * self.scaling))
initial_images = dict()
redetect_images = dict()
for channel in sequence.channels():
rect = sequence.frame(0).groundtruth().convert(
RegionType.RECTANGLE)
halfsize = int(max(rect.width, rect.height) * self.scaling / 2)
x, y = rect.center()
image = Image.fromarray(sequence.frame(0).image())
box = (x - halfsize, y - halfsize, x + halfsize, y + halfsize)
template = image.crop(box)
initial = Image.new(image.mode, size)
initial.paste(image, (0, 0))
redetect = Image.new(image.mode, size)
redetect.paste(
template,
(size[0] - template.width, size[1] - template.height))
initial_images[channel] = initial
redetect_images[channel] = redetect
generated.append(initial_images, sequence.frame(0).groundtruth())
generated.append(
redetect_images,
sequence.frame(0).groundtruth().move(
size[0] - template.width, size[1] - template.height))
write_sequence(chache_dir, generated)
source = VOTSequence(chache_dir, name=sequence.name)
mapping = [0] * self.initialization + [1] * (self.length -
self.initialization)
return FrameMapSequence(source, mapping)
class TrackerSorter(Attributee):
experiment = String(default=None)
analysis = String(default=None)
result = Integer(val_min=0, default=0)
def __call__(self, experiments, trackers, sequences):
if self.experiment is None or self.analysis is None:
return range(len(trackers))
experiment = next(filter(lambda x: x.identifier == self.experiment, experiments), None)
if experiment is None:
raise RuntimeError("Experiment not found")
analysis = next(filter(lambda x: x.name == self.analysis, experiment.analyses), None)
if analysis is None:
raise RuntimeError("Analysis not found")
future = analysis.commit(experiment, trackers, sequences)
result = future.result()
scores = [x[self.result] for x in result]
indices = [i[0] for i in sorted(enumerate(scores), reverse=True, key=lambda x: x[1])]
return indices
class AverageAccuracyPerSequence(FullySeparableAnalysis):
burnin = Integer(default=10, val_min=0)
ignore_unknown = Boolean(default=True)
bounded = Boolean(default=True)
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiRunExperiment)
@property
def title(self):
return "Average accurarcy"
def describe(self):
return Measure("Accuracy", "AUC", 0, 1, Sorting.DESCENDING),
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequence: Sequence):
if isinstance(experiment, MultiRunExperiment):
trajectories = experiment.gather(tracker, sequence)
if len(trajectories) == 0:
raise MissingResultsException()
cummulative = 0
for trajectory in trajectories:
accuracy, _ = compute_accuracy(trajectory.regions(), sequence,
self.burnin,
self.ignore_unknown,
self.bounded)
cummulative = cummulative + accuracy
return cummulative / len(trajectories),
class SupervisedExperiment(MultiRunExperiment):
skip_initialize = Integer(val_min=1, default=1)
skip_tags = List(String(), default=[])
failure_overlap = Float(val_min=0, val_max=1, default=0)
def execute(self, tracker: Tracker, sequence: Sequence, force: bool = False, callback: Callable = None):
results = self.results(tracker, sequence)
with self._get_runtime(tracker, sequence) as runtime:
for i in range(1, self.repetitions+1):
name = "%s_%03d" % (sequence.name, i)
if Trajectory.exists(results, name) and not force:
continue
if self._can_stop(tracker, sequence):
return
trajectory = Trajectory(sequence.length)
frame = 0
while frame < sequence.length:
_, properties, elapsed = runtime.initialize(sequence.frame(frame), self._get_initialization(sequence, frame))
properties["time"] = elapsed
trajectory.set(frame, Special(Special.INITIALIZATION), properties)
frame = frame + 1
while frame < sequence.length:
region, properties, elapsed = runtime.update(sequence.frame(frame))
properties["time"] = elapsed
if calculate_overlap(region, sequence.groundtruth(frame), sequence.size) <= self.failure_overlap:
trajectory.set(frame, Special(Special.FAILURE), properties)
frame = frame + self.skip_initialize
if self.skip_tags:
while frame < sequence.length:
if not [t for t in sequence.tags(frame) if t in self.skip_tags]:
break
frame = frame + 1
break
else:
trajectory.set(frame, region, properties)
frame = frame + 1
if callback:
callback(i / self.repetitions)
trajectory.write(results, name)
class AccuracyRobustness(SequenceAveragingAnalysis):
sensitivity = Float(default=30, val_min=1)
burnin = Integer(default=10, val_min=0)
ignore_unknown = Boolean(default=True)
bounded = Boolean(default=True)
@property
def name(self):
return "AR analysis"
def describe(self):
return Measure("Accuracy", "A", minimal=0, maximal=1, direction=Sorting.DESCENDING), \
Measure("Robustness", "R", minimal=0, direction=Sorting.DESCENDING), \
Point("AR plot", dimensions=2, abbreviation="AR", minimal=(0, 0), \
maximal=(1, 1), labels=("Robustness", "Accuracy"), hints=Hints.AXIS_EQUAL), \
None
def compatible(self, experiment: Experiment):
return isinstance(experiment, SupervisedExperiment)
def collapse(self, _: Tracker, sequences: List[Sequence],
results: List[tuple]):
failures = 0
accuracy = 0
weight_total = 0
for a, f, _, w in results:
failures += f * w
accuracy += a * w
weight_total += w
ar = (accuracy / weight_total,
math.exp(-(failures / weight_total) * float(self.sensitivity)))
return accuracy / weight_total, failures / weight_total, ar, weight_total
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequence: Sequence):
trajectories = experiment.gather(tracker, sequence)
if len(trajectories) == 0:
raise MissingResultsException()
accuracy = 0
failures = 0
for trajectory in trajectories:
failures += count_failures(trajectory.regions())[0]
accuracy += compute_accuracy(trajectory.regions(), sequence,
self.burnin, self.ignore_unknown,
self.bounded)[0]
ar = (accuracy / len(trajectories),
math.exp(-(float(failures) / len(trajectories)) *
float(self.sensitivity)))
return accuracy / len(trajectories), failures / len(
trajectories), ar, len(trajectories[0])
class EAOCurve(TrackerSeparableAnalysis):
burnin = Integer(default=10, val_min=0)
bounded = Boolean(default=True)
@property
def title(self):
return "EAO Curve"
def describe(self):
return Plot("Expected Average Overlap", "EAO", minimal=0, maximal=1, trait="eao"),
def compatible(self, experiment: Experiment):
return isinstance(experiment, SupervisedExperiment)
def subcompute(self, experiment: Experiment, tracker: Tracker, sequences: List[Sequence]):
overlaps_all = []
weights_all = []
success_all = []
for sequence in sequences:
trajectories = experiment.gather(tracker, sequence)
if len(trajectories) == 0:
raise MissingResultsException()
for trajectory in trajectories:
overlaps = calculate_overlaps(trajectory.regions(), sequence.groundtruth(), (sequence.size) if self.bounded else None)
fail_idxs, init_idxs = locate_failures_inits(trajectory.regions())
if len(fail_idxs) > 0:
for i in range(len(fail_idxs)):
overlaps_all.append(overlaps[init_idxs[i]:fail_idxs[i]])
success_all.append(False)
weights_all.append(1)
# handle last initialization
if len(init_idxs) > len(fail_idxs):
# tracker was initilized, but it has not failed until the end of the sequence
overlaps_all.append(overlaps[init_idxs[-1]:])
success_all.append(True)
weights_all.append(1)
else:
overlaps_all.append(overlaps)
success_all.append(True)
weights_all.append(1)
result = Grid((1,1))
result[0, 0] = (compute_eao_curve(overlaps_all, weights_all, success_all),)
return result
class EAOScore(DependentAnalysis):
eaocurve = Include(EAOCurve)
low = Integer()
high = Integer()
@property
def title(self):
return "EAO analysis"
def describe(self):
return Measure("Expected average overlap", "EAO", 0, 1, Sorting.DESCENDING),
def compatible(self, experiment: Experiment):
return isinstance(experiment, SupervisedExperiment)
def dependencies(self):
return self.eaocurve,
def join(self, experiment: Experiment, trackers: List[Tracker], sequences: List[Sequence], results: List[Grid]):
return [(float(np.mean(x[0][self.low:self.high + 1])), ) for x in results[0]]
def axes(self):
return Axis.TRACKERS,
class MultiRunExperiment(Experiment):
repetitions = Integer(val_min=1, default=1)
early_stop = Boolean(default=True)
def _can_stop(self, tracker: Tracker, sequence: Sequence):
if not self.early_stop:
return False
trajectories = self.gather(tracker, sequence)
if len(trajectories) < 3:
return False
for trajectory in trajectories[1:]:
if not trajectory.equals(trajectories[0]):
return False
return True
def scan(self, tracker: Tracker, sequence: Sequence):
results = self.results(tracker, sequence)
files = []
complete = True
for i in range(1, self.repetitions+1):
name = "%s_%03d" % (sequence.name, i)
if Trajectory.exists(results, name):
files.extend(Trajectory.gather(results, name))
elif self._can_stop(tracker, sequence):
break
else:
complete = False
break
return complete, files, results
def gather(self, tracker: Tracker, sequence: Sequence):
trajectories = list()
results = self.results(tracker, sequence)
for i in range(1, self.repetitions+1):
name = "%s_%03d" % (sequence.name, i)
if Trajectory.exists(results, name):
trajectories.append(Trajectory.read(results, name))
return trajectories
class AverageAccuracy(SequenceAveragingAnalysis):
burnin = Integer(default=10, val_min=0)
ignore_unknown = Boolean(default=True)
bounded = Boolean(default=True)
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiRunExperiment)
@property
def name(self):
return "Average accurarcy per sequence"
def describe(self):
return Measure("Accuracy", "AUC", 0, 1, Sorting.DESCENDING),
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequence: Sequence):
if isinstance(experiment, MultiRunExperiment):
trajectories = experiment.gather(tracker, sequence)
if len(trajectories) == 0:
raise MissingResultsException()
cummulative = 0
for trajectory in trajectories:
accuracy, _ = compute_accuracy(trajectory.regions(), sequence,
self.burnin,
self.ignore_unknown,
self.bounded)
cummulative = cummulative + accuracy
return cummulative / len(trajectories),
def collapse(self, _: Tracker, sequences: List[Sequence],
results: List[tuple]):
accuracy = 0
frames = 0
for sequence, a in zip(sequences, results):
accuracy = accuracy + a[0] * len(sequence)
frames = frames + len(sequence)
return accuracy / frames,
class RealtimeConfig(Attributee):
grace = Integer(val_min=0, default=0)
fps = Float(val_min=0, default=20)
class AccuracyRobustnessMultiStart(SequenceAveragingAnalysis):
burnin = Integer(default=10, val_min=0)
grace = Integer(default=10, val_min=0)
bounded = Boolean(default=True)
threshold = Float(default=0.1, val_min=0, val_max=1)
@property
def title(self):
return "AR analysis"
def describe(self):
return Measure("Accuracy", "A", minimal=0, maximal=1, direction=Sorting.DESCENDING), \
Measure("Robustness", "R", minimal=0, direction=Sorting.DESCENDING), \
Point("AR plot", dimensions=2, abbreviation="AR",
minimal=(0, 0), maximal=(1, 1), labels=("Robustness", "Accuracy"), trait="ar"), \
None, None
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiStartExperiment)
def collapse(self, tracker: Tracker, sequences: List[Sequence],
results: List[tuple]):
total_accuracy = 0
total_robustness = 0
weight_accuracy = 0
weight_robustness = 0
for accuracy, robustness, _, accuracy_w, robustness_w in results:
total_accuracy += accuracy * accuracy_w
total_robustness += robustness * robustness_w
weight_accuracy += accuracy_w
weight_robustness += robustness_w
ar = (total_robustness / weight_robustness,
total_accuracy / weight_accuracy)
return total_accuracy / weight_accuracy, total_robustness / weight_robustness, ar, weight_accuracy, weight_robustness
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequence: Sequence):
results = experiment.results(tracker, sequence)
forward, backward = find_anchors(sequence, experiment.anchor)
if not forward and not backward:
raise RuntimeError("Sequence does not contain any anchors")
robustness = 0
accuracy = 0
total = 0
for i, reverse in [(f, False) for f in forward] + [(f, True)
for f in backward]:
name = "%s_%08d" % (sequence.name, i)
if not Trajectory.exists(results, name):
raise MissingResultsException()
if reverse:
proxy = FrameMapSequence(sequence,
list(reversed(range(0, i + 1))))
else:
proxy = FrameMapSequence(sequence,
list(range(i, sequence.length)))
trajectory = Trajectory.read(results, name)
overlaps = calculate_overlaps(
trajectory.regions(), proxy.groundtruth(),
(proxy.size) if self.burnin else None)
grace = self.grace
progress = len(proxy)
for j, overlap in enumerate(overlaps):
if overlap <= self.threshold and not proxy.groundtruth(
j).is_empty():
grace = grace - 1
if grace == 0:
progress = j + 1 - self.grace # subtract since we need actual point of the failure
break
else:
grace = self.grace
robustness += progress # simplified original equation: len(proxy) * (progress / len(proxy))
accuracy += sum(overlaps[0:progress])
total += len(proxy)
ar = (robustness / total,
accuracy / robustness if robustness > 0 else 0)
return accuracy / robustness if robustness > 0 else 0, robustness / total, ar, robustness, len(
sequence)
class AttributeMultiStart(SequenceAveragingAnalysis):
burnin = Integer(default=10, val_min=0)
grace = Integer(default=10, val_min=0)
bounded = Boolean(default=True)
threshold = Float(default=0.1, val_min=0, val_max=1)
tags = List(String())
@property
def name(self):
return "AR per-attribute analysis"
def describe(self):
accuracy = [
Measure("Accuracy: " + t,
"A " + t,
minimal=0,
maximal=1,
direction=Sorting.DESCENDING) for t in self.tags
]
robustness = [
Measure("Robutsness" + t,
"R " + t,
minimal=0,
maximal=1,
direction=Sorting.DESCENDING) for t in self.tags
]
length = [None] * len(self.tags)
return tuple(
functools.reduce(
operator.add,
[[a, r, n] for a, r, n in zip(accuracy, robustness, length)]))
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiStartExperiment)
def collapse(self, tracker: Tracker, sequences: typing.List[Sequence],
results: typing.List[tuple]):
accuracy = Counter()
robustness = Counter()
attribute_total = Counter()
for seq_acc, seq_rob, seq_attr_count in results:
for t in seq_attr_count:
accuracy[t] += (seq_acc[t]
if t in seq_acc else 0) * seq_attr_count[t]
robustness[t] += seq_rob * seq_attr_count[t]
attribute_total[t] += seq_attr_count[t]
accuracy = [accuracy[t] / attribute_total[t] for t in self.tags]
robustness = [robustness[t] / attribute_total[t] for t in self.tags]
length = [attribute_total[t] for t in self.tags]
return tuple(
functools.reduce(
operator.add,
[[a, r, n] for a, r, n in zip(accuracy, robustness, length)]))
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequence: Sequence):
results = experiment.results(tracker, sequence)
forward, backward = find_anchors(sequence, experiment.anchor)
if len(forward) == 0 and len(backward) == 0:
raise RuntimeError("Sequence does not contain any anchors")
accuracy_ = Counter()
tags_count_ = Counter()
robustness_ = 0
total_ = 0
for i, reverse in [(f, False) for f in forward] + [(f, True)
for f in backward]:
name = "%s_%08d" % (sequence.name, i)
if not Trajectory.exists(results, name):
raise MissingResultsException()
if reverse:
proxy = FrameMapSequence(sequence,
list(reversed(range(0, i + 1))))
else:
proxy = FrameMapSequence(sequence,
list(range(i, sequence.length)))
trajectory = Trajectory.read(results, name)
overlaps = calculate_overlaps(trajectory.regions(),
proxy.groundtruth(),
proxy.size if self.burnin else None)
grace = self.grace
progress = len(proxy)
for j, overlap in enumerate(overlaps):
if overlap <= self.threshold and not proxy.groundtruth(
j).is_empty():
grace = grace - 1
if grace == 0:
progress = j + 1 - self.grace # subtract since we need actual point of the failure
break
else:
grace = self.grace
for j in range(progress):
overlap = overlaps[j]
tags = proxy.tags(j)
if len(tags) == 0:
tags = ['empty']
for t in tags:
accuracy_[t] += overlap
tags_count_[t] += 1
robustness_ += progress
total_ += len(proxy)
seq_robustness = robustness_ / total_
seq_accuracy = {}
for t in accuracy_:
seq_accuracy[t] = accuracy_[t] / tags_count_[t]
# calculate weights for each attribute
attribute_counter = Counter()
for frame_idx in range(len(sequence)):
tags = sequence.tags(frame_idx)
if len(tags) == 0:
tags = ['empty']
for t in tags:
attribute_counter[t] += 1
return seq_accuracy, seq_robustness, attribute_counter
class AttributeDifficultyLevelMultiStart(SequenceAveragingAnalysis):
burnin = Integer(default=10, val_min=0)
grace = Integer(default=10, val_min=0)
bounded = Boolean(default=True)
threshold = Float(default=0.1, val_min=0, val_max=1)
fail_interval = Integer(default=30, val_min=1)
tags = List(String())
@property
def name(self):
return "Attribute difficulty"
def describe(self):
return tuple([
Measure(t, t, minimal=0, maximal=1, direction=Sorting.DESCENDING)
for t in self.tags
] + [None] * len(self.tags))
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiStartExperiment)
def collapse(self, tracker: Tracker, sequences: typing.List[Sequence],
results: typing.List[tuple]):
attribute_difficulty = Counter()
attribute_counter = Counter()
for seq_tags_not_failed, seq_tags_count, seq_attr_count in results:
for tag in seq_tags_count:
if tag in seq_tags_not_failed:
seq_attr_difficulty = seq_tags_not_failed[
tag] / seq_tags_count[tag]
else:
seq_attr_difficulty = 0
attribute_difficulty[
tag] += seq_attr_difficulty * seq_attr_count[tag]
attribute_counter[tag] += seq_attr_count[tag]
return tuple([
attribute_difficulty[tag] / attribute_counter[tag]
for tag in self.tags
] + [attribute_counter[tag] for tag in self.tags])
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequence: Sequence):
results = experiment.results(tracker, sequence)
forward, backward = find_anchors(sequence, experiment.anchor)
if len(forward) == 0 and len(backward) == 0:
raise RuntimeError("Sequence does not contain any anchors")
tags_count = Counter()
tags_not_failed = Counter()
for i, reverse in [(f, False) for f in forward] + [(f, True)
for f in backward]:
name = "%s_%08d" % (sequence.name, i)
if not Trajectory.exists(results, name):
raise MissingResultsException()
if reverse:
proxy = FrameMapSequence(sequence,
list(reversed(range(0, i + 1))))
else:
proxy = FrameMapSequence(sequence,
list(range(i, sequence.length)))
trajectory = Trajectory.read(results, name)
overlaps = calculate_overlaps(trajectory.regions(),
proxy.groundtruth(),
proxy.size if self.burnin else None)
grace = self.grace
progress = len(proxy)
for j, overlap in enumerate(overlaps):
if overlap <= self.threshold and not proxy.groundtruth(
j).is_empty():
grace = grace - 1
if grace == 0:
progress = j + 1 - self.grace # subtract since we need actual point of the failure
break
else:
grace = self.grace
for j in range(progress):
tags = proxy.tags(j)
if len(tags) == 0:
tags = ['empty']
for t in tags:
tags_count[t] += 1
if progress == len(
proxy) or j < progress - self.fail_interval:
tags_not_failed[t] += 1
attribute_counter = Counter()
for frame_idx in range(len(sequence)):
tags = sequence.tags(frame_idx)
if len(tags) == 0:
tags = ['empty']
for t in tags:
attribute_counter[t] += 1
return tags_not_failed, tags_count, attribute_counter
class EAOCurveMultiStart(SequenceAveragingAnalysis):
burnin = Integer(default=10, val_min=0)
grace = Integer(default=10, val_min=0)
bounded = Boolean(default=True)
threshold = Float(default=0.1, val_min=0, val_max=1)
low = Integer()
high = Integer()
@property
def title(self):
return "EAO Curve"
def describe(self):
return Plot("Expected average overlap", "EAO", minimal=0, maximal=1, wrt="frames", trait="eao"),
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiStartExperiment)
def collapse(self, tracker: Tracker, sequences: List[Sequence], results: Grid):
eao_curve = self.high * [float(0)]
eao_weights = self.high * [float(0)]
for (seq_eao_curve, eao_active), seq_w in results:
for i, (eao_, active_) in enumerate(zip(seq_eao_curve, eao_active)):
eao_curve[i] += eao_ * active_ * seq_w
eao_weights[i] += active_ * seq_w
return [eao_ / w_ if w_ > 0 else 0 for eao_, w_ in zip(eao_curve, eao_weights)],
def subcompute(self, experiment: Experiment, tracker: Tracker, sequence: Sequence):
results = experiment.results(tracker, sequence)
forward, backward = find_anchors(sequence, experiment.anchor)
if len(forward) == 0 and len(backward) == 0:
raise RuntimeError("Sequence does not contain any anchors")
overlaps_all = []
success_all = []
for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]:
name = "%s_%08d" % (sequence.name, i)
if not Trajectory.exists(results, name):
raise MissingResultsException()
if reverse:
proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1))))
else:
proxy = FrameMapSequence(sequence, list(range(i, sequence.length)))
trajectory = Trajectory.read(results, name)
overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None)
grace = self.grace
progress = len(proxy)
for j, overlap in enumerate(overlaps):
if overlap <= self.threshold and not proxy.groundtruth(j).is_empty():
grace = grace - 1
if grace == 0:
progress = j + 1 - self.grace # subtract since we need actual point of the failure
break
else:
grace = self.grace
success = True
if progress < len(overlaps):
# tracker has failed during this run
overlaps[progress:] = (len(overlaps) - progress) * [float(0)]
success = False
overlaps_all.append(overlaps)
success_all.append(success)
return compute_eao_partial(overlaps_all, success_all, self.high), 1
class InjectConfig(Attributee):
# Not implemented yet
placeholder = Integer(default=1)
class PrecisionRecallCurve(TrackerSeparableAnalysis):
resolution = Integer(default=100)
ignore_unknown = Boolean(default=True)
bounded = Boolean(default=True)
@property
def title(self):
return "Tracking precision/recall"
def describe(self):
return Curve("Precision Recall curve",
dimensions=2,
abbreviation="PR",
minimal=(0, 0),
maximal=(1, 1),
labels=("Recall", "Precision")), None
def compatible(self, experiment: Experiment):
return isinstance(experiment, UnsupervisedExperiment)
def subcompute(self, experiment: Experiment, tracker: Tracker,
sequences: List[Sequence]):
# calculate thresholds
total_scores = 0
for sequence in sequences:
trajectories = experiment.gather(tracker, sequence)
if len(trajectories) == 0:
raise MissingResultsException(
"Missing results for sequence {}".format(sequence.name))
for trajectory in trajectories:
total_scores += len(trajectory)
# allocate memory for all scores
scores_all = total_scores * [float(0)]
idx = 0
for sequence in sequences:
trajectories = experiment.gather(tracker, sequence)
for trajectory in trajectories:
conf_ = [
trajectory.properties(i).get('confidence', 0)
for i in range(len(trajectory))
]
scores_all[idx:idx + len(conf_)] = conf_
idx += len(conf_)
thresholds = determine_thresholds(scores_all, self.resolution)
# calculate per-sequence Precision and Recall curves
pr_curves = []
re_curves = []
for sequence in sequences:
trajectories = experiment.gather(tracker, sequence)
if len(trajectories) == 0:
raise MissingResultsException()
pr = len(thresholds) * [float(0)]
re = len(thresholds) * [float(0)]
for trajectory in trajectories:
conf_ = [
trajectory.properties(i).get('confidence', 0)
for i in range(len(trajectory))
]
pr_, re_ = compute_tpr_curves(trajectory.regions(), conf_,
sequence, thresholds,
self.ignore_unknown,
self.bounded)
pr = [p1 + p2 for p1, p2 in zip(pr, pr_)]
re = [r1 + r2 for r1, r2 in zip(re, re_)]
pr = [p1 / len(trajectories) for p1 in pr]
re = [r1 / len(trajectories) for r1 in re]
pr_curves.append(pr)
re_curves.append(re)
# calculate a single Precision, Recall and F-score curves for a given tracker
# average Pr-Re curves over the sequences
pr_curve = len(thresholds) * [float(0)]
re_curve = len(thresholds) * [float(0)]
for i, _ in enumerate(thresholds):
for j, _ in enumerate(pr_curves):
pr_curve[i] += pr_curves[j][i]
re_curve[i] += re_curves[j][i]
curve = [(re / len(pr_curves), pr / len(pr_curves))
for pr, re in zip(pr_curve, re_curve)]
return curve, thresholds
class EAOCurveMultiStart2(TrackerSeparableAnalysis):
burnin = Integer(default=10, val_min=0)
grace = Integer(default=10, val_min=0)
bounded = Boolean(default=True)
threshold = Float(default=0.1, val_min=0, val_max=1)
@property
def title(self):
return "EAO Curve"
def describe(self):
return Plot("Expected Average Overlap", "EAO", minimal=0, maximal=1),
def compatible(self, experiment: Experiment):
return isinstance(experiment, MultiStartExperiment)
def subcompute(self, experiment: Experiment, tracker: Tracker, sequences: List[Sequence]):
overlaps_all = []
weights_all = []
success_all = []
frames_total = 0
for sequence in sequences:
results = experiment.results(tracker, sequence)
forward, backward = find_anchors(sequence, experiment.anchor)
if len(forward) == 0 and len(backward) == 0:
raise RuntimeError("Sequence does not contain any anchors")
weights_per_run = []
for i, reverse in [(f, False) for f in forward] + [(f, True) for f in backward]:
name = "%s_%08d" % (sequence.name, i)
if not Trajectory.exists(results, name):
raise MissingResultsException()
if reverse:
proxy = FrameMapSequence(sequence, list(reversed(range(0, i + 1))))
else:
proxy = FrameMapSequence(sequence, list(range(i, sequence.length)))
trajectory = Trajectory.read(results, name)
overlaps = calculate_overlaps(trajectory.regions(), proxy.groundtruth(), proxy.size if self.burnin else None)
grace = self.grace
progress = len(proxy)
for j, overlap in enumerate(overlaps):
if overlap <= self.threshold and not proxy.groundtruth(j).is_empty():
grace = grace - 1
if grace == 0:
progress = j + 1 - self.grace # subtract since we need actual point of the failure
break
else:
grace = self.grace
success = True
if progress < len(overlaps):
# tracker has failed during this run
overlaps[progress:] = (len(overlaps) - progress) * [float(0)]
success = False
overlaps_all.append(overlaps)
success_all.append(success)
weights_per_run.append(len(proxy))
for w in weights_per_run:
weights_all.append((w / sum(weights_per_run)) * len(sequence))
frames_total += len(sequence)
weights_all = [w / frames_total for w in weights_all]
return compute_eao_curve(overlaps_all, weights_all, success_all),