def test_iou(self):
rects1 = np.random.rand(1000, 4) * 100
rects2 = np.random.rand(1000, 4) * 100
bound = (50, 100)
ious1 = rect_iou(rects1, rects2, bound=bound)
ious2 = poly_iou(rects1, rects2, bound=bound)
self.assertTrue((ious1 - ious2).max() < 1e-14)
polys1 = self._rect2corner(rects1)
polys2 = self._rect2corner(rects2)
ious3 = poly_iou(polys1, polys2, bound=bound)
self.assertTrue((ious1 - ious3).max() < 1e-14)
def anchor_ratio_performances(dataset, trackers, vot_un=False, disc=True):
ratios = []
mean_ratio = 0
performances = []
mean_performance = 0
video_perf = []
total_length = 0
video_ranks = []
if isinstance(dataset, ExperimentVOT):
for s, (img_files, anno, _) in enumerate(dataset.dataset):
seq_name = dataset.dataset.seq_names[s]
bound = Image.open(img_files[0]).size
if vot_un:
anchor_file = os.path.join(
dataset.result_dir,
"Ours",
"unsupervised",
seq_name,
"%s_anchor.pkl" % seq_name,
)
perf = get_performance(dataset, trackers[0],
"performance_un.json")
video_per = perf[
trackers[0]]["seq_wise"][seq_name]["success_score"]
succ_rank = [
perf[name]["seq_wise"][seq_name]["success_score"]
for name in trackers
]
succ_rank = len(succ_rank) + 1 - rankdata(succ_rank,
method="ordinal")
else:
anchor_file = os.path.join(
dataset.result_dir,
"Ours",
"baseline",
seq_name,
"%s_anchor.pkl" % seq_name,
)
perf = get_performance(dataset, trackers[0])
video_per = perf[
trackers[0]]["seq_wise"][seq_name]["success_score"]
succ_rank = [
perf[name]["seq_wise"][seq_name]["success_score"]
for name in trackers
]
succ_rank = len(succ_rank) + 1 - rankdata(succ_rank,
method="ordinal")
with open(anchor_file, "rb") as f:
anchors = pickle.load(f)
performance = 0
ratio = 0
for f, _ in enumerate(img_files):
if anchors[f] is not None and ~np.any(np.isnan(anno[f])):
iou = (poly_iou(np.array(anchors[f][-1]), anno[f],
bound)[0]
if len(anchors[f][-1]) > 1 else np.NaN)
performance += iou
ratio += 1
if ratio != 0:
mean_performance += performance
performance /= ratio
video_perf.append(video_per)
video_ranks.append(succ_rank[0])
performances.append(performance)
mean_ratio += ratio
total_length += len(img_files)
ratio /= len(img_files)
ratios.append(ratio)
else:
for s, (img_files, anno) in enumerate(dataset.dataset):
seq_name = dataset.dataset.seq_names[s]
bound = Image.open(img_files[0]).size
anchor_file = os.path.join(dataset.result_dir, trackers[0],
"anchor/%s_anchor.pkl" % seq_name)
with open(anchor_file, "rb") as f:
anchors = pickle.load(f)
perf = get_performance(dataset, trackers[0])
video_per = perf[
trackers[0]]["seq_wise"][seq_name]["success_score"]
succ_rank = [
perf[name]["seq_wise"][seq_name]["success_score"]
for name in trackers
]
succ_rank = len(succ_rank) + 1 - rankdata(succ_rank,
method="ordinal")
ratio = 0
performance = 0
for f, _ in enumerate(img_files):
if anchors[f] is not None and ~np.any(np.isnan(anno[f])):
iou = rect_iou(np.array(anchors[f][-1])[None, :],
anno[f][None, :],
bound=bound)[0]
performance += iou
ratio += 1
if ratio != 0:
mean_performance += performance
performance /= ratio
video_perf.append(video_per)
video_ranks.append(succ_rank[0])
performances.append(performance)
mean_ratio += ratio
total_length += len(img_files)
ratio /= len(img_files)
ratios.append(ratio)
mean_performance /= mean_ratio
mean_ratio /= total_length
return (
ratios,
mean_ratio,
performances,
mean_performance,
video_perf,
dataset.dataset.seq_names,
video_ranks,
)
def run_realtime(root, vot, ours, trackers, ex):
context = zmq.Context()
ventilators = [context.socket(zmq.PUSH) for _ in range(len(trackers))]
for ventilator, tracker in zip(ventilators, trackers):
ventilator.connect("tcp://%s:8888" % tracker)
sinks = [context.socket(zmq.PULL) for _ in range(len(trackers))]
for sink, tracker in zip(sinks, trackers):
sink.connect("tcp://%s:6006" % tracker)
for s, (img_files, anno, _) in enumerate(vot.dataset):
seq_name = vot.dataset.seq_names[s]
print("--Ex : %s, Sequence %d/%d: %s" %
(ex, s + 1, len(vot.dataset), seq_name))
record_files = [
os.path.join(vot.result_dir, tracker, "realtime", seq_name,
"%s_001.txt" % seq_name) for tracker in trackers
]
origin_files = [
os.path.join(
vot.result_dir,
tracker,
"baseline",
seq_name,
"%s_001_origin.txt" % seq_name,
) for tracker in trackers
]
record_file = os.path.join(vot.result_dir, ours.name, "realtime",
seq_name, "%s_001.txt" % seq_name)
results = dict()
results[ours.name] = {
"boxes": [],
"times": [],
"anchor_frames": [],
"weights": [],
}
for tracker in trackers:
results[tracker] = {
"boxes": [],
"origin_boxes": [],
"times": [],
"failure": False,
"next_start": 0,
"failed_frame": -1,
"total_time": 0.0,
"grace": 3 - 1,
"offset": 0,
"current": 0,
}
anno_rects = anno.copy()
if anno_rects.shape[1] == 8:
anno_rects = vot.dataset._corner2rect(anno_rects)
# tracking loop
for f, img_file in tqdm(enumerate(img_files)):
image = Image.open(img_file)
image = image.convert("RGB")
if f == 0:
data = {
"image": image,
"box": anno_rects[0, :],
"target": "all"
}
message = Message(MessageType["init"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
for i in range(len(trackers)):
results[trackers[i]]["boxes"].append([1])
results[trackers[i]]["origin_boxes"].append([1])
results[trackers[i]]["times"].append(0)
else:
data = {"image": image, "target": "all"}
message = Message(MessageType["track"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
for sink in sinks:
result = sink.recv_pyobj()
result = result.data
name = result["name"]
duration = result["time"]
box = result["box"]
results[name]["boxes"].append(box)
results[name]["origin_boxes"].append(box)
results[name]["times"].append(duration)
for i in range(len(trackers)):
key = trackers[i]
value = results[key]
box = value["boxes"][-1]
if f == value["next_start"]:
value["failure"] = False
value["failed_frame"] = -1
value["total_time"] = 0.0
value["grace"] = 3 - 1
value["offset"] = f
data = {
"image": image,
"box": anno_rects[f],
"target": key
}
message = Message(MessageType["init"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
value["boxes"][-1] = [1]
value["origin_boxes"][-1] = [1]
elif not value["failure"]:
# during success frames
# calculate current frame
if value["grace"] > 0:
value["total_time"] += 1000.0 / 25
value["grace"] -= 1
else:
value["total_time"] += max(
1000.0 / 25, value["times"][-2] * 1000.0)
value["current"] = value["offset"] + int(
np.round(
np.floor(value["total_time"] * 25) / 1000.0))
# delayed/tracked bounding box
if f < value["current"]:
value["boxes"][-1] = value["boxes"][-2]
value["origin_boxes"][-1] = value["origin_boxes"][
-2]
iou = poly_iou(anno[f],
value["boxes"][-1],
bound=image.size)
if iou <= 0.0:
# tracking failure
value["failure"] = True
value["failed_frame"] = f
value["next_start"] = value[
"current"] + vot.skip_initialize
value["boxes"][-1] = [2]
else:
# during failure frames
if f < value["current"]:
# skipping frame due to slow speed
value["boxes"][-1] = [0]
value["origin_boxes"][-1] = [0]
value["times"][-1] = np.NaN
elif f == value["current"]:
# current frame
iou = poly_iou(anno[f],
value["boxes"][-1],
bound=image.size)
if iou <= 0.0:
# tracking failure
value["boxes"][-1] = [2]
value["boxes"][value["failed_frame"]] = [0]
value["times"][value["failed_frame"]] = np.NaN
elif f < value["next_start"]:
# skipping frame due to failure
value["boxes"][-1] = [0]
value["origin_boxes"][-1] = [0]
value["times"][-1] = np.NaN
expert_boxes = [
results[trackers[i]]["origin_boxes"][-1]
for i in range(len(trackers))
]
if f == 0:
ours.init(image, anno_rects[0, :])
box = [1]
offline_results = None
duration = 0
else:
start_time = time.time()
box, offline_results = ours.update(image, expert_boxes)
duration = time.time() - start_time
results[ours.name]["boxes"].append(box)
results[ours.name]["times"].append(duration)
results[ours.name]["anchor_frames"].append(offline_results)
results[ours.name]["weights"].append(ours.algorithm.w)
if f != 0 and offline_results is not None and ours.reset_tracker:
data = {"image": image, "box": box, "target": "all"}
message = Message(MessageType["init"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
for i in range(len(trackers)):
vot._record(
record_files[i],
results[trackers[i]]["boxes"],
results[trackers[i]]["times"],
)
vot._record(record_file, results[ours.name]["boxes"],
results[ours.name]["times"])
for i in range(len(trackers)):
lines = []
for box in results[trackers[i]]["origin_boxes"]:
if len(box) == 1:
lines.append("%d" % box[0])
else:
lines.append(str.join(",", ["%.4f" % t for t in box]))
# record bounding boxes
record_dir = os.path.dirname(origin_files[i])
if not os.path.isdir(record_dir):
os.makedirs(record_dir)
with open(origin_files[i], "w") as f:
f.write(str.join("\n", lines))
anchor_file = record_file[:record_file.rfind("_")] + "_anchor.pkl"
with open(anchor_file, "wb") as fp:
pickle.dump(results[ours.name]["anchor_frames"], fp)
weights_file = record_file[:record_file.rfind("_")] + "_weights.txt"
np.savetxt(weights_file, results[ours.name]["weights"], fmt="%f")
def anchor_ratio_diff_performances(dataset, trackers, vot_un=False, disc=True):
# function for loading results
def read_record(filename):
with open(filename) as f:
record = f.read().strip().split("\n")
record = [[float(t) for t in line.split(",")] for line in record]
return record
ratios = []
mean_ratio = 0
performances = []
mean_performance = 0
diffs = []
mean_diff = 0
total_length = 0
if isinstance(dataset, ExperimentVOT):
for s, (img_files, anno, _) in enumerate(dataset.dataset):
seq_name = dataset.dataset.seq_names[s]
bound = Image.open(img_files[0]).size
if vot_un:
record_files = [
os.path.join(
dataset.result_dir,
name,
"unsupervised",
seq_name,
"%s_%03d.txt" % (seq_name, 1),
) for name in trackers
]
anchor_file = os.path.join(
dataset.result_dir,
"Ours",
"unsupervised",
seq_name,
"%s_anchor.pkl" % seq_name,
)
else:
record_files = [
os.path.join(
dataset.result_dir,
name,
"baseline",
seq_name,
"%s_%03d.txt" % (seq_name, 1),
) for name in trackers
]
anchor_file = os.path.join(
dataset.result_dir,
"Ours",
"baseline",
seq_name,
"%s_anchor.pkl" % seq_name,
)
boxes = [read_record(f) for f in record_files]
with open(anchor_file, "rb") as f:
anchors = pickle.load(f)
diff = 0
performance = 0
ratio = 0
for f, _ in enumerate(img_files):
if anchors[f] is not None and ~np.any(np.isnan(anno[f])):
iou = (poly_iou(np.array(anchors[f][-1]), anno[f],
bound)[0]
if len(anchors[f][-1]) > 1 else np.NaN)
best_iou = max([
poly_iou(np.array(box[f]), anno[f], bound)[0]
if len(box[f]) > 1 else np.NaN for box in boxes
])
if best_iou > iou + 0.01:
if disc:
diff += 1
else:
diff += best_iou - iou
performance += iou
ratio += 1
if ratio != 0:
mean_diff += diff
mean_performance += performance
diff /= ratio
performance /= ratio
diffs.append(diff)
performances.append(performance)
mean_ratio += ratio
total_length += len(img_files)
ratio /= len(img_files)
ratios.append(ratio)
else:
for s, (img_files, anno) in enumerate(dataset.dataset):
seq_name = dataset.dataset.seq_names[s]
bound = Image.open(img_files[0]).size
record_files = [
os.path.join(dataset.result_dir, name, "%s.txt" % seq_name)
for name in trackers
]
boxes = [
np.loadtxt(record_file, delimiter=",")
for record_file in record_files
]
anchor_file = os.path.join(dataset.result_dir, trackers[0],
"anchor/%s_anchor.pkl" % seq_name)
with open(anchor_file, "rb") as f:
anchors = pickle.load(f)
ratio = 0
performance = 0
diff = 0
for f, _ in enumerate(img_files):
if anchors[f] is not None and ~np.any(np.isnan(anno[f])):
iou = rect_iou(np.array(anchors[f][-1])[None, :],
anno[f][None, :],
bound=bound)[0]
best_iou = max([
rect_iou(np.array(box[f])[None, :],
anno[f][None, :],
bound=bound)[0] for box in boxes
])
if best_iou > iou + 0.01:
if disc:
diff += 1
else:
diff += best_iou - iou
performance += iou
ratio += 1
if ratio != 0:
mean_diff += diff
mean_performance += performance
diff /= ratio
performance /= ratio
diffs.append(diff)
performances.append(performance)
mean_ratio += ratio
total_length += len(img_files)
ratio /= len(img_files)
ratios.append(ratio)
mean_performance /= mean_ratio
mean_diff /= mean_ratio
mean_ratio /= total_length
return (
ratios,
mean_ratio,
diffs,
mean_diff,
performances,
mean_performance,
dataset.dataset.seq_names,
)
def run_supervised(root, vot, ours, trackers, ex):
context = zmq.Context()
ventilators = [context.socket(zmq.PUSH) for _ in range(len(trackers))]
for ventilator, tracker in zip(ventilators, trackers):
ventilator.connect("tcp://%s:8888" % tracker)
sinks = [context.socket(zmq.PULL) for _ in range(len(trackers))]
for sink, tracker in zip(sinks, trackers):
sink.connect("tcp://%s:6006" % tracker)
for s, (img_files, anno, _) in enumerate(vot.dataset):
seq_name = vot.dataset.seq_names[s]
print("--Ex : %s, Sequence %d/%d: %s" %
(ex, s + 1, len(vot.dataset), seq_name))
anno_rects = anno.copy()
if anno_rects.shape[1] == 8:
anno_rects = vot.dataset._corner2rect(anno_rects)
for r in tqdm(range(vot.repetitions)):
record_files = [
os.path.join(
vot.result_dir,
tracker,
"baseline",
seq_name,
"%s_%03d.txt" % (seq_name, r + 1),
) for tracker in trackers
]
origin_files = [
os.path.join(
vot.result_dir,
tracker,
"baseline",
seq_name,
"%s_%03d_origin.txt" % (seq_name, r + 1),
) for tracker in trackers
]
record_file = os.path.join(
vot.result_dir,
ours.name,
"baseline",
seq_name,
"%s_%03d.txt" % (seq_name, r + 1),
)
results = dict()
results[ours.name] = {
"boxes": [],
"times": [],
"anchor_frames": [],
"weights": [],
"failure": False,
"next_start": -1,
}
for tracker in trackers:
results[tracker] = {
"boxes": [],
"origin_boxes": [],
"times": [],
"failure": False,
"next_start": -1,
}
# tracking loop
for f, img_file in tqdm(enumerate(img_files)):
image = Image.open(img_file)
image = image.convert("RGB")
if f == 0:
data = {
"image": image,
"box": anno_rects[0, :],
"target": "all"
}
message = Message(MessageType["init"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
for i in range(len(trackers)):
results[trackers[i]]["boxes"].append([1])
results[trackers[i]]["origin_boxes"].append([1])
results[trackers[i]]["times"].append(0)
else:
data = {"image": image, "target": "all"}
message = Message(MessageType["track"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
for sink in sinks:
result = sink.recv_pyobj()
result = result.data
name = result["name"]
duration = result["time"]
box = result["box"]
results[name]["boxes"].append(box)
results[name]["origin_boxes"].append(box)
results[name]["times"].append(duration)
for i in range(len(trackers)):
key = trackers[i]
value = results[key]
box = value["boxes"][-1]
if value["failure"]:
# during failure frames
if f == value["next_start"]:
value["failure"] = False
data = {
"image": image,
"box": anno_rects[f],
"target": key,
}
message = Message(MessageType["init"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
value["boxes"][-1] = [1]
# value['origin_boxes'][-1] = [1]
else:
results[name]["times"][-1] = np.NaN
value["boxes"][-1] = [0]
# value['origin_boxes'][-1] = [0]
else:
# during success frames
iou = poly_iou(anno[f], box, bound=image.size)
if iou <= 0.0:
# tracking failure
value["failure"] = True
value["next_start"] = f + vot.skip_initialize
value["boxes"][-1] = [2]
expert_boxes = [
results[trackers[i]]["origin_boxes"][f]
for i in range(len(trackers))
]
if f == 0:
ours.init(image, anno_rects[0, :])
box = [1]
offline_results = None
duration = 0
else:
if results[ours.name]["failure"]:
if f == results[ours.name]["next_start"]:
results[ours.name]["failure"] = False
box = [1]
duration = 0
ours.init(image, anno_rects[f])
else:
duration = np.NaN
box = [0]
offline_results = None
else:
start_time = time.time()
box, offline_results = ours.update(image, expert_boxes)
duration = time.time() - start_time
iou = poly_iou(anno[f], box, bound=image.size)
if iou <= 0.0:
results[ours.name]["failure"] = True
results[ours.name][
"next_start"] = f + vot.skip_initialize
box = [2]
results[ours.name]["boxes"].append(box)
results[ours.name]["times"].append(duration)
results[ours.name]["anchor_frames"].append(offline_results)
results[ours.name]["weights"].append(ours.algorithm.w)
if f != 0 and offline_results is not None and ours.reset_tracker:
data = {"image": image, "box": box, "target": "all"}
message = Message(MessageType["init"], data)
for ventilator in ventilators:
ventilator.send_pyobj(message)
for i in range(len(trackers)):
vot._record(
record_files[i],
results[trackers[i]]["boxes"],
results[trackers[i]]["times"],
)
vot._record(record_file, results[ours.name]["boxes"],
results[ours.name]["times"])
for i in range(len(trackers)):
lines = []
for box in results[trackers[i]]["origin_boxes"]:
if len(box) == 1:
lines.append("%d" % box[0])
else:
lines.append(str.join(",", ["%.4f" % t for t in box]))
# record bounding boxes
record_dir = os.path.dirname(origin_files[i])
if not os.path.isdir(record_dir):
os.makedirs(record_dir)
with open(origin_files[i], "w") as f:
f.write(str.join("\n", lines))
anchor_file = record_file[:record_file.rfind("_")] + "_anchor.pkl"
with open(anchor_file, "wb") as fp:
pickle.dump(results[ours.name]["anchor_frames"], fp)
weights_file = record_file[:record_file.rfind("_"
)] + "_weights.txt"
np.savetxt(weights_file, results[ours.name]["weights"], fmt="%f")