def frameMABO(dname, redo=False):
d = GetDataset(dname)
dirname = os.path.join( os.path.dirname(__file__), '../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "frameMABO.pkl")
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
BO = pickle.load(fid)
else:
vlist = d.test_vlist()
BO = {l: [] for l in d.labels} # best overlap
for v in vlist:
gt = d.gttubes(v)
h, w = d.resolution(v)
# load per-frame detections
vdets = {i: np.empty((0,4), dtype=np.float32) for i in range(1, 1+d.nframes(v))}
# load results for each chunk
for i in xrange(1, 1 + d.nframes(v) - K + 1):
resname = os.path.join(dirname, d.frame_format(v,i) + '.pkl')
if not os.path.isfile(resname):
print("ERROR: Missing extracted tubelets " + resname)
sys.exit()
with open(resname, 'rb') as fid:
dets, _ = pickle.load(fid)
for k in xrange(K):
vdets[i+k] = np.concatenate((vdets[i + k], dets[:, 2+4*k:6+4*k]), axis=0)
# for each frame
for i in xrange(1, 1 + d.nframes(v)):
for ilabel in gt:
label = d.labels[ilabel]
for t in gt[ilabel]:
# the gt tube does not cover frame i
if not i in t[:,0]:
continue
gtbox = t[t[:,0] == i, 1:5] # box of gt tube at frame i
if vdets[i].size == 0: # we missed it
BO[label].append(0)
continue
ious = iou2d(vdets[i], gtbox)
BO[label].append( np.max(ious) )
# save file
with open(eval_file, 'wb') as fid:
pickle.dump( BO, fid)
# print MABO results
ABO = {la: 100 * np.mean(np.array(BO[la])) for la in d.labels} # average best overlap
for la in d.labels:
print("{:20s} {:6.2f}".format(la, ABO[la]))
print("{:20s} {:6.2f}".format("MABO", np.mean(np.array(ABO.values()))))
def videoAP(dname, th=0.5, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__),
'../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "videoAP{:g}.pkl".format(th))
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
res = pickle.load(fid)
else:
vlist = d.test_vlist()
# load detections
# alldets = for each label in 1..nlabels, list of tuple (v,score,tube as Kx5 array)
alldets = {ilabel: [] for ilabel in range(d.nlabels)}
for v in vlist:
tubename = os.path.join(dirname, v + '_tubes.pkl')
if not os.path.isfile(tubename):
print("ERROR: Missing extracted tubes " + tubename)
sys.exit()
with open(tubename, 'rb') as fid:
tubes = pickle.load(fid)
for ilabel in range(d.nlabels):
ltubes = tubes[ilabel]
idx = nms3dt(ltubes, 0.3)
alldets[ilabel] += [(v, ltubes[i][1], ltubes[i][0])
for i in idx]
# compute AP for each class
res = {}
for ilabel in range(d.nlabels):
detections = alldets[ilabel]
# load ground-truth
gt = {}
for v in vlist:
tubes = d.gttubes(v)
if not ilabel in tubes:
continue
gt[v] = tubes[ilabel]
if len(gt[v]) == 0:
del gt[v]
# precision,recall
pr = np.empty((len(detections) + 1, 2), dtype=np.float32)
pr[0, 0] = 1.0
pr[0, 1] = 0.0
fn = sum([len(g) for g in gt.values()]) # false negatives
fp = 0 # false positives
tp = 0 # true positives
for i, j in enumerate(
np.argsort(-np.array([dd[1] for dd in detections]))):
v, score, tube = detections[j]
ispositive = False
if v in gt:
ious = [iou3dt(g, tube) for g in gt[v]]
amax = np.argmax(ious)
if ious[amax] >= th:
ispositive = True
del gt[v][amax]
if len(gt[v]) == 0:
del gt[v]
if ispositive:
tp += 1
fn -= 1
else:
fp += 1
pr[i + 1, 0] = float(tp) / float(tp + fp)
pr[i + 1, 1] = float(tp) / float(tp + fn)
res[d.labels[ilabel]] = pr
# save results
with open(eval_file, 'wb') as fid:
pickle.dump(res, fid)
# display results
ap = 100 * np.array([pr_to_ap(res[label]) for label in d.labels])
print("frameAP")
for il, _ in enumerate(d.labels):
print("{:20s} {:8.2f}".format('', ap[il]))
print("{:20s} {:8.2f}".format("mAP", np.mean(ap)))
print("")
def frameCLASSIF(dname, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__),
'../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "frameCLASSIF.pkl")
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
CLASSIF = pickle.load(fid)
else:
vlist = d.test_vlist()
CORRECT = [0 for ilabel in range(d.nlabels)]
TOTAL = [0 for ilabel in range(d.nlabels)]
for v in vlist:
nframes = d.nframes(v)
# load all tubelets
VDets = {}
for startframe in range(1, nframes + 2 - K):
resname = os.path.join(dirname,
d.frame_format(v, startframe) + '.pkl')
if not os.path.isfile(resname):
print("ERROR: Missing extracted tubelets " + resname)
sys.exit()
with open(resname, 'rb') as fid:
_, VDets[startframe] = pickle.load(fid)
# iterate over ground-truth
tubes = d.gttubes(v)
for ilabel in tubes:
for g in tubes[ilabel]:
for i in range(g.shape[0]):
frame = int(g[i, 0])
# just in case a tube is longer than the video
if frame > nframes:
continue
gtbox = g[i, 1:5]
scores = np.zeros((d.nlabels, ), dtype=np.float32)
# average the score over the 6 frames
for sf in range(max(1, frame - K + 1),
min(nframes - K + 1, frame) + 1):
overlaps = iou2d(
VDets[sf][:, 4 * (frame - sf):4 *
(frame - sf) + 4], gtbox)
scores += np.sum(VDets[sf][overlaps >= 0.7,
4 * K + 1:],
axis=0)
# check classif
if np.argmax(scores) == ilabel:
CORRECT[ilabel] += 1
TOTAL[ilabel] += 1
CLASSIF = [
float(CORRECT[ilabel]) / float(TOTAL[ilabel])
for ilabel in range(d.nlabels)
]
with open(eval_file, 'wb') as fid:
pickle.dump(CLASSIF, fid)
# print classif results
for il, la in enumerate(d.labels):
print("{:20s} {:6.2f}".format(la, 100 * CLASSIF[il]))
print("{:20s} {:6.2f}".format("CLASSIF", 100 * np.mean(np.array(CLASSIF))))
def frameMABO(dname, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__),
'../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "frameMABO.pkl")
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
BO = pickle.load(fid)
else:
vlist = d.test_vlist()
BO = {l: [] for l in d.labels} # best overlap
for v in vlist:
gt = d.gttubes(v)
h, w = d.resolution(v)
# load per-frame detections
vdets = {
i: np.empty((0, 4), dtype=np.float32)
for i in range(1, 1 + d.nframes(v))
}
# load results for each chunk
for i in range(1, 1 + d.nframes(v) - K + 1):
resname = os.path.join(dirname, d.frame_format(v, i) + '.pkl')
if not os.path.isfile(resname):
print("ERROR: Missing extracted tubelets " + resname)
sys.exit()
with open(resname, 'rb') as fid:
dets, _ = pickle.load(fid)
for k in range(K):
vdets[i + k] = np.concatenate(
(vdets[i + k], dets[:, 2 + 4 * k:6 + 4 * k]), axis=0)
# for each frame
for i in range(1, 1 + d.nframes(v)):
for ilabel in gt:
label = d.labels[ilabel]
for t in gt[ilabel]:
# the gt tube does not cover frame i
if not i in t[:, 0]:
continue
gtbox = t[t[:, 0] == i,
1:5] # box of gt tube at frame i
if vdets[i].size == 0: # we missed it
BO[label].append(0)
continue
ious = iou2d(vdets[i], gtbox)
BO[label].append(np.max(ious))
# save file
with open(eval_file, 'wb') as fid:
pickle.dump(BO, fid)
# print MABO results
ABO = {la: 100 * np.mean(np.array(BO[la]))
for la in d.labels} # average best overlap
for la in d.labels:
print("{:20s} {:6.2f}".format(la, ABO[la]))
print("{:20s} {:6.2f}".format("MABO", np.mean(np.array(ABO.values()))))
def frameAP_error(dname, th=0.5, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__),
'../results/ACT-detector/', dname)
eval_file = os.path.join(dirname,
"frameAP{:g}ErrorAnalysis.pkl".format(th))
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
res = pickle.load(fid)
else:
vlist = d.test_vlist()
# load per-frame detections
alldets = load_frame_detections(d, vlist, dirname, 0.3)
res = {}
# compute AP for each class
for ilabel, label in enumerate(d.labels):
# detections of this class
detections = alldets[alldets[:, 2] == ilabel, :]
gt = {}
othergt = {}
labellist = {}
for iv, v in enumerate(vlist):
tubes = d.gttubes(v)
labellist[v] = tubes.keys()
for il in tubes:
for tube in tubes[il]:
for i in range(tube.shape[0]):
k = (iv, int(tube[i, 0]))
if il == ilabel:
if k not in gt:
gt[k] = []
gt[k].append(tube[i, 1:5].tolist())
else:
if k not in othergt:
othergt[k] = []
othergt[k].append(tube[i, 1:5].tolist())
for k in gt:
gt[k] = np.array(gt[k])
for k in othergt:
othergt[k] = np.array(othergt[k])
dupgt = deepcopy(gt)
# pr will be an array containing precision-recall values and 4 types of errors:
# localization, classification, timing, others
pr = np.empty((detections.shape[0] + 1, 6),
dtype=np.float32) # precision, recall
pr[0, 0] = 1.0
pr[0, 1:] = 0.0
fn = sum([g.shape[0] for g in gt.values()]) # false negatives
fp = 0 # false positives
tp = 0 # true positives
EL = 0 # localization errors
EC = 0 # classification error: overlap >=0.5 with an another object
EO = 0 # other errors
ET = 0 # timing error: the video contains the action but not at this frame
for i, j in enumerate(np.argsort(-detections[:, 3])):
k = (int(detections[j, 0]), int(detections[j, 1]))
box = detections[j, 4:8]
ispositive = False
if k in dupgt:
if k in gt:
ious = iou2d(gt[k], box)
amax = np.argmax(ious)
if k in gt and ious[amax] >= th:
ispositive = True
gt[k] = np.delete(gt[k], amax, 0)
if gt[k].size == 0:
del gt[k]
else:
EL += 1
elif k in othergt:
ious = iou2d(othergt[k], box)
if np.max(ious) >= th:
EC += 1
else:
EO += 1
elif ilabel in labellist[k[0]]:
ET += 1
else:
EO += 1
if ispositive:
tp += 1
fn -= 1
else:
fp += 1
pr[i + 1, 0] = float(tp) / float(tp + fp)
pr[i + 1, 1] = float(tp) / float(tp + fn)
pr[i + 1, 2] = float(EL) / float(tp + fp)
pr[i + 1, 3] = float(EC) / float(tp + fp)
pr[i + 1, 4] = float(ET) / float(tp + fp)
pr[i + 1, 5] = float(EO) / float(tp + fp)
res[label] = pr
# save results
with open(eval_file, 'wb') as fid:
pickle.dump(res, fid)
# display results
AP = 100 * np.array(
[pr_to_ap(res[label][:, [0, 1]]) for label in d.labels])
othersap = [
100 * np.array([pr_to_ap(res[label][:, [j, 1]]) for label in d.labels])
for j in range(2, 6)
]
EL = othersap[0]
EC = othersap[1]
ET = othersap[2]
EO = othersap[3]
EM = 100 - 100 * np.array([res[label][-1, 1] for label in d.labels
]) # missed detections = 1 - recall
LIST = [AP, EL, EC, ET, EO, EM]
print("Error Analysis")
print("")
print("{:20s} {:8s} {:8s} {:8s} {:8s} {:8s} {:8s}".format(
'label', ' AP ', ' Loc. ', ' Cls. ', ' Time ', ' Other ',
' missed '))
print("")
for il, label in enumerate(d.labels):
print("{:20s} ".format(label) +
" ".join(["{:8.2f}".format(L[il]) for L in LIST]))
print("")
print("{:20s} ".format("mean") +
" ".join(["{:8.2f}".format(np.mean(L)) for L in LIST]))
print("")
def frameAP(dname, th=0.5, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__),
'../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "frameAP{:g}.pkl".format(th))
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
res = pickle.load(fid)
else:
vlist = d.test_vlist()
# load per-frame detections
alldets = load_frame_detections(d, vlist, dirname, 0.3)
res = {}
# compute AP for each class
for ilabel, label in enumerate(d.labels):
# detections of this class
detections = alldets[alldets[:, 2] == ilabel, :]
# load ground-truth of this class
gt = {}
for iv, v in enumerate(vlist):
tubes = d.gttubes(v)
if not ilabel in tubes:
continue
for tube in tubes[ilabel]:
for i in range(tube.shape[0]):
k = (iv, int(tube[i, 0]))
if not k in gt:
gt[k] = []
gt[k].append(tube[i, 1:5].tolist())
for k in gt:
gt[k] = np.array(gt[k])
# pr will be an array containing precision-recall values
pr = np.empty((detections.shape[0] + 1, 2),
dtype=np.float32) # precision,recall
pr[0, 0] = 1.0
pr[0, 1] = 0.0
fn = sum([g.shape[0] for g in gt.values()]) # false negatives
fp = 0 # false positives
tp = 0 # true positives
for i, j in enumerate(np.argsort(-detections[:, 3])):
k = (int(detections[j, 0]), int(detections[j, 1]))
box = detections[j, 4:8]
ispositive = False
if k in gt:
ious = iou2d(gt[k], box)
amax = np.argmax(ious)
if ious[amax] >= th:
ispositive = True
gt[k] = np.delete(gt[k], amax, 0)
if gt[k].size == 0:
del gt[k]
if ispositive:
tp += 1
fn -= 1
else:
fp += 1
pr[i + 1, 0] = float(tp) / float(tp + fp)
pr[i + 1, 1] = float(tp) / float(tp + fn)
res[label] = pr
# save results
with open(eval_file, 'wb') as fid:
pickle.dump(res, fid)
# display results
ap = 100 * np.array([pr_to_ap(res[label]) for label in d.labels])
print("frameAP")
for il, _ in enumerate(d.labels):
print("{:20s} {:8.2f}".format('', ap[il]))
print("{:20s} {:8.2f}".format("mAP", np.mean(ap)))
print("")
def videoAP(dname, th=0.5, redo=False):
d = GetDataset(dname)
dirname = os.path.join( os.path.dirname(__file__), '../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "videoAP{:g}.pkl".format(th))
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
res = pickle.load(fid)
else:
vlist = d.test_vlist()
# load detections
# alldets = for each label in 1..nlabels, list of tuple (v,score,tube as Kx5 array)
alldets = {ilabel: [] for ilabel in xrange(d.nlabels)}
for v in vlist:
tubename = os.path.join(dirname, v + '_tubes.pkl')
if not os.path.isfile(tubename):
print("ERROR: Missing extracted tubes " + tubename)
sys.exit()
with open(tubename, 'rb') as fid:
tubes = pickle.load(fid)
for ilabel in xrange(d.nlabels):
ltubes = tubes[ilabel]
idx = nms3dt(ltubes, 0.3)
alldets[ilabel] += [(v,ltubes[i][1], ltubes[i][0]) for i in idx]
# compute AP for each class
res = {}
for ilabel in xrange(d.nlabels):
detections = alldets[ilabel]
# load ground-truth
gt = {}
for v in vlist:
tubes = d.gttubes(v)
if not ilabel in tubes:
continue
gt[v] = tubes[ilabel]
if len(gt[v])==0:
del gt[v]
# precision,recall
pr = np.empty((len(detections) + 1, 2), dtype=np.float32)
pr[0,0] = 1.0
pr[0,1] = 0.0
fn = sum([ len(g) for g in gt.values()]) # false negatives
fp = 0 # false positives
tp = 0 # true positives
for i, j in enumerate( np.argsort(-np.array([dd[1] for dd in detections]))):
v, score, tube = detections[j]
ispositive = False
if v in gt:
ious = [iou3dt(g, tube) for g in gt[v]]
amax = np.argmax(ious)
if ious[amax] >= th:
ispositive = True
del gt[v][amax]
if len(gt[v]) == 0:
del gt[v]
if ispositive:
tp += 1
fn -= 1
else:
fp += 1
pr[i+1,0] = float(tp) / float(tp + fp)
pr[i+1,1] = float(tp) / float(tp + fn)
res[d.labels[ilabel]] = pr
# save results
with open(eval_file, 'wb') as fid:
pickle.dump(res, fid)
# display results
ap = 100 * np.array([pr_to_ap(res[label]) for label in d.labels])
print "frameAP"
for il, _ in enumerate(d.labels):
print("{:20s} {:8.2f}".format('', ap[il]))
print("{:20s} {:8.2f}".format("mAP", np.mean(ap)))
print("")
def frameCLASSIF(dname, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__), '../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "frameCLASSIF.pkl")
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
CLASSIF = pickle.load(fid)
else:
vlist = d.test_vlist()
#print(vlist)
CORRECT = [0 for ilabel in xrange(d.nlabels)]
TOTAL = [0 for ilabel in xrange(d.nlabels)]
for v in vlist:
nframes = d.nframes(v)
# load all tubelets
VDets = {}
for startframe in xrange(1, nframes + 2 - K):
resname = os.path.join(dirname, d.frame_format(v, startframe) + '.pkl')
if not os.path.isfile(resname):
print("ERROR: Missing extracted tubelets " + resname)
sys.exit()
with open(resname, 'rb') as fid:
_, VDets[startframe] = pickle.load(fid)
# iterate over ground-truth
tubes = d.gttubes(v)
for ilabel in tubes:
for g in tubes[ilabel]:
for i in xrange(g.shape[0]):
frame = int(g[i, 0])
# just in case a tube is longer than the video
if frame > nframes:
continue
gtbox = g[i, 1:5]
scores = np.zeros((d.nlabels,), dtype=np.float32)
# average the score over the 6 frames
for sf in xrange(max(1, frame - K + 1), min(nframes - K + 1, frame) + 1):
overlaps = iou2d(VDets[sf][:, 4*(frame-sf):4*(frame-sf)+4], gtbox)
scores += np.sum(VDets[sf][overlaps >= 0.7, 4*K + 1:],axis=0)
# check classif
if np.argmax(scores) == ilabel:
CORRECT[ilabel] += 1
TOTAL[ilabel] += 1
print(TOTAL)
print(CORRECT)
CLASSIF = [float(CORRECT[ilabel]) / float(TOTAL[ilabel]) for ilabel in xrange(d.nlabels) if TOTAL[ilabel] != 0 ]
with open(eval_file, 'wb') as fid:
pickle.dump(CLASSIF, fid)
# print classif results
for il, la in enumerate(d.labels):
print("{:20s} {:6.2f}".format(la, 100*CLASSIF[il]))
print("{:20s} {:6.2f}".format("CLASSIF", 100*np.mean(np.array(CLASSIF))))
def frameAP(dname, th=0.5, redo=False):
d = GetDataset(dname)
dirname = os.path.join(os.path.dirname(__file__), '../results/ACT-detector/', dname)
eval_file = os.path.join(dirname, "frameAP{:g}.pkl".format(th))
if os.path.isfile(eval_file) and not redo:
with open(eval_file, 'rb') as fid:
res = pickle.load(fid)
else:
vlist = d.test_vlist()
print(vlist)
# load per-frame detections
alldets = load_frame_detections(d, vlist, dirname, 0.3)
res = {}
# compute AP for each class
for ilabel,label in enumerate(d.labels):
# detections of this class
detections = alldets[alldets[:, 2] == ilabel, :]
# load ground-truth of this class
gt = {}
for iv, v in enumerate(vlist):
tubes = d.gttubes(v)
if not ilabel in tubes:
continue
for tube in tubes[ilabel]:
for i in xrange(tube.shape[0]):
k = (iv, int(tube[i, 0]))
if not k in gt:
gt[k] = []
gt[k].append(tube[i, 1:5].tolist())
for k in gt:
gt[k] = np.array( gt[k] )
# pr will be an array containing precision-recall values
pr = np.empty((detections.shape[0] + 1, 2), dtype=np.float32)# precision,recall
pr[0, 0] = 1.0
pr[0, 1] = 0.0
fn = sum([g.shape[0] for g in gt.values()]) # false negatives
fp = 0 # false positives
tp = 0 # true positives
for i, j in enumerate(np.argsort(-detections[:,3])):
k = (int(detections[j,0]), int(detections[j,1]))
box = detections[j, 4:8]
ispositive = False
if k in gt:
ious = iou2d(gt[k], box)
amax = np.argmax(ious)
if ious[amax] >= th:
ispositive = True
gt[k] = np.delete(gt[k], amax, 0)
if gt[k].size == 0:
del gt[k]
if ispositive:
tp += 1
fn -= 1
else:
fp += 1
if((tp+fn)>0):
pr[i+1, 0] = float(tp) / float(tp + fp)
pr[i+1, 1] = float(tp) / float(tp + fn)
else:
pr[i+1, 0] = 0
pr[i+1, 1] = 0
res[label] = pr
# save results
with open(eval_file, 'wb') as fid:
pickle.dump(res, fid)
# display results
ap = 100*np.array([pr_to_ap(res[label]) for label in d.labels])
print "frameAP"
for il, _ in enumerate(d.labels):
print("{:20s} {:8.2f}".format('', ap[il]))
print("{:20s} {:8.2f}".format("mAP", np.mean(ap)))
print("")