def test(itr, dataset, args, model, logger, device):
done = False
instance_logits_stack = []
element_logits_stack = []
labels_stack = []
while not done:
if dataset.currenttestidx % 100 == 0:
print('Testing test data point %d of %d' %
(dataset.currenttestidx, len(dataset.testidx)))
features, labels, done = dataset.load_data(is_training=False)
features = torch.from_numpy(features).float().to(device)
with torch.no_grad():
_, element_logits = model(Variable(features),
is_training=False) # (1,T,20)
tmp = F.softmax(torch.mean(torch.topk(element_logits,
k=int(np.ceil(len(features) /
8)),
dim=0)[0],
dim=0),
dim=0).cpu().data.numpy()
element_logits = element_logits.cpu().data.numpy() # (1,T,20)
instance_logits_stack.append(tmp)
element_logits_stack.append(element_logits)
labels_stack.append(labels)
instance_logits_stack = np.array(instance_logits_stack)
labels_stack = np.array(labels_stack)
dmap, iou = dmAP(element_logits_stack, dataset.path_to_annotations, args)
if args.dataset_name == 'Thumos14':
test_set = sio.loadmat('test_set_meta.mat')['test_videos'][0]
for i in range(np.shape(labels_stack)[0]):
if test_set[i]['background_video'] == 'YES':
labels_stack[i, :] = np.zeros_like(labels_stack[i, :])
cmap = cmAP(instance_logits_stack, labels_stack) # (212,20), (212,20)
print('Classification map %f' % cmap)
print('Detection map @ %f = %f' % (iou[0], dmap[0]))
print('Detection map @ %f = %f' % (iou[1], dmap[1]))
print('Detection map @ %f = %f' % (iou[2], dmap[2]))
print('Detection map @ %f = %f' % (iou[3], dmap[3]))
print('Detection map @ %f = %f' % (iou[4], dmap[4]))
logger.log_value('Test Classification mAP', cmap, itr)
for item in list(zip(dmap, iou)):
logger.log_value('Test Detection mAP @ IoU = ' + str(item[1]), item[0],
itr)
utils.write_to_file(args.dataset_name, dmap, cmap, itr)
def test(itr, dataset, args, model, logger, device):
model.eval()
done = False
instance_logits_stack = []
element_logits_stack = []
labels_stack = []
print("TESTING")
while not done:
features, labels, done = dataset.load_data(is_training=False)
features = torch.from_numpy(features).float().to(device)
features = features.unsqueeze(0)
_, element_logits = model(features)
element_logits = element_logits.squeeze(0)
tmp = (F.softmax(
torch.mean(
torch.topk(element_logits,
k=int(np.ceil(len(features) / args.topk)),
dim=0)[0],
dim=0,
),
dim=0,
).cpu().data.numpy())
element_logits = element_logits.cpu().data.numpy()
instance_logits_stack.append(tmp)
element_logits_stack.append(element_logits)
labels_stack.append(labels)
instance_logits_stack = np.array(instance_logits_stack)
labels_stack = np.array(labels_stack)
iou = [0.1, 0.3, 0.5, 0.7]
dmap_detect = ANETdetection(dataset.path_to_annotations, iou, args=args)
dmap_detect._import_prediction(element_logits_stack)
dmap = dmap_detect.evaluate()
if args.dataset_name == "Thumos14":
test_set = sio.loadmat("test_set_meta.mat")["test_videos"][0]
for i in range(np.shape(labels_stack)[0]):
if test_set[i]["background_video"] == "YES":
labels_stack[i, :] = np.zeros_like(labels_stack[i, :])
cmap = cmAP(instance_logits_stack, labels_stack)
print("Classification map %f" % cmap)
for k in range(len(iou)):
print("Detection map @ %f = %f" % (iou[k], dmap[k] * 100))
return (dmap[-1] + dmap[-2])
def test(itr, dataset, args, model, logger, device, exp_name):
done = False
instance_logits_stack = []
element_logits_stack = []
labels_stack = []
while not done:
if dataset.currenttestidx % 100 == 0:
print('Testing test data point %d of %d' %
(dataset.currenttestidx, len(dataset.testidx)))
features, labels, done = dataset.load_data(is_training=False)
features = torch.from_numpy(features).float().to(device)
#print ('test: ', features.shape, labels.shape)
with torch.no_grad():
_, element_logits = model(Variable(features), is_training=False)
tmp = F.softmax(torch.mean(torch.topk(element_logits,
k=int(np.ceil(len(features) /
8)),
dim=0)[0],
dim=0),
dim=0).cpu().data.numpy()
element_logits = element_logits.cpu().data.numpy()
instance_logits_stack.append(tmp)
element_logits_stack.append(element_logits)
labels_stack.append(labels)
np.save('./ckpt/' + exp_name + '/%s_%06d_cas.npy' % (exp_name, itr),
np.array(element_logits_stack),
allow_pickle=True)
np.save('./ckpt/' + exp_name + '/%s_%06d_pmf.npy' % (exp_name, itr),
np.array(instance_logits_stack),
allow_pickle=True)
np.save('./ckpt/' + exp_name + '/%s_%06d_lab.npy' % (exp_name, itr),
np.array(labels_stack),
allow_pickle=True)
instance_logits_stack = np.array(instance_logits_stack)
labels_stack = np.array(labels_stack)
dmap, iou = dmAP(element_logits_stack, dataset.path_to_annotations, args)
cmap = cmAP(instance_logits_stack, labels_stack)
print('Classification map %f' % cmap)
print('Detection map @ %f = %f' % (iou[0], dmap[0]))
print('Detection map @ %f = %f' % (iou[1], dmap[1]))
print('Detection map @ %f = %f' % (iou[2], dmap[2]))
print('Detection map @ %f = %f' % (iou[3], dmap[3]))
print('Detection map @ %f = %f' % (iou[4], dmap[4]))
print('Detection map @ %f = %f' % (iou[5], dmap[5]))
print('Detection map @ %f = %f' % (iou[6], dmap[6]), flush=True)
#logger.scalar_summary('test/c_mAP', cmap, step=itr)
logger.add_scalar('test/c_mAP', cmap, global_step=itr)
for item in list(zip(dmap, iou)):
#logger.scalar_summary('test/d_mAP_at_%s'%str(item[1]), item[0], step=itr)
logger.add_scalar('test/d_mAP_at_%s' % str(item[1]),
item[0],
global_step=itr)
def evaluate(itr,
dataset,
model,
logger,
groundtruth_filename,
prediction_filename,
device=torch.device('cuda'),
background=True,
subset='test',
fps=25,
stride=16,
threshold_type='mean',
frame_type='max',
adjust_mean=1.0,
act_weight=1.0,
tiou_thresholds=np.linspace(0.1, 0.7, 7),
use_anchor=False):
'''
generate proposals and evaluate
'''
with open(groundtruth_filename, 'r') as fr:
gt_info = json.load(fr)['database']
save_dict = {'version': dataset.dataset_name, 'external_data': 'None'}
frame_dict = {'version': dataset.dataset_name, 'external_data': 'abc'}
rs = defaultdict(list)
rs2 = defaultdict(list)
instance_logits_stack = []
labels_stack = []
inds = dataset.get_testidx()
classlist = dataset.get_classlist()
tps = [0] * len(classlist)
aps = [0] * len(classlist)
res = [0] * len(classlist)
one_hots = np.eye(len(classlist))
for idx in inds:
feat = dataset.get_feature(idx)
vname = dataset.get_vname(idx)
duration = dataset.get_duration(idx)
feat = torch.from_numpy(np.expand_dims(feat,
axis=0)).float().to(device)
frame_label = dataset.get_gt_frame_label(idx)
video_label = dataset.get_video_label(idx)
if len(video_label) < 1:
continue
with torch.no_grad():
_, logits_f, _, logits_r, tcam, att_logits_f, att_logits_r, att_logits = model(
Variable(feat), device, is_training=False)
logits_f, logits_r, tcam = logits_f[0], logits_r[0], tcam[0]
topk = int(np.ceil(len(feat[0]) / 8))
tmp = F.softmax(torch.mean(torch.topk(logits_f, k=topk, dim=0)[0],
dim=0),
dim=0).cpu().data.numpy()
tmp += F.softmax(torch.mean(torch.topk(logits_r, k=topk, dim=0)[0],
dim=0),
dim=0).cpu().data.numpy()
tmp += F.softmax(torch.mean(torch.topk(tcam, k=topk, dim=0)[0], dim=0),
dim=0).cpu().data.numpy()
if background:
tcam = tcam[:, 1:]
tmp = tmp[1:]
instance_logits_stack.append(tmp)
labels_stack.append(np.sum(one_hots[video_label], axis=0))
tcam = tcam.cpu().data.numpy()
pred_label = np.argmax(tcam, axis=-1)
assert len(pred_label) == len(frame_label)
for gt in frame_label:
for g in gt:
res[g - 1] += 1
score = np.zeros((len(tcam), 1))
if use_anchor:
att = att_logits[0].cpu().data.numpy()
score = att.squeeze()
segments, frames = generate_segment(tcam,
score,
threshold_type=threshold_type,
frame_type=frame_type,
act_weight=act_weight,
adjust_mean=adjust_mean)
fps = gt_info[vname].get('fps', fps)
for frame in frames:
aps[frame[0]] += 1
if frame[0] in frame_label[frame[1]]:
tps[frame[0]] += 1
rs2[vname] += [{
'score': float(frame[2] / 100.0),
'label': classlist[frame[0]].decode('utf-8'),
'frame': float(frame[1] * stride / fps)
}]
for seg in segments:
rs[vname] += [{
'score':
float(seg[3] / 100.0),
'label':
str(classlist[seg[0]].decode('utf-8')),
'segment':
[float(seg[1] * stride / fps),
float(seg[2] * stride / fps)]
}]
save_dict['results'] = rs
frame_dict['results'] = rs2
with open(prediction_filename, 'w') as fw:
json.dump(save_dict, fw)
frame_detection = FrameDetection(groundtruth_filename,
frame_dict,
subset=subset,
verbose=True,
check_status=False)
frame_detection.evaluate()
anet_detection = ANETdetection(groundtruth_filename,
save_dict,
subset=subset,
tiou_thresholds=tiou_thresholds,
verbose=True,
check_status=False)
dmap = anet_detection.evaluate()
for i in range(len(dmap)):
logger.log_value('mAP/IoU@%s' % (str(tiou_thresholds[i])), dmap[i],
itr)
labels_stack = np.array(labels_stack)
instance_logits_stack = np.array(instance_logits_stack)
cmap = cmAP(instance_logits_stack, labels_stack)
print(cmap)
tp = np.sum(tps)
ap = np.sum(aps)
recall = np.sum(res)
print(
'All act frames %d, predict all frames : %d, right frames: %d, AP: %0.5f, Recall: %0.5f'
% (recall, ap, tp, tp / ap, tp / recall))
acc = dmap[-3]
return np.mean(acc)
def test(dataset, args, itr):
# Placeholders
feature_seq = tf.placeholder(tf.float32,
[1, args.max_seqlen, args.feature_size])
seq_len = tf.cast(
tf.reduce_sum(tf.sign(tf.reduce_max(tf.abs(feature_seq), axis=2)),
axis=1), tf.int32)
fseq = feature_seq[:, :tf.reduce_max(seq_len), :]
sgn = tf.sign(tf.reduce_sum(tf.abs(fseq), keep_dims=True, axis=2))
seq_len = tf.cast(
tf.reduce_sum(tf.sign(tf.reduce_max(tf.abs(fseq), axis=2)), axis=1),
tf.int32)
k = tf.cast(tf.ceil(tf.cast(seq_len, tf.float32) / 8), tf.int32)
with tf.device('/gpu:0'):
with tf.variable_scope('Fully_Connected', reuse=True):
fc_W = _variable_with_weight_decay(
'fc_w', [args.feature_size, args.feature_size], 0.0005)
fc_b = _variable_with_weight_decay('fc_b', [args.feature_size],
0.0000)
feature = tf.matmul(
fseq, tf.tile(tf.expand_dims(fc_W, 0), [1, 1, 1])) + fc_b
feature = tf.nn.relu(feature)
feature = tf.nn.dropout(feature, 1.0)
with tf.variable_scope('Attention', reuse=True) as an:
atn_W = _variable_with_weight_decay(
'atn_w', [args.feature_size, args.num_class], 0.0005)
atn_b = _variable_with_weight_decay('atn_b', [args.num_class],
0.0000)
temporal_logits = tf.matmul(
feature, tf.tile(tf.expand_dims(atn_W, 0), [1, 1, 1])) + atn_b
# Initialize everything
init = tf.global_variables_initializer()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(init)
saver = tf.train.Saver()
saver.restore(
sess, tf.train.latest_checkpoint('./ckpt/' + args.model_name + '/'))
# Test
element_logit_stack = []
instance_logit_stack = []
label_stack = []
done = False
while not done:
features, label, done = dataset.load_data(is_training=False)
element_logit = []
features = np.concatenate([
features,
np.zeros((args.max_seqlen - len(features) % args.max_seqlen,
args.feature_size))
],
axis=0)
for i in range(0, len(features), args.max_seqlen):
tmp = sess.run(temporal_logits,
feed_dict={
feature_seq:
np.expand_dims(features[i:i + args.max_seqlen],
axis=0)
})
if len(element_logit) == 0:
element_logit = np.squeeze(tmp)
else:
element_logit = np.concatenate(
[element_logit, np.squeeze(tmp)], axis=0)
element_logit = np.array(element_logit)
instance_logit = np.mean(np.sort(
element_logit,
axis=0)[::-1][:max(1, int(np.shape(element_logit)[0] / 8)), :],
axis=0)
instance_logit = np.exp(instance_logit)
instance_logit /= np.sum(instance_logit)
instance_logit_stack.append(instance_logit)
element_logit_stack.append(element_logit)
label_stack.append(label)
instance_logit_stack = np.array(instance_logit_stack)
label_stack = np.array(label_stack)
dmap, iou = dmAP(element_logit_stack, dataset.path_to_annotations)
if args.dataset_name == 'Thumos14':
test_set = sio.loadmat('test_set_meta.mat')['test_videos'][0]
for i in range(np.shape(label_stack)[0]):
if test_set[i]['background_video'] == 'YES':
label_stack[i, :] = np.zeros_like(label_stack[i, :])
cmap = cmAP(instance_logit_stack, label_stack)
np.save('package-1.npy', (instance_logit_stack, label_stack))
print('Classification map %f' % cmap)
print('Detection map @ %f = %f' % (iou[0], dmap[0]))
print('Detection map @ %f = %f' % (iou[1], dmap[1]))
print('Detection map @ %f = %f' % (iou[2], dmap[2]))
print('Detection map @ %f = %f' % (iou[3], dmap[3]))
print('Detection map @ %f = %f' % (iou[4], dmap[4]))
utils.write_to_file(args.dataset_name, dmap, cmap, itr)
def test(itr, dataset, args, model, logger, device):
done = False
# flow,rgb,tcam的综合分类结果
instance_logits_stack = [] #(2380,100)
# tcam的分类结果
tcam_stack = [] #(2380,T,100)
# label标签
labels_stack = [] #(2380,100)
while not done:
if dataset.currenttestidx % 100 ==0:
print('Testing test data point %d of %d' %(dataset.currenttestidx, len(dataset.testidx)))
features, labels, done = dataset.load_data(is_training=False)
seq_len = np.sum(np.max(np.abs(features), axis=2) > 0, axis=1)
features = torch.from_numpy(features).float().to(device)
with torch.no_grad():
_, logits_f, _, logits_r, tcam, _ = model(Variable(features), device, is_training=False, seq_len=torch.from_numpy(seq_len).to(device))
logits_f, logits_r, tcam = logits_f[0], logits_r[0], tcam[0]
topk = int(np.ceil(len(features[0])/8))
tmp = F.softmax(torch.mean(torch.topk(logits_f, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy()
tmp += F.softmax(torch.mean(torch.topk(logits_r, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy()
tmp += F.softmax(torch.mean(torch.topk(tcam, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy()
tcam = tcam.cpu().data.numpy()
instance_logits_stack.append(tmp)
tcam_stack.append(tcam)
labels_stack.append(labels)
instance_logits_stack = np.array(instance_logits_stack) #(video_num,20)
labels_stack = np.array(labels_stack) # (video_num,20)
if args.dataset_name.find('Thumos14')!= -1 and args.num_class == 101:
test_set = sio.loadmat('test_set_meta.mat')['test_videos'][0]
bg_vid = 0
for i in range(np.shape(labels_stack)[0]):
if test_set[i]['background_video'] == 'YES':
bg_vid += 1
labels_stack[i,:] = np.zeros_like(labels_stack[i,:])
cmap = cmAP(instance_logits_stack, labels_stack)
print(cmap)
# tcam_stack : (video_num,T,20), dataset.path_to_annotations: GT 标注 , None, None
dmap, iou = dmAP(tcam_stack, dataset.path_to_annotations, args.activity_net, valid_id=dataset.lst_valid)
print('Classification map %f' %cmap)
for k in range(len(dmap)):
print('Detection map @ %f = %f' %(iou[k], dmap[k]))
print('Mean Detection map = %f' %(np.mean(dmap)))
dmap += [np.mean(dmap)]
logger.log_value('Test Classification mAP', cmap, itr)
for item in list(zip(dmap,iou)):
logger.log_value('Test Detection mAP @ IoU = ' + str(item[1]), item[0], itr)
utils.write_to_file(args.dataset_name + args.model_name, dmap, cmap, itr)
if args.activity_net:
return dmap.pop()
else:
return dmap[4]
