def test(itr, dataset, args, model, logger, device): done = False instance_logits_stack = [] element_logits_stack = [] labels_stack = [] while not done: if dataset.currenttestidx % 100 == 0: print('Testing test data point %d of %d' % (dataset.currenttestidx, len(dataset.testidx))) features, labels, done = dataset.load_data(is_training=False) features = torch.from_numpy(features).float().to(device) with torch.no_grad(): _, element_logits = model(Variable(features), is_training=False) # (1,T,20) tmp = F.softmax(torch.mean(torch.topk(element_logits, k=int(np.ceil(len(features) / 8)), dim=0)[0], dim=0), dim=0).cpu().data.numpy() element_logits = element_logits.cpu().data.numpy() # (1,T,20) instance_logits_stack.append(tmp) element_logits_stack.append(element_logits) labels_stack.append(labels) instance_logits_stack = np.array(instance_logits_stack) labels_stack = np.array(labels_stack) dmap, iou = dmAP(element_logits_stack, dataset.path_to_annotations, args) if args.dataset_name == 'Thumos14': test_set = sio.loadmat('test_set_meta.mat')['test_videos'][0] for i in range(np.shape(labels_stack)[0]): if test_set[i]['background_video'] == 'YES': labels_stack[i, :] = np.zeros_like(labels_stack[i, :]) cmap = cmAP(instance_logits_stack, labels_stack) # (212,20), (212,20) print('Classification map %f' % cmap) print('Detection map @ %f = %f' % (iou[0], dmap[0])) print('Detection map @ %f = %f' % (iou[1], dmap[1])) print('Detection map @ %f = %f' % (iou[2], dmap[2])) print('Detection map @ %f = %f' % (iou[3], dmap[3])) print('Detection map @ %f = %f' % (iou[4], dmap[4])) logger.log_value('Test Classification mAP', cmap, itr) for item in list(zip(dmap, iou)): logger.log_value('Test Detection mAP @ IoU = ' + str(item[1]), item[0], itr) utils.write_to_file(args.dataset_name, dmap, cmap, itr)
def test(itr, dataset, args, model, logger, device): model.eval() done = False instance_logits_stack = [] element_logits_stack = [] labels_stack = [] print("TESTING") while not done: features, labels, done = dataset.load_data(is_training=False) features = torch.from_numpy(features).float().to(device) features = features.unsqueeze(0) _, element_logits = model(features) element_logits = element_logits.squeeze(0) tmp = (F.softmax( torch.mean( torch.topk(element_logits, k=int(np.ceil(len(features) / args.topk)), dim=0)[0], dim=0, ), dim=0, ).cpu().data.numpy()) element_logits = element_logits.cpu().data.numpy() instance_logits_stack.append(tmp) element_logits_stack.append(element_logits) labels_stack.append(labels) instance_logits_stack = np.array(instance_logits_stack) labels_stack = np.array(labels_stack) iou = [0.1, 0.3, 0.5, 0.7] dmap_detect = ANETdetection(dataset.path_to_annotations, iou, args=args) dmap_detect._import_prediction(element_logits_stack) dmap = dmap_detect.evaluate() if args.dataset_name == "Thumos14": test_set = sio.loadmat("test_set_meta.mat")["test_videos"][0] for i in range(np.shape(labels_stack)[0]): if test_set[i]["background_video"] == "YES": labels_stack[i, :] = np.zeros_like(labels_stack[i, :]) cmap = cmAP(instance_logits_stack, labels_stack) print("Classification map %f" % cmap) for k in range(len(iou)): print("Detection map @ %f = %f" % (iou[k], dmap[k] * 100)) return (dmap[-1] + dmap[-2])
def test(itr, dataset, args, model, logger, device, exp_name): done = False instance_logits_stack = [] element_logits_stack = [] labels_stack = [] while not done: if dataset.currenttestidx % 100 == 0: print('Testing test data point %d of %d' % (dataset.currenttestidx, len(dataset.testidx))) features, labels, done = dataset.load_data(is_training=False) features = torch.from_numpy(features).float().to(device) #print ('test: ', features.shape, labels.shape) with torch.no_grad(): _, element_logits = model(Variable(features), is_training=False) tmp = F.softmax(torch.mean(torch.topk(element_logits, k=int(np.ceil(len(features) / 8)), dim=0)[0], dim=0), dim=0).cpu().data.numpy() element_logits = element_logits.cpu().data.numpy() instance_logits_stack.append(tmp) element_logits_stack.append(element_logits) labels_stack.append(labels) np.save('./ckpt/' + exp_name + '/%s_%06d_cas.npy' % (exp_name, itr), np.array(element_logits_stack), allow_pickle=True) np.save('./ckpt/' + exp_name + '/%s_%06d_pmf.npy' % (exp_name, itr), np.array(instance_logits_stack), allow_pickle=True) np.save('./ckpt/' + exp_name + '/%s_%06d_lab.npy' % (exp_name, itr), np.array(labels_stack), allow_pickle=True) instance_logits_stack = np.array(instance_logits_stack) labels_stack = np.array(labels_stack) dmap, iou = dmAP(element_logits_stack, dataset.path_to_annotations, args) cmap = cmAP(instance_logits_stack, labels_stack) print('Classification map %f' % cmap) print('Detection map @ %f = %f' % (iou[0], dmap[0])) print('Detection map @ %f = %f' % (iou[1], dmap[1])) print('Detection map @ %f = %f' % (iou[2], dmap[2])) print('Detection map @ %f = %f' % (iou[3], dmap[3])) print('Detection map @ %f = %f' % (iou[4], dmap[4])) print('Detection map @ %f = %f' % (iou[5], dmap[5])) print('Detection map @ %f = %f' % (iou[6], dmap[6]), flush=True) #logger.scalar_summary('test/c_mAP', cmap, step=itr) logger.add_scalar('test/c_mAP', cmap, global_step=itr) for item in list(zip(dmap, iou)): #logger.scalar_summary('test/d_mAP_at_%s'%str(item[1]), item[0], step=itr) logger.add_scalar('test/d_mAP_at_%s' % str(item[1]), item[0], global_step=itr)
def evaluate(itr, dataset, model, logger, groundtruth_filename, prediction_filename, device=torch.device('cuda'), background=True, subset='test', fps=25, stride=16, threshold_type='mean', frame_type='max', adjust_mean=1.0, act_weight=1.0, tiou_thresholds=np.linspace(0.1, 0.7, 7), use_anchor=False): ''' generate proposals and evaluate ''' with open(groundtruth_filename, 'r') as fr: gt_info = json.load(fr)['database'] save_dict = {'version': dataset.dataset_name, 'external_data': 'None'} frame_dict = {'version': dataset.dataset_name, 'external_data': 'abc'} rs = defaultdict(list) rs2 = defaultdict(list) instance_logits_stack = [] labels_stack = [] inds = dataset.get_testidx() classlist = dataset.get_classlist() tps = [0] * len(classlist) aps = [0] * len(classlist) res = [0] * len(classlist) one_hots = np.eye(len(classlist)) for idx in inds: feat = dataset.get_feature(idx) vname = dataset.get_vname(idx) duration = dataset.get_duration(idx) feat = torch.from_numpy(np.expand_dims(feat, axis=0)).float().to(device) frame_label = dataset.get_gt_frame_label(idx) video_label = dataset.get_video_label(idx) if len(video_label) < 1: continue with torch.no_grad(): _, logits_f, _, logits_r, tcam, att_logits_f, att_logits_r, att_logits = model( Variable(feat), device, is_training=False) logits_f, logits_r, tcam = logits_f[0], logits_r[0], tcam[0] topk = int(np.ceil(len(feat[0]) / 8)) tmp = F.softmax(torch.mean(torch.topk(logits_f, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy() tmp += F.softmax(torch.mean(torch.topk(logits_r, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy() tmp += F.softmax(torch.mean(torch.topk(tcam, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy() if background: tcam = tcam[:, 1:] tmp = tmp[1:] instance_logits_stack.append(tmp) labels_stack.append(np.sum(one_hots[video_label], axis=0)) tcam = tcam.cpu().data.numpy() pred_label = np.argmax(tcam, axis=-1) assert len(pred_label) == len(frame_label) for gt in frame_label: for g in gt: res[g - 1] += 1 score = np.zeros((len(tcam), 1)) if use_anchor: att = att_logits[0].cpu().data.numpy() score = att.squeeze() segments, frames = generate_segment(tcam, score, threshold_type=threshold_type, frame_type=frame_type, act_weight=act_weight, adjust_mean=adjust_mean) fps = gt_info[vname].get('fps', fps) for frame in frames: aps[frame[0]] += 1 if frame[0] in frame_label[frame[1]]: tps[frame[0]] += 1 rs2[vname] += [{ 'score': float(frame[2] / 100.0), 'label': classlist[frame[0]].decode('utf-8'), 'frame': float(frame[1] * stride / fps) }] for seg in segments: rs[vname] += [{ 'score': float(seg[3] / 100.0), 'label': str(classlist[seg[0]].decode('utf-8')), 'segment': [float(seg[1] * stride / fps), float(seg[2] * stride / fps)] }] save_dict['results'] = rs frame_dict['results'] = rs2 with open(prediction_filename, 'w') as fw: json.dump(save_dict, fw) frame_detection = FrameDetection(groundtruth_filename, frame_dict, subset=subset, verbose=True, check_status=False) frame_detection.evaluate() anet_detection = ANETdetection(groundtruth_filename, save_dict, subset=subset, tiou_thresholds=tiou_thresholds, verbose=True, check_status=False) dmap = anet_detection.evaluate() for i in range(len(dmap)): logger.log_value('mAP/IoU@%s' % (str(tiou_thresholds[i])), dmap[i], itr) labels_stack = np.array(labels_stack) instance_logits_stack = np.array(instance_logits_stack) cmap = cmAP(instance_logits_stack, labels_stack) print(cmap) tp = np.sum(tps) ap = np.sum(aps) recall = np.sum(res) print( 'All act frames %d, predict all frames : %d, right frames: %d, AP: %0.5f, Recall: %0.5f' % (recall, ap, tp, tp / ap, tp / recall)) acc = dmap[-3] return np.mean(acc)
def test(dataset, args, itr): # Placeholders feature_seq = tf.placeholder(tf.float32, [1, args.max_seqlen, args.feature_size]) seq_len = tf.cast( tf.reduce_sum(tf.sign(tf.reduce_max(tf.abs(feature_seq), axis=2)), axis=1), tf.int32) fseq = feature_seq[:, :tf.reduce_max(seq_len), :] sgn = tf.sign(tf.reduce_sum(tf.abs(fseq), keep_dims=True, axis=2)) seq_len = tf.cast( tf.reduce_sum(tf.sign(tf.reduce_max(tf.abs(fseq), axis=2)), axis=1), tf.int32) k = tf.cast(tf.ceil(tf.cast(seq_len, tf.float32) / 8), tf.int32) with tf.device('/gpu:0'): with tf.variable_scope('Fully_Connected', reuse=True): fc_W = _variable_with_weight_decay( 'fc_w', [args.feature_size, args.feature_size], 0.0005) fc_b = _variable_with_weight_decay('fc_b', [args.feature_size], 0.0000) feature = tf.matmul( fseq, tf.tile(tf.expand_dims(fc_W, 0), [1, 1, 1])) + fc_b feature = tf.nn.relu(feature) feature = tf.nn.dropout(feature, 1.0) with tf.variable_scope('Attention', reuse=True) as an: atn_W = _variable_with_weight_decay( 'atn_w', [args.feature_size, args.num_class], 0.0005) atn_b = _variable_with_weight_decay('atn_b', [args.num_class], 0.0000) temporal_logits = tf.matmul( feature, tf.tile(tf.expand_dims(atn_W, 0), [1, 1, 1])) + atn_b # Initialize everything init = tf.global_variables_initializer() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) sess.run(init) saver = tf.train.Saver() saver.restore( sess, tf.train.latest_checkpoint('./ckpt/' + args.model_name + '/')) # Test element_logit_stack = [] instance_logit_stack = [] label_stack = [] done = False while not done: features, label, done = dataset.load_data(is_training=False) element_logit = [] features = np.concatenate([ features, np.zeros((args.max_seqlen - len(features) % args.max_seqlen, args.feature_size)) ], axis=0) for i in range(0, len(features), args.max_seqlen): tmp = sess.run(temporal_logits, feed_dict={ feature_seq: np.expand_dims(features[i:i + args.max_seqlen], axis=0) }) if len(element_logit) == 0: element_logit = np.squeeze(tmp) else: element_logit = np.concatenate( [element_logit, np.squeeze(tmp)], axis=0) element_logit = np.array(element_logit) instance_logit = np.mean(np.sort( element_logit, axis=0)[::-1][:max(1, int(np.shape(element_logit)[0] / 8)), :], axis=0) instance_logit = np.exp(instance_logit) instance_logit /= np.sum(instance_logit) instance_logit_stack.append(instance_logit) element_logit_stack.append(element_logit) label_stack.append(label) instance_logit_stack = np.array(instance_logit_stack) label_stack = np.array(label_stack) dmap, iou = dmAP(element_logit_stack, dataset.path_to_annotations) if args.dataset_name == 'Thumos14': test_set = sio.loadmat('test_set_meta.mat')['test_videos'][0] for i in range(np.shape(label_stack)[0]): if test_set[i]['background_video'] == 'YES': label_stack[i, :] = np.zeros_like(label_stack[i, :]) cmap = cmAP(instance_logit_stack, label_stack) np.save('package-1.npy', (instance_logit_stack, label_stack)) print('Classification map %f' % cmap) print('Detection map @ %f = %f' % (iou[0], dmap[0])) print('Detection map @ %f = %f' % (iou[1], dmap[1])) print('Detection map @ %f = %f' % (iou[2], dmap[2])) print('Detection map @ %f = %f' % (iou[3], dmap[3])) print('Detection map @ %f = %f' % (iou[4], dmap[4])) utils.write_to_file(args.dataset_name, dmap, cmap, itr)
def test(itr, dataset, args, model, logger, device): done = False # flow,rgb,tcam的综合分类结果 instance_logits_stack = [] #(2380,100) # tcam的分类结果 tcam_stack = [] #(2380,T,100) # label标签 labels_stack = [] #(2380,100) while not done: if dataset.currenttestidx % 100 ==0: print('Testing test data point %d of %d' %(dataset.currenttestidx, len(dataset.testidx))) features, labels, done = dataset.load_data(is_training=False) seq_len = np.sum(np.max(np.abs(features), axis=2) > 0, axis=1) features = torch.from_numpy(features).float().to(device) with torch.no_grad(): _, logits_f, _, logits_r, tcam, _ = model(Variable(features), device, is_training=False, seq_len=torch.from_numpy(seq_len).to(device)) logits_f, logits_r, tcam = logits_f[0], logits_r[0], tcam[0] topk = int(np.ceil(len(features[0])/8)) tmp = F.softmax(torch.mean(torch.topk(logits_f, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy() tmp += F.softmax(torch.mean(torch.topk(logits_r, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy() tmp += F.softmax(torch.mean(torch.topk(tcam, k=topk, dim=0)[0], dim=0), dim=0).cpu().data.numpy() tcam = tcam.cpu().data.numpy() instance_logits_stack.append(tmp) tcam_stack.append(tcam) labels_stack.append(labels) instance_logits_stack = np.array(instance_logits_stack) #(video_num,20) labels_stack = np.array(labels_stack) # (video_num,20) if args.dataset_name.find('Thumos14')!= -1 and args.num_class == 101: test_set = sio.loadmat('test_set_meta.mat')['test_videos'][0] bg_vid = 0 for i in range(np.shape(labels_stack)[0]): if test_set[i]['background_video'] == 'YES': bg_vid += 1 labels_stack[i,:] = np.zeros_like(labels_stack[i,:]) cmap = cmAP(instance_logits_stack, labels_stack) print(cmap) # tcam_stack : (video_num,T,20), dataset.path_to_annotations: GT 标注 , None, None dmap, iou = dmAP(tcam_stack, dataset.path_to_annotations, args.activity_net, valid_id=dataset.lst_valid) print('Classification map %f' %cmap) for k in range(len(dmap)): print('Detection map @ %f = %f' %(iou[k], dmap[k])) print('Mean Detection map = %f' %(np.mean(dmap))) dmap += [np.mean(dmap)] logger.log_value('Test Classification mAP', cmap, itr) for item in list(zip(dmap,iou)): logger.log_value('Test Detection mAP @ IoU = ' + str(item[1]), item[0], itr) utils.write_to_file(args.dataset_name + args.model_name, dmap, cmap, itr) if args.activity_net: return dmap.pop() else: return dmap[4]