def mini_eval(self, current_step): if current_step % self.config.TRAIN.mini_eval_step != 0: return temp_meter_rec = AverageMeter() temp_meter_pred = AverageMeter() self.STAE.eval() for data in self.val_dataloader: # get the reconstruction and prediction video clip time_len = data.shape[2] rec_time = time_len // 2 inupt_rec_mini = data[:, :, 0:rec_time, :, :].cuda() # 0 ~ t//2 frame input_pred_mini = data[:, :, rec_time:time_len, :, :].cuda( ) # t//2 ~ t frame # Use the model, get the output output_rec_mini, output_pred_mini = self.STAE(inupt_rec_mini) rec_psnr_mini = psnr_error(output_rec_mini.detach(), inupt_rec_mini) pred_psnr_mini = psnr_error(output_pred_mini.detach(), input_pred_mini) temp_meter_rec.update(rec_psnr_mini.detach()) temp_meter_pred.update(pred_psnr_mini.detach()) self.logger.info( f'&^*_*^& ==> Step:{current_step}/{self.max_steps} the rec PSNR is {temp_meter_rec.avg:.3f}, the pred PSNR is {temp_meter_pred.avg:.3f}' )
def mini_eval(self, current_step): if current_step % self.config.TRAIN.mini_eval_step != 0: return temp_meter_frame = AverageMeter() temp_meter_flow = AverageMeter() self.G.eval() self.D.eval() for data in self.val_dataloader: # get the data target_mini = data[:, :, 1, :, :] input_data_mini = data[:, :, 0, :, :] # squeeze the dimension target_mini = target_mini.view(target_mini.shape[0], -1, target_mini.shape[-2], target_mini.shape[-1]).cuda() input_data_mini = input_data_mini.view( input_data_mini.shape[0], -1, input_data_mini.shape[-2], input_data_mini.shape[-1]).cuda() # Use the model, get the output output_flow_G_mini, output_frame_G_mini = self.G(input_data_mini) input_gtFlowEstimTensor = torch.cat([input_data_mini, target_mini], 1) gtFlow, _ = flow_batch_estimate(self.F, input_gtFlowEstimTensor) frame_psnr_mini = psnr_error(output_frame_G_mini.detach(), target_mini) flow_psnr_mini = psnr_error(output_flow_G_mini.detach(), gtFlow) temp_meter_frame.update(frame_psnr_mini.detach()) temp_meter_flow.update(flow_psnr_mini.detach()) self.logger.info( f'&^*_*^& ==> Step:{current_step}/{self.max_steps} the frame PSNR is {temp_meter_frame.avg:.3f}, the flow PSNR is {temp_meter_flow.avg:.3f}' )
def mini_eval(self, current_step): if current_step % self.config.TRAIN.mini_eval_step != 0: return temp_meter_A = AverageMeter() temp_meter_B = AverageMeter() temp_meter_C = AverageMeter() self.A.eval() self.B.eval() self.C.eval() self.Detector.eval() for data in self.val_dataloader: # base on the D to get each frame # in this method, D = 3 and not change future_mini = data[:, -1, :, :, :].cuda() # t+1 frame current_mini = data[:, 1, :, :, :].cuda() # t frame past_mini = data[:, 0, :, :, :].cuda() # t-1 frame bboxs_mini = get_batch_dets(self.Detector, current_mini) for index, bbox in enumerate(bboxs_mini): if bbox.numel() == 0: bbox = bbox.new_zeros([1, 4]) # get the crop objects input_currentObject_B, _ = multi_obj_grid_crop( current_mini[index], bbox) future_object, _ = multi_obj_grid_crop(future_mini[index], bbox) future2current = torch.stack( [future_object, input_currentObject_B], dim=1) past_object, _ = multi_obj_grid_crop(past_mini[index], bbox) current2past = torch.stack( [input_currentObject_B, past_object], dim=1) _, _, input_objectGradient_A = frame_gradient(future2current) input_objectGradient_A = input_objectGradient_A.sum(1) _, _, input_objectGradient_C = frame_gradient(current2past) input_objectGradient_C = input_objectGradient_A.sum(1) _, output_recGradient_A = self.A(input_objectGradient_A) _, output_recObject_B = self.B(input_currentObject_B) _, output_recGradient_C = self.C(input_objectGradient_C) psnr_A = psnr_error(output_recGradient_A.detach(), input_objectGradient_A) psnr_B = psnr_error(output_recObject_B.detach(), input_currentObject_B) psnr_C = psnr_error(output_recGradient_C.detach(), input_objectGradient_C) temp_meter_A.update(psnr_A.detach()) temp_meter_B.update(psnr_B.detach()) temp_meter_C.update(psnr_C.detach()) self.logger.info( f'&^*_*^& ==> Step:{current_step}/{self.max_steps} the A PSNR is {temp_meter_A.avg:.2f}, the B PSNR is {temp_meter_B.avg:.2f}, the C PSNR is {temp_meter_C.avg:.2f}' )
def mini_eval(self, current_step): if current_step % 10 != 0 or current_step == 0: return temp_meter_frame = AverageMeter() temp_meter_flow = AverageMeter() self.G.eval() self.D.eval() for data in self.val_dataloader: # base on the D to get each frame target_mini = data[:, :, -1, :, :].cuda() # t+1 frame input_data = data[:, :, :-1, :, :] # 0 ~ t frame input_last_mini = input_data[:, :, -1, :, :].cuda() # t frame # squeeze the D dimension to C dimension, shape comes to [N, C, H, W] input_data_mini = input_data.view(input_data.shape[0], -1, input_data.shape[-2], input_data.shape[-1]).cuda() output_pred_G = self.G(input_data_mini) gtFlow, _ = flow_batch_estimate(self.F, torch.cat([input_last_mini, target_mini], 1)) predFlow, _ = flow_batch_estimate(self.F, torch.cat([input_last_mini, output_pred_G], 1)) frame_psnr_mini = psnr_error(output_pred_G.detach(), target_mini) flow_psnr_mini = psnr_error(predFlow, gtFlow) temp_meter_frame.update(frame_psnr_mini.detach()) temp_meter_flow.update(flow_psnr_mini.detach()) self.logger.info(f'&^*_*^& ==> Step:{current_step}/{self.max_steps} the PSNR is {temp_meter_frame.avg:.2f}, the flow PSNR is {temp_meter_flow.avg:.2f}')
def mini_eval(self, current_step): if current_step % self.config.TRAIN.mini_eval_step != 0: return temp_meter_frame = AverageMeter() self.MemAE.eval() for data in self.val_dataloader: # get the data input_data_mini = data.cuda() output_rec, _ = self.MemAE(input_data_mini) frame_psnr_mini = psnr_error(output_rec.detach(), input_data_mini) temp_meter_frame.update(frame_psnr_mini.detach()) self.logger.info( f'&^*_*^& ==> Step:{current_step}/{self.max_steps} the frame PSNR is {temp_meter_frame.avg:.3f}' )
def evaluate(self, current_step): ''' Evaluate the results of the model !!! Will change, e.g. accuracy, mAP..... !!! Or can call other methods written by the official ''' tb_writer = self.trainer.kwargs['writer_dict']['writer'] global_steps = self.trainer.kwargs['writer_dict']['global_steps_{}'.format(self.trainer.kwargs['model_type'])] frame_num = self.config.DATASET.test_clip_length psnr_records=[] score_records=[] total = 0 # for dirs in video_dirs: random_video_sn = torch.randint(0, len(self.trainer.test_dataset_keys), (1,)) for sn, video_name in enumerate(self.trainer.test_dataset_keys): # _temp_test_folder = os.path.join(self.testing_data_folder, dir) # need to improve # dataset = AvenueTestOld(_temp_test_folder, clip_length=frame_num) dataset = self.trainer.test_dataset_dict[video_name] len_dataset = dataset.pics_len test_iters = len_dataset - frame_num + 1 test_counter = 0 data_loader = DataLoader(dataset=dataset, batch_size=1, shuffle=False, num_workers=1) # import ipdb; ipdb.set_trace() psnrs = np.empty(shape=(len_dataset,),dtype=np.float32) scores = np.empty(shape=(len_dataset,),dtype=np.float32) # for test_input, _ in data_loader: vis_range = range(int(len_dataset*0.5), int(len_dataset*0.5 + 5)) for frame_sn, test_input in enumerate(data_loader): test_target = test_input[:, -1].cuda() test_input = test_input[:, :-1].transpose_(1,2).cuda() g_output = self.trainer.G(test_input, test_target) test_psnr = psnr_error(g_output, test_target) test_psnr = test_psnr.tolist() psnrs[test_counter+frame_num-1]=test_psnr scores[test_counter+frame_num-1]=test_psnr if sn == random_video_sn and (frame_sn in vis_range): self.add_images(test_target, g_output, tb_writer, global_steps) test_counter += 1 total+=1 if test_counter >= test_iters: psnrs[:frame_num-1]=psnrs[frame_num-1] scores[:frame_num-1]=(scores[frame_num-1],) smax = max(scores) smin = min(scores) normal_scores = np.array([np.divide(s-smin, smax-smin) for s in scores]) psnr_records.append(psnrs) score_records.append(normal_scores) print(f'finish test video set {video_name}') break result_dict = {'dataset': self.trainer.config.DATASET.name, 'psnr': psnr_records, 'flow': [], 'names': [], 'diff_mask': [], 'score':score_records, 'num_vides':len(psnr_records)} result_path = os.path.join(self.trainer.config.TEST.result_output, f'{self.trainer.verbose}_cfg#{self.trainer.config_name}@{self.trainer.kwargs["time_stamp"]}_results.pkl') with open(result_path, 'wb') as writer: pickle.dump(result_dict, writer, pickle.HIGHEST_PROTOCOL) # results = eval_api.evaluate('compute_auc_score', result_path, self.trainer.logger, self.trainer.config) results = self.trainer.evaluate_function(result_path, self.trainer.logger, self.trainer.config) self.trainer.logger.info(results) tb_writer.add_text('anopcn: AUC of ROC curve', f'auc is {results.auc}',global_steps) return results.auc
def evaluate(self, current_step): ''' Evaluate the results of the model !!! Will change, e.g. accuracy, mAP..... !!! Or can call other methods written by the official ''' tb_writer = self.trainer.kwargs['writer_dict']['writer'] global_steps = self.trainer.kwargs['writer_dict'][ 'global_steps_{}'.format(self.trainer.kwargs['model_type'])] frame_num = self.trainer.config.DATASET.test_clip_length psnr_records = [] score_records = [] # score_records_w=[] w_dict = OrderedDict() # total = 0 # calc the scores for the training set for video_name in self.trainer.test_dataset_keys_w: dataset = self.trainer.test_dataset_dict_w[video_name] len_dataset = dataset.pics_len test_iters = len_dataset - frame_num + 1 test_counter = 0 data_loader = DataLoader(dataset=dataset, batch_size=1, shuffle=False, num_workers=1) scores = [0.0 for i in range(len_dataset)] for test_input in data_loader: # import ipdb; ipdb.set_trace() test_target = test_input[:, -1].cuda() test_input = test_input[:, :-1].view( test_input.shape[0], -1, test_input.shape[-2], test_input.shape[-1]).cuda() # import ipdb; ipdb.set_trace() g_output_flow, g_output_frame = self.trainer.G(test_input) gt_flow_esti_tensor = torch.cat([test_input, test_target], 1) # flow_gt, _ = self.trainer.batch_estimate(gt_flow_esti_tensor) flow_gt, _ = flow_batch_estimate(self.trainer.F, gt_flow_esti_tensor) # score = amc_score(test_input, g_output_frame, flow_gt, g_output_flow) diff_appe, diff_flow = simple_diff(test_input, g_output_frame, flow_gt, g_output_flow) max_patch_appe, max_patch_flow = find_max_patch( diff_appe, diff_flow) score_appe = torch.mean(max_patch_appe) score_flow = torch.mean(max_patch_flow) scores[test_counter + frame_num - 1] = [score_appe, score_flow] test_counter += 1 if test_counter >= test_iters: scores[:frame_num - 1] = [scores[frame_num - 1]] scores = torch.tensor(scores) frame_w = torch.mean(scores[:, 0]) flow_w = torch.mean(scores[:, 1]) w_dict[video_name] = [len_dataset, frame_w, flow_w] print( f'finish calc the scores of training set {video_name} in step:{current_step}' ) break wf, wi = calc_w(w_dict) # wf , wi = 1.0, 1.0 tb_writer.add_text('weight of train set', f'w_f:{wf:.3f}, w_i:{wi:.3f}', global_steps) num_videos = 0 random_video_sn = torch.randint(0, len(self.trainer.test_dataset_keys), (1, )) # calc the score for the test dataset for sn, video_name in enumerate(self.trainer.test_dataset_keys): num_videos += 1 # need to improve dataset = self.trainer.test_dataset_dict[video_name] len_dataset = dataset.pics_len test_iters = len_dataset - frame_num + 1 test_counter = 0 data_loader = DataLoader(dataset=dataset, batch_size=1, shuffle=False, num_workers=1) vis_range = range(int(len_dataset * 0.5), int(len_dataset * 0.5 + 5)) psnrs = np.empty(shape=(len_dataset, ), dtype=np.float32) scores = np.empty(shape=(len_dataset, ), dtype=np.float32) # scores = [0.0 for i in range(len_dataset)] for frame_sn, test_input in enumerate(data_loader): test_target = test_input[:, -1].cuda() test_input = test_input[:, :-1].view( test_input.shape[0], -1, test_input.shape[-2], test_input.shape[-1]).cuda() g_output_flow, g_output_frame = self.trainer.G(test_input) gt_flow_esti_tensor = torch.cat([test_input, test_target], 1) # flow_gt,_ = self.trainer.batch_estimate(gt_flow_esti_tensor) flow_gt, _ = flow_batch_estimate(self.trainer.F, gt_flow_esti_tensor) # score = amc_score(test_input, g_output_frame, flow_gt, g_output_flow) test_psnr = psnr_error(g_output_frame, test_target) score = amc_score(test_input, g_output_frame, flow_gt, g_output_flow, wf, wi) # import ipdb; ipdb.set_trace() test_psnr = test_psnr.tolist() score = score.tolist() psnrs[test_counter + frame_num - 1] = test_psnr scores[test_counter + frame_num - 1] = score test_counter += 1 if sn == random_video_sn and (frame_sn in vis_range): self.add_images(test_target, flow_gt, g_output_frame, g_output_flow, tb_writer, global_steps) if test_counter >= test_iters: psnrs[:frame_num - 1] = psnrs[frame_num - 1] # import ipdb; ipdb.set_trace() scores[:frame_num - 1] = ( scores[frame_num - 1], ) # fix the bug: TypeError: can only assign an iterable smax = max(scores) normal_scores = np.array( [np.divide(s, smax) for s in scores]) psnr_records.append(psnrs) score_records.append(normal_scores) print(f'finish test video set {video_name}') break result_dict = { 'dataset': self.trainer.config.DATASET.name, 'psnr': psnr_records, 'flow': [], 'names': [], 'diff_mask': [], 'score': score_records, 'num_videos': num_videos } result_path = os.path.join( self.trainer.config.TEST.result_output, f'{self.trainer.verbose}_cfg#{self.trainer.config_name}#step{current_step}@{self.trainer.kwargs["time_stamp"]}_results.pkl' ) with open(result_path, 'wb') as writer: pickle.dump(result_dict, writer, pickle.HIGHEST_PROTOCOL) # results = eval_api.evaluate('compute_auc_score', result_path, self.trainer.logger, self.trainer.config) results = self.trainer.evaluate_function( result_path, self.trainer.logger, self.trainer.config, self.trainer.config.DATASET.score_type) self.trainer.logger.info(results) tb_writer.add_text('amc: AUC of ROC curve', f'auc is {results.auc}', global_steps) return results.auc