def estimate_signatures_softmax(make_model, qs, distances_matrix, r_matrix, dists, iters=50, learning_rate=0.1, loss_eval_interval=5, draw_interval=20, prev_total_loss_story=None, different_d_sum=False, print_results=False, loss_name="softmax"): total_loss_story = list() if prev_total_loss_story is None else prev_total_loss_story for dist in dists: d = (make_model(dist.d_sum) if different_d_sum else make_model()) loss_story = list() opt = tf.keras.optimizers.Adam(learning_rate=learning_rate) for i in tqdm.tqdm_notebook(range(iters)): loss = lambda : d.get_matrix_loss( qs, None, r_matrix, distance=dist, loss=loss_name, symmetric=True ) distortion = loss().numpy() print(f"{i}:{distortion}") m = opt.minimize( loss, var_list=d.get_weights() + dist.get_weights() ) if (i % loss_eval_interval == 0) or (i + 1 == iters): pred = d.get_matrix_loss( qs, qs, None, distance=dist, loss=None, symmetric=True ) loss_story.append(( round(distortion, 7), round(metrics.mAP(pred, r_matrix, True), 7), round(metrics.NDCG_graph_v1(pred, distances_matrix), 7) )) if (i % draw_interval == 0) or (i + 1 == iters): plt.figure(figsize=(10, 5)) plt.title(f"|V| = {qs.shape[0]}") for i, prev_loss_story in enumerate(total_loss_story): plt.plot([x[-2] for x in prev_loss_story], label=f"{dists[i]}:{prev_loss_story[-1]}") plt.plot([x[-2] for x in loss_story], label=f"{dist}:{loss_story[-1]}") plt.legend() plt.grid() clear_output() plt.savefig(strftime(f"./pngs/softmax-V-{qs.shape[0]}--%Y-%m-%d-%Hh.png", gmtime())) plt.show() # print(np.mean(loss_story[-20:])) total_loss_story.append(loss_story) if print_results: for d, l in zip(dists, total_loss_story): print(f"{l[-1][1]}\t{d}") return total_loss_story
def update_pbar(self, masks_predictions, masks_targets, pbar, average_meter, pbar_description): average_meter.add('iou', iou(masks_predictions > 0.5, masks_targets.byte())) average_meter.add('mAP', mAP(masks_predictions > 0.5, masks_targets.byte())) pbar.set_description( pbar_description + ''.join( [' {}:{:6.4f}'.format(k, v) for k, v in average_meter.get_all().items()] ) ) pbar.update()
def update(self, predictions, targets): optimal_map_score = 0 optimal_t = 0 for i in range(1, 10): t = 1 / i map_score = metrics.mAP(predictions > t, targets.byte()) if map_score > optimal_map_score: optimal_map_score = map_score optimal_t = t self.meter.add('threshold', optimal_t) return optimal_map_score, optimal_t
def evaluation_metrics(self): self.q_features = l2_norm_standardize(self.q_features) self.g_features = l2_norm_standardize(self.g_features) # Scores against all feature vectors in the gallery image for each # image in the query data. scores = joint_scores(self.q_features, self.q_cam_ids, self.q_frames, self.g_features, self.g_cam_ids, self.g_frames, self.trainer.datamodule.st_distribution) if self.rerank: scores = re_ranking(scores) # Metrics & Evaluation mean_ap, cmc = mAP(scores, self.q_targets, self.q_cam_ids, self.g_targets, self.g_cam_ids) return mean_ap, cmc
H = [] for images, _ in train_loader: in_aff, out_aff = rbf_affnty(images, anchor, topk=20) images = Variable(images).cuda() in_aff = Variable(in_aff).cuda() out_aff = Variable(out_aff).cuda() out, _ = gcn(images, in_aff, out_aff, 1) H.append(out.data.cpu().numpy()) R = np.concatenate(H) H = np.sign(R) HammTrainTest = 0.5 * (n_bits - np.dot(H, tH.T)) HammingRank = np.argsort(HammTrainTest, axis=0) ap = mAP(dset.cateTrainTest, HammingRank) pre, rec = topK(dset.cateTrainTest, HammingRank, 500) print('%d bits: mAP: %.4f, precision@500: %.4f, recall@500: %.4F' % (n_bits, ap, pre, rec)) ''' torch.save({'state_dict':gcn.state_dict(), 'anchor': anchor, 'anchor_affnty': anchor_affnty}, './%s_%d_%d_%.4f_%.4f' % (dataset, n_labeled, n_bits, ap, pre)) import scipy.io as sio sio.savemat('./%s_%d_%d' % (dataset, n_labeled, n_bits), {'H': H, 'tH': tH, 'R': R, 'tR': tR}) '''
def test(net, dataloader, criterion=None, evaluate=True, predict=True, device=torch.device('cuda:0'), num_class=2): ''' 使用训练好的net进行预测或评价 args: net,训练好的RetinaNet对象; dataloader,需要进行预测或评价的数据集的dataloader; criterion,计算损失的函数,如果是None则不计算loss; evaluate,如果是True则进行模型的评价(即计算mAP),如果 是False,则只进行预测或计算loss; predict,是否进行预测,如果True则会返回预测框的类别和坐标,如果False, 则不会返回; returns: all_labels_pred, all_markers_pred,如果predict=True,则返回预测的每张图 片的预测框的标签和loc; losses, cls_losses, loc_losses,如果criterion不是None,则得到其3个loss; APs, mAP_score,如果evaluate=True,则返回每一类的AP值和其mAP; ''' data_encoder = dataloader.dataset.y_encoder y_encoder_mode = dataloader.dataset.y_encoder_mode assert (evaluate and y_encoder_mode in ['object', 'all']) or not evaluate assert (criterion is not None and y_encoder_mode in ['all', 'anchor']) or \ criterion is None print('Testing ...') results = [] with torch.no_grad(): losses = 0. cls_losses = 0. loc_losses = 0. all_labels_pred = [] # 预测的labels, all_markers_pred = [] all_labels_true = [] all_markers_true = [] for imgs, labels, markers in pb.progressbar(dataloader): imgs = imgs.to(device) if y_encoder_mode == 'all': # 如果需要计算mAP,则除了dataloader除了输出已经编码好的每个 # anchor对应的偏移量和对应的标签,还需要没有编码的每张图片 # 对应的obj loc tensor和obj label tensor,因为每张图片对应的 # objs的数量不同,所以只能使用list来储存(通过重写dataloader # 的collate_fn) # labels是list,每个元素是一个tensor,指代一张图片中所有的obj # 的label,[(img1#obj,), (img2#obj,), (img3#obj,), ...] # cls_trues是一个tensor,(#imgs, #anchors, #class),是每张图片 # 中每个anchor对应的gtbb的类别 labels, cls_trues = labels # labels in cpu, cls_trues in gpu cls_trues = cls_trues.to(device) # markers是list,每个元素是一个tensor,指代一张图片中所有的obj # 的xyxy loc,[(img1#obj, 4), (img2#obj, 4), (img3#obj, 4), # ...] # loc_trues是一个tensor,(#imgs, #anchors, 4),是每张图片中每个 # anchor和其对应的gtbb的位置偏移量 markers, loc_trues = markers # marker in cpu, loc_trues in gpu loc_trues = loc_trues.to(device) elif y_encoder_mode == 'anchor': # 如果只需要计算loss,则只需要输出编码好的偏移量和anchor的标签 # 即可 cls_trues = labels.to(device) loc_trues = markers.to(device) cls_preds, loc_preds = net(imgs) if criterion is not None: loss, cls_loss, loc_loss = criterion(cls_trues, loc_trues, cls_preds, loc_preds) losses += loss.item() cls_losses += cls_loss.item() loc_losses += loc_loss.item() # 使用decode得到的是list,其每个元素是batch中每个图片的预测框的 # tensor label_preds, marker_preds = data_encoder.decode(cls_preds, loc_preds, device=device) # 使用append则输出的的list的len是图片的数量,这样的输出更加明白 all_markers_pred.append(marker_preds) all_labels_pred.append(label_preds) if evaluate: all_labels_true.append(labels) all_markers_true.append(markers) if predict: results.append((all_labels_pred, all_markers_pred)) if criterion is not None: losses = losses / len(dataloader.dataset) cls_losses = cls_losses / len(dataloader.dataset) loc_losses = loc_losses / len(dataloader.dataset) results.append((losses, cls_losses, loc_losses)) if evaluate: # 使用chain将两层的嵌套list变成一层,符合mAP函数的输出要求 APs, mAP_score = mAP(list(chain.from_iterable(all_labels_true)), list(chain.from_iterable(all_markers_true)), list(chain.from_iterable(all_labels_pred)), list(chain.from_iterable(all_markers_pred)), num_class=num_class) results.append((APs, mAP_score)) return tuple(results)
def train(net, criterion, optimizer, dataloaders, epoch, lr_schedular=None, clip_norm=None, best_num=1, best_metric='mAP', history=History(), num_class=2): ''' 对RetinaNet进行训练, args: net,实例化的RetinaNet网络,在训练的过程中其发生了改变; criterion,loss,即实例化的FocalLoss对象; optimizer,优化器; dataloaders,dataloaders组成的dict,其key必须有'train',可以有'valid', 'test',如果有test则会在最后使用训练过程中最好的net进行预测; epoch,int,一共需要进行多少个epoch的训练; lr_schedular,使用的学习率策略; clip_norm, 使用的梯度截断的参数,如果是None则不进行梯度截断; best_num,要保存的最好的模型的数目; best_metric,来判断模型好坏的指标,可以选择mAP或loss; history,History对象,用于储存训练时期的结果,默认是History()的默认配置; returns: history,History对象,储存每个epoch train的loss和valid的loss、mAP和最好 的几个模型; (另外,实际上输入的net会发生改变,成为在训练过程中valid上最好的net) test_loss, test_mAP,如果有dataloaders中还有test,则会返回; ''' for e in range(epoch): if 'valid' in dataloaders.keys(): phases = ['train', 'valid'] else: phases = ['train'] for phase in phases: # 计算每个epoch的loss epoch_loss = 0. epoch_cls_loss = 0. epoch_loc_loss = 0. # 分不同的phase进行处理 if phase == 'train': net.train() if lr_schedular is not None: lr_schedular.step() widgets = [ 'epoch: %d' % e, '| ', pb.Counter(), pb.Bar(), pb.AdaptiveETA(), ] iterator = pb.progressbar(dataloaders[phase], widgets=widgets) else: net.eval() iterator = dataloaders[phase] # 进行一个epoch中所有batches的迭代 y_encoder = dataloaders[phase].dataset.y_encoder all_label_preds = [] all_marker_preds = [] all_label_trues = [] all_marker_trues = [] for imgs, labels, markers in iterator: imgs = imgs.cuda() if phase == 'train': cls_trues = labels.cuda() loc_trues = markers.cuda() else: labels, cls_trues = labels cls_trues = cls_trues.cuda() markers, loc_trues = markers loc_trues = loc_trues.cuda() if phase == 'train': optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): cls_preds, loc_preds = net(imgs) loss, cls_loss, loc_loss = criterion( cls_trues, loc_trues, cls_preds, loc_preds) if phase == 'train': loss.backward() if clip_norm != 0.0: torch.nn.utils.clip_grad_norm_( net.parameters(), clip_norm) optimizer.step() else: label_preds, marker_preds = y_encoder.decode( cls_preds, loc_preds) all_label_preds.extend(label_preds) all_marker_preds.extend(marker_preds) all_label_trues.extend(labels) all_marker_trues.extend(markers) with torch.no_grad(): epoch_loss += loss.item() epoch_cls_loss += cls_loss.item() epoch_loc_loss += loc_loss.item() # 每个epoch的所有batches都结束,计算在epoch上的loss平均 with torch.no_grad(): num_batches = len(dataloaders[phase].dataset) epoch_loss /= num_batches epoch_cls_loss /= num_batches epoch_loc_loss /= num_batches if phase == 'train': history.update_hist(loss=epoch_loss, cls_loss=epoch_cls_loss, loc_loss=epoch_loc_loss) print('%s, loss: %.4f, cls_loss: %.4f, loc_loss: %.4f' % (phase, epoch_loss, epoch_cls_loss, epoch_loc_loss)) elif phase == 'valid': _, map_score = mAP(all_label_trues, all_marker_trues, all_label_preds, all_marker_preds, num_class=num_class) history.update_hist(val_loss=epoch_loss, val_cls_loss=epoch_cls_loss, val_loc_loss=epoch_loc_loss, mAP=map_score) if map_score is np.nan: map_score = 0 history.update_best(mAP=map_score, loss=epoch_loss, cls_loss=epoch_cls_loss, loc_loss=epoch_loc_loss, epoch=e, model_wts=copy.deepcopy( net.state_dict())) print(('%s, loss: %.4f, cls_loss: %.4f,' ' loc_loss: %.4f, mAP: %.4f') % (phase, epoch_loss, epoch_cls_loss, epoch_loc_loss, map_score)) if 'valid' in dataloaders.keys(): net.load_state_dict(history.best[0]['model_wts']) print('valid best loss: %.4f, best mAP: %.4f' % (history.best[0]['loss'], history.best[0]['mAP'])) # 如果有test,则再进行test的预测 if 'test' in dataloaders.keys(): test_loss, test_map = test(net, dataloaders['test'], criterion=criterion, evaluate=True, predict=False, num_class=num_class) print('test loss: %.4f, test mAP: %.4f' % (test_loss[0], test_map[1])) return history, (test_loss, test_map) return history
def train(net, criterion, optimizer, dataloaders, epoch, device=torch.device('cuda:0'), hist_args={}): history = History(**hist_args) for e in range(epoch): epoch_loss = 0. for phase in ['train', 'valid']: y_encoder = dataloaders[phase].dataset.y_encoder if phase == 'train': net.train() prefix = '%d, Training: ' % e else: net.eval() prefix = '%d, Validation: ' % e # validation时需要记录所有的真实标签和预测 epoch_label, epoch_loc = [], [] epoch_res_c, epoch_res_s, epoch_res_l = [], [], [] for batch in pb.progressbar(dataloaders[phase], prefix=prefix): imgs, targets = batch[0].to(device), batch[-1].to(device) with torch.set_grad_enabled(phase == 'train'): preds = net(imgs) loss = criterion(preds, targets) if phase == 'train': optimizer.zero_grad() loss.backward() optimizer.step() else: # 如果是eval,需要记录所有的预测,用于计算mAP img_size = list(imgs.shape[-1:-3:-1]) epoch_label.extend(batch[1]) epoch_loc.extend(batch[2]) for pred in preds: res_c, res_s, res_l = y_encoder.decode( pred, img_size) epoch_res_c.append(res_c) epoch_res_s.append(res_s) epoch_res_l.append(res_l) with torch.no_grad(): epoch_loss += loss.item() # 整个epoch的所有batches循环结束 with torch.no_grad(): num_batches = len(dataloaders[phase].dataset) epoch_loss /= num_batches if phase == 'train': history.update_hist(loss=epoch_loss) print('%d, %s, loss: %.4f' % (e, phase, epoch_loss)) else: aps, map_score = mAP(epoch_label, epoch_loc, epoch_res_c, epoch_res_s, epoch_res_l) history.update_hist(val_loss=epoch_loss, mAP=map_score) if map_score is np.nan: map_score = 0 history.update_best(mAP=map_score, loss=epoch_loss, epoch=e, model_wts=copy.deepcopy( net.state_dict())) print('%d, %s, loss: %.4f, mAP: %.4f' % (e, phase, epoch_loss, map_score)) if 'valid' in dataloaders.keys(): net.load_state_dict(history.best[0]['model_wts']) print('valid best loss: %.4f, best mAP: %.4f' % (history.best[0]['loss'], history.best[0]['mAP'])) return history
def test(net, dataloader, criterion=None, evaluate=True, predict=True, device=torch.device('cuda:0')): ''' 使用训练好的net进行预测或评价 args: net,训练好的RetinaNet对象; dataloader,需要进行预测或评价的数据集的dataloader; criterion,计算损失的函数,如果是None则不计算loss; evaluate,如果是True则进行模型的评价(即计算mAP),如果 是False,则只进行预测或计算loss; predict,是否进行预测,如果True则会返回预测框的类别和坐标,如果False, 则不会返回; returns: all_labels_pred, all_markers_pred,如果predict=True,则返回预测的每张图 片的预测框的标签和loc; losses, cls_losses, loc_losses,如果criterion不是None,则得到其3个loss; APs, mAP_score,如果evaluate=True,则返回每一类的AP值和其mAP; ''' y_encoder = dataloader.dataset.y_encoder y_encoder_mode = dataloader.dataset.out assert (evaluate and y_encoder_mode in ['obj', 'all']) or not evaluate assert (criterion is not None and y_encoder_mode in ['all', 'encode']) or \ criterion is None print('Testing ...') results = [] with torch.no_grad(): losses = 0. all_labels_pred = [] # 预测的labels, all_scores_pred = [] all_markers_pred = [] all_labels_true = [] all_markers_true = [] for batch in pb.progressbar(dataloader): imgs = batch[0].to(device) if y_encoder_mode in ['all', 'obj']: labels_true, locs_true = batch[1:3] if y_encoder_mode in ['all', 'encode']: targets = batch[-1].to(device) preds = net(imgs) if criterion is not None: loss = criterion(preds, targets) losses += loss.item() if predict or evaluate: img_size = list(imgs.shape[-1:-3:-1]) for pred in preds: res_c, res_s, res_l = y_encoder.decode(pred, img_size) all_labels_pred.append(res_c) all_scores_pred.append(res_s) all_markers_pred.append(res_l) if evaluate: all_labels_true.extend(labels_true) all_markers_true.extend(locs_true) if predict: results.append((all_labels_pred, all_markers_pred)) if criterion is not None: losses = losses / len(dataloader.dataset) results.append(losses) if evaluate: # 使用chain将两层的嵌套list变成一层,符合mAP函数的输出要求 APs, mAP_score = mAP( all_labels_true, all_markers_true, all_labels_pred, all_scores_pred, all_markers_pred, ) results.append((APs, mAP_score)) return tuple(results)
def evaluate_tagging(self, experiment_path: str, tag_file='tagging_predictions_{}.txt', **kwargs): exppath = Path(experiment_path) if exppath.is_file(): # Best model passed! model_parameters = torch.load( str(exppath), map_location=lambda storage, loc: storage) experiment_path = exppath.parent # Just set upper path as default else: model_parameters = torch.load( glob.glob("{}/run_model*".format(experiment_path))[0], map_location=lambda storage, loc: storage) config = torch.load(glob.glob( "{}/run_config*".format(experiment_path))[0], map_location=lambda storage, loc: storage) logger = utils.getfile_outlogger(None) # Use previous config, but update data such as kwargs config_parameters = dict(config, **kwargs) # Default columns to search for in data config_parameters.setdefault('colname', ('filename', 'encoded')) encoder = torch.load(glob.glob( '{}/run_encoder*'.format(experiment_path))[0], map_location=lambda storage, loc: storage) test_data_filename = os.path.splitext( os.path.basename(config_parameters['label']))[0] strong_labels_df = pd.read_csv(config_parameters['label'], sep='\s+') # Evaluation is done via the filenames, not full paths if not np.issubdtype(strong_labels_df['filename'].dtype, np.number): strong_labels_df['filename'] = strong_labels_df['filename'].apply( os.path.basename) if 'audiofilepath' in strong_labels_df.columns: # In case of ave dataset, the audiofilepath column is the main column strong_labels_df['audiofilepath'] = strong_labels_df[ 'audiofilepath'].apply(os.path.basename) colname = 'audiofilepath' # AVE else: colname = 'filename' # Dcase etc. weak_labels_df = strong_labels_df.groupby( colname)['event_label'].unique().apply( tuple).to_frame().reset_index() if "event_labels" in strong_labels_df.columns: assert False, "Data with the column event_labels are used to train not to evaluate" weak_labels_array, encoder = utils.encode_labels( labels=weak_labels_df['event_label'], encoder=encoder) # assert (weak_labels_df['encoded'].apply(lambda x: sum(x)) > # 0).all(), "No targets found, is the encoder maybe not right?" for k, v in config_parameters.items(): logger.info(f"{k}:{v}") dataloader = dataset.getdataloader( { 'filename': weak_labels_df['filename'].values, 'encoded': weak_labels_array }, config_parameters['data'], batch_size=1, shuffle=False, colname=config_parameters[ 'colname'], # For other datasets with different key names num_workers=3, ) model = getattr(models, config_parameters['model'])( inputdim=dataloader.dataset.datadim, outputdim=len(encoder.classes_), **config_parameters['model_args']) model.load_state_dict(model_parameters) model = model.to(DEVICE).eval() y_pred, y_true = [], [] with torch.no_grad(): for batch in tqdm(dataloader, unit='file', leave=False): _, target, filenames = batch clip_pred, _, _ = self._forward(model, batch) clip_pred = clip_pred.cpu().detach().numpy() y_pred.append(clip_pred) y_true.append(target.numpy()) y_pred = np.concatenate(y_pred) y_true = np.concatenate(y_true) mAP = np.nan_to_num(metrics.mAP(y_true, y_pred)) auc = np.nan_to_num(metrics.roc(y_true, y_pred)) with open( os.path.join(experiment_path, tag_file.format(test_data_filename)), 'w') as wp: print(f"mAP:{mAP.mean():.3f}", file=wp) print(f"mAP:\n{mAP.mean():.3f}") print(f"AuC:{auc.mean():.3f}", file=wp) print(f"AuC:\n{auc.mean():.3f}")
def evaluate( self, experiment_path: str, pred_file='hard_predictions_{}.txt', tag_file='tagging_predictions_{}.txt', event_file='event_{}.txt', segment_file='segment_{}.txt', class_result_file='class_result_{}.txt', time_ratio=10. / 500, postprocessing='double', threshold=None, window_size=None, save_seq=False, sed_eval=True, # Do evaluation on sound event detection ( time stamps, segemtn/evaluation based) **kwargs): """evaluate :param experiment_path: Path to already trained model using train :type experiment_path: str :param pred_file: Prediction output file, put into experiment dir :param time_resolution: Resolution in time (1. represents the model resolution) :param **kwargs: Overwrite standard args, please pass `data` and `label` """ # Update config parameters with new kwargs config = torch.load(list(Path(f'{experiment_path}').glob("run_config*"))[0], map_location='cpu') # Use previous config, but update data such as kwargs config_parameters = dict(config, **kwargs) # Default columns to search for in data config_parameters.setdefault('colname', ('filename', 'encoded')) model_parameters = torch.load( glob.glob("{}/run_model*".format(experiment_path))[0], map_location=lambda storage, loc: storage) encoder = torch.load(glob.glob( '{}/run_encoder*'.format(experiment_path))[0], map_location=lambda storage, loc: storage) strong_labels_df = pd.read_csv(config_parameters['label'], sep='\t') # Evaluation is done via the filenames, not full paths if not np.issubdtype(strong_labels_df['filename'].dtype, np.number): strong_labels_df['filename'] = strong_labels_df['filename'].apply( os.path.basename) if 'audiofilepath' in strong_labels_df.columns: # In case of ave dataset, the audiofilepath column is the main column strong_labels_df['audiofilepath'] = strong_labels_df[ 'audiofilepath'].apply(os.path.basename) colname = 'audiofilepath' # AVE else: colname = 'filename' # Dcase etc. # Problem is that we iterate over the strong_labels_df, which is ambigious # In order to conserve some time and resources just reduce strong_label to weak_label format weak_labels_df = strong_labels_df.groupby( colname)['event_label'].unique().apply( tuple).to_frame().reset_index() if "event_labels" in strong_labels_df.columns: assert False, "Data with the column event_labels are used to train not to evaluate" weak_labels_array, encoder = utils.encode_labels( labels=weak_labels_df['event_label'], encoder=encoder) dataloader = dataset.getdataloader( { 'filename': weak_labels_df['filename'].values, 'encoded': weak_labels_array, }, config_parameters['data'], batch_size=1, shuffle=False, colname=config_parameters[ 'colname'] # For other datasets with different key names ) model = getattr(models, config_parameters['model'])( inputdim=dataloader.dataset.datadim, outputdim=len(encoder.classes_), **config_parameters['model_args']) model.load_state_dict(model_parameters) model = model.to(DEVICE).eval() time_predictions, clip_predictions = [], [] sequences_to_save = [] mAP_pred, mAP_tar = [], [] with torch.no_grad(): for batch in tqdm(dataloader, unit='file', leave=False): _, target, filenames = batch clip_pred, pred, _ = self._forward(model, batch) clip_pred = clip_pred.cpu().detach().numpy() mAP_tar.append(target.numpy().squeeze(0)) mAP_pred.append(clip_pred.squeeze(0)) pred = pred.cpu().detach().numpy() if postprocessing == 'median': if threshold is None: thres = 0.5 else: thres = threshold if window_size is None: window_size = 1 filtered_pred = utils.median_filter( pred, window_size=window_size, threshold=thres) decoded_pred = utils.decode_with_timestamps( encoder, filtered_pred) elif postprocessing == 'cATP-SDS': # cATP-SDS postprocessing uses an "Optimal" configurations, assumes we have a prior # Values are taken from the Surface Disentange paper # Classes are (DCASE2018 only) # ['Alarm_bell_ringing' 'Blender' 'Cat' 'Dishes' 'Dog' # 'Electric_shaver_toothbrush' 'Frying' 'Running_water' 'Speech' # 'Vacuum_cleaner'] assert pred.shape[ -1] == 10, "Only supporting DCASE2018 for now" if threshold is None: thres = 0.5 else: thres = threshold if window_size is None: window_size = [17, 42, 17, 9, 16, 74, 85, 64, 18, 87] # P(y|x) > alpha clip_pred = utils.binarize(clip_pred, threshold=thres) pred = pred * clip_pred filtered_pred = np.zeros_like(pred) # class specific filtering via median filter for cl in range(pred.shape[-1]): # Median filtering also applies thresholding filtered_pred[..., cl] = utils.median_filter( pred[..., cl], window_size=window_size[cl], threshold=thres) decoded_pred = utils.decode_with_timestamps( encoder, filtered_pred) elif postprocessing == 'double': # Double thresholding as described in # https://arxiv.org/abs/1904.03841 if threshold is None: hi_thres, low_thres = (0.75, 0.2) else: hi_thres, low_thres = threshold filtered_pred = utils.double_threshold(pred, high_thres=hi_thres, low_thres=low_thres) decoded_pred = utils.decode_with_timestamps( encoder, filtered_pred) elif postprocessing == 'triple': # Triple thresholding as described in # Using frame level + clip level predictions if threshold is None: clip_thres, hi_thres, low_thres = (0.5, 0.75, 0.2) else: clip_thres, hi_thres, low_thres = threshold clip_pred = utils.binarize(clip_pred, threshold=clip_thres) # Apply threshold to pred = clip_pred * pred filtered_pred = utils.double_threshold(pred, high_thres=hi_thres, low_thres=low_thres) decoded_pred = utils.decode_with_timestamps( encoder, filtered_pred) for num_batch in range(len(decoded_pred)): filename = filenames[num_batch] cur_pred = pred[num_batch] cur_clip = clip_pred[num_batch].reshape(1, -1) # Clip predictions, independent of per-frame predictions bin_clips = utils.binarize(cur_clip) # Binarize with default threshold 0.5 For clips bin_clips = encoder.inverse_transform( bin_clips.reshape(1, -1))[0] # 0 since only single sample # Add each label individually into list for clip_label in bin_clips: clip_predictions.append({ 'filename': filename, 'event_label': clip_label, }) # Save each frame output, for later visualization if save_seq: labels = weak_labels_df.loc[weak_labels_df['filename'] == filename]['event_label'] to_save_df = pd.DataFrame(pred[num_batch], columns=encoder.classes_) # True labels to_save_df.rename({'variable': 'event'}, axis='columns', inplace=True) to_save_df['filename'] = filename to_save_df['pred_labels'] = np.array(labels).repeat( len(to_save_df)) sequences_to_save.append(to_save_df) label_prediction = decoded_pred[num_batch] for event_label, onset, offset in label_prediction: time_predictions.append({ 'filename': filename, 'event_label': event_label, 'onset': onset, 'offset': offset }) assert len(time_predictions) > 0, "No outputs, lower threshold?" pred_df = pd.DataFrame( time_predictions, columns=['filename', 'event_label', 'onset', 'offset']) clip_pred_df = pd.DataFrame( clip_predictions, columns=['filename', 'event_label', 'probability']) test_data_filename = os.path.splitext( os.path.basename(config_parameters['label']))[0] if save_seq: pd.concat(sequences_to_save).to_csv(os.path.join( experiment_path, 'probabilities.csv'), index=False, sep='\t', float_format="%.4f") pred_df = utils.predictions_to_time(pred_df, ratio=time_ratio) if pred_file: pred_df.to_csv(os.path.join(experiment_path, pred_file.format(test_data_filename)), index=False, sep="\t") tagging_df = metrics.audio_tagging_results(strong_labels_df, pred_df) clip_tagging_df = metrics.audio_tagging_results( strong_labels_df, clip_pred_df) print("Tagging Classwise Result: \n{}".format( tabulate(clip_tagging_df, headers='keys', showindex=False, tablefmt='github'))) print("mAP: {}".format( metrics.mAP(np.array(mAP_tar), np.array(mAP_pred)))) if tag_file: clip_tagging_df.to_csv(os.path.join( experiment_path, tag_file.format(test_data_filename)), index=False, sep='\t') if sed_eval: event_result, segment_result = metrics.compute_metrics( strong_labels_df, pred_df, time_resolution=1.0) print("Event Based Results:\n{}".format(event_result)) event_results_dict = event_result.results_class_wise_metrics() class_wise_results_df = pd.DataFrame().from_dict({ f: event_results_dict[f]['f_measure'] for f in event_results_dict.keys() }).T class_wise_results_df.to_csv(os.path.join( experiment_path, class_result_file.format(test_data_filename)), sep='\t') print("Class wise F1-Macro:\n{}".format( tabulate(class_wise_results_df, headers='keys', tablefmt='github'))) if event_file: with open( os.path.join(experiment_path, event_file.format(test_data_filename)), 'w') as wp: wp.write(event_result.__str__()) print("=" * 100) print(segment_result) if segment_file: with open( os.path.join(experiment_path, segment_file.format(test_data_filename)), 'w') as wp: wp.write(segment_result.__str__()) event_based_results = pd.DataFrame( event_result.results_class_wise_average_metrics()['f_measure'], index=['event_based']) segment_based_results = pd.DataFrame( segment_result.results_class_wise_average_metrics() ['f_measure'], index=['segment_based']) result_quick_report = pd.concat(( event_based_results, segment_based_results, )) # Add two columns tagging_macro_f1, tagging_macro_pre, tagging_macro_rec = tagging_df.loc[ tagging_df['label'] == 'macro'].values[0][1:] static_tagging_macro_f1, static_tagging_macro_pre, static_tagging_macro_rec = clip_tagging_df.loc[ clip_tagging_df['label'] == 'macro'].values[0][1:] result_quick_report.loc['Time Tagging'] = [ tagging_macro_f1, tagging_macro_pre, tagging_macro_rec ] result_quick_report.loc['Clip Tagging'] = [ static_tagging_macro_f1, static_tagging_macro_pre, static_tagging_macro_rec ] with open( os.path.join( experiment_path, 'quick_report_{}.md'.format(test_data_filename)), 'w') as wp: print(tabulate(result_quick_report, headers='keys', tablefmt='github'), file=wp) print("Quick Report: \n{}".format( tabulate(result_quick_report, headers='keys', tablefmt='github')))
predictions = [] masks = [] with tqdm(total=len(val), leave=False) as pbar: for id in val: _, mask, _ = dataset.get_by_id(id) test_prediction = np.concatenate([predictions[id] for predictions in test_predictions_experiment], axis=0) prediction = torch.FloatTensor(test_prediction) mask = torch.FloatTensor(mask) predictions.append(prediction) masks.append(mask) predictions = torch.stack(predictions, dim=0).cuda() masks = torch.stack(masks, dim=0).cuda() predictions = torch.sigmoid(predictions) predictions = torch.mean(predictions, dim=1) test_predictions = utils.TestPredictions(output, mode='val') test_predictions.add_predictions(zip(predictions.cpu().numpy(), train_samples)) if output: test_predictions.save() predictions = (predictions > 0.5).float() map = metrics.mAP(predictions, masks) split_map.append(map) print(np.mean(split_map), split_map)