def estimate_signatures_softmax(make_model, qs, distances_matrix, r_matrix, dists, iters=50, learning_rate=0.1, loss_eval_interval=5, draw_interval=20, prev_total_loss_story=None, different_d_sum=False, print_results=False, loss_name="softmax"):
total_loss_story = list() if prev_total_loss_story is None else prev_total_loss_story
for dist in dists:
d = (make_model(dist.d_sum) if different_d_sum else make_model())
loss_story = list()
opt = tf.keras.optimizers.Adam(learning_rate=learning_rate)
for i in tqdm.tqdm_notebook(range(iters)):
loss = lambda : d.get_matrix_loss(
qs,
None,
r_matrix,
distance=dist,
loss=loss_name,
symmetric=True
)
distortion = loss().numpy()
print(f"{i}:{distortion}")
m = opt.minimize(
loss,
var_list=d.get_weights() + dist.get_weights()
)
if (i % loss_eval_interval == 0) or (i + 1 == iters):
pred = d.get_matrix_loss(
qs,
qs,
None,
distance=dist,
loss=None,
symmetric=True
)
loss_story.append((
round(distortion, 7),
round(metrics.mAP(pred, r_matrix, True), 7),
round(metrics.NDCG_graph_v1(pred, distances_matrix), 7)
))
if (i % draw_interval == 0) or (i + 1 == iters):
plt.figure(figsize=(10, 5))
plt.title(f"|V| = {qs.shape[0]}")
for i, prev_loss_story in enumerate(total_loss_story):
plt.plot([x[-2] for x in prev_loss_story], label=f"{dists[i]}:{prev_loss_story[-1]}")
plt.plot([x[-2] for x in loss_story], label=f"{dist}:{loss_story[-1]}")
plt.legend()
plt.grid()
clear_output()
plt.savefig(strftime(f"./pngs/softmax-V-{qs.shape[0]}--%Y-%m-%d-%Hh.png", gmtime()))
plt.show()
# print(np.mean(loss_story[-20:]))
total_loss_story.append(loss_story)
if print_results:
for d, l in zip(dists, total_loss_story):
print(f"{l[-1][1]}\t{d}")
return total_loss_story
def update_pbar(self, masks_predictions, masks_targets, pbar, average_meter, pbar_description):
average_meter.add('iou', iou(masks_predictions > 0.5, masks_targets.byte()))
average_meter.add('mAP', mAP(masks_predictions > 0.5, masks_targets.byte()))
pbar.set_description(
pbar_description + ''.join(
[' {}:{:6.4f}'.format(k, v) for k, v in average_meter.get_all().items()]
)
)
pbar.update()
def update(self, predictions, targets):
optimal_map_score = 0
optimal_t = 0
for i in range(1, 10):
t = 1 / i
map_score = metrics.mAP(predictions > t, targets.byte())
if map_score > optimal_map_score:
optimal_map_score = map_score
optimal_t = t
self.meter.add('threshold', optimal_t)
return optimal_map_score, optimal_t
def evaluation_metrics(self):
self.q_features = l2_norm_standardize(self.q_features)
self.g_features = l2_norm_standardize(self.g_features)
# Scores against all feature vectors in the gallery image for each
# image in the query data.
scores = joint_scores(self.q_features, self.q_cam_ids, self.q_frames,
self.g_features, self.g_cam_ids, self.g_frames,
self.trainer.datamodule.st_distribution)
if self.rerank:
scores = re_ranking(scores)
# Metrics & Evaluation
mean_ap, cmc = mAP(scores, self.q_targets, self.q_cam_ids,
self.g_targets, self.g_cam_ids)
return mean_ap, cmc
H = []
for images, _ in train_loader:
in_aff, out_aff = rbf_affnty(images, anchor, topk=20)
images = Variable(images).cuda()
in_aff = Variable(in_aff).cuda()
out_aff = Variable(out_aff).cuda()
out, _ = gcn(images, in_aff, out_aff, 1)
H.append(out.data.cpu().numpy())
R = np.concatenate(H)
H = np.sign(R)
HammTrainTest = 0.5 * (n_bits - np.dot(H, tH.T))
HammingRank = np.argsort(HammTrainTest, axis=0)
ap = mAP(dset.cateTrainTest, HammingRank)
pre, rec = topK(dset.cateTrainTest, HammingRank, 500)
print('%d bits: mAP: %.4f, precision@500: %.4f, recall@500: %.4F' %
(n_bits, ap, pre, rec))
'''
torch.save({'state_dict':gcn.state_dict(),
'anchor': anchor,
'anchor_affnty': anchor_affnty}, './%s_%d_%d_%.4f_%.4f' % (dataset,
n_labeled, n_bits, ap, pre))
import scipy.io as sio
sio.savemat('./%s_%d_%d' % (dataset, n_labeled, n_bits), {'H': H, 'tH': tH,
'R': R, 'tR': tR})
'''
def test(net,
dataloader,
criterion=None,
evaluate=True,
predict=True,
device=torch.device('cuda:0'),
num_class=2):
'''
使用训练好的net进行预测或评价
args：
net，训练好的RetinaNet对象；
dataloader，需要进行预测或评价的数据集的dataloader；
criterion，计算损失的函数，如果是None则不计算loss；
evaluate，如果是True则进行模型的评价（即计算mAP），如果
是False，则只进行预测或计算loss；
predict，是否进行预测，如果True则会返回预测框的类别和坐标，如果False，
则不会返回；
returns：
all_labels_pred, all_markers_pred，如果predict=True，则返回预测的每张图
片的预测框的标签和loc；
losses, cls_losses, loc_losses，如果criterion不是None，则得到其3个loss；
APs, mAP_score，如果evaluate=True，则返回每一类的AP值和其mAP；
'''
data_encoder = dataloader.dataset.y_encoder
y_encoder_mode = dataloader.dataset.y_encoder_mode
assert (evaluate and y_encoder_mode in ['object', 'all']) or not evaluate
assert (criterion is not None and y_encoder_mode in ['all', 'anchor']) or \
criterion is None
print('Testing ...')
results = []
with torch.no_grad():
losses = 0.
cls_losses = 0.
loc_losses = 0.
all_labels_pred = [] # 预测的labels，
all_markers_pred = []
all_labels_true = []
all_markers_true = []
for imgs, labels, markers in pb.progressbar(dataloader):
imgs = imgs.to(device)
if y_encoder_mode == 'all':
# 如果需要计算mAP，则除了dataloader除了输出已经编码好的每个
# anchor对应的偏移量和对应的标签，还需要没有编码的每张图片
# 对应的obj loc tensor和obj label tensor，因为每张图片对应的
# objs的数量不同，所以只能使用list来储存（通过重写dataloader
# 的collate_fn）
# labels是list，每个元素是一个tensor，指代一张图片中所有的obj
# 的label，[(img1#obj,), (img2#obj,), (img3#obj,), ...]
# cls_trues是一个tensor，(#imgs, #anchors, #class)，是每张图片
# 中每个anchor对应的gtbb的类别
labels, cls_trues = labels
# labels in cpu, cls_trues in gpu
cls_trues = cls_trues.to(device)
# markers是list，每个元素是一个tensor，指代一张图片中所有的obj
# 的xyxy loc，[(img1#obj, 4), (img2#obj, 4), (img3#obj, 4),
# ...]
# loc_trues是一个tensor，(#imgs, #anchors, 4)，是每张图片中每个
# anchor和其对应的gtbb的位置偏移量
markers, loc_trues = markers
# marker in cpu, loc_trues in gpu
loc_trues = loc_trues.to(device)
elif y_encoder_mode == 'anchor':
# 如果只需要计算loss，则只需要输出编码好的偏移量和anchor的标签
# 即可
cls_trues = labels.to(device)
loc_trues = markers.to(device)
cls_preds, loc_preds = net(imgs)
if criterion is not None:
loss, cls_loss, loc_loss = criterion(cls_trues, loc_trues,
cls_preds, loc_preds)
losses += loss.item()
cls_losses += cls_loss.item()
loc_losses += loc_loss.item()
# 使用decode得到的是list，其每个元素是batch中每个图片的预测框的
# tensor
label_preds, marker_preds = data_encoder.decode(cls_preds,
loc_preds,
device=device)
# 使用append则输出的的list的len是图片的数量，这样的输出更加明白
all_markers_pred.append(marker_preds)
all_labels_pred.append(label_preds)
if evaluate:
all_labels_true.append(labels)
all_markers_true.append(markers)
if predict:
results.append((all_labels_pred, all_markers_pred))
if criterion is not None:
losses = losses / len(dataloader.dataset)
cls_losses = cls_losses / len(dataloader.dataset)
loc_losses = loc_losses / len(dataloader.dataset)
results.append((losses, cls_losses, loc_losses))
if evaluate:
# 使用chain将两层的嵌套list变成一层，符合mAP函数的输出要求
APs, mAP_score = mAP(list(chain.from_iterable(all_labels_true)),
list(chain.from_iterable(all_markers_true)),
list(chain.from_iterable(all_labels_pred)),
list(chain.from_iterable(all_markers_pred)),
num_class=num_class)
results.append((APs, mAP_score))
return tuple(results)
def train(net,
criterion,
optimizer,
dataloaders,
epoch,
lr_schedular=None,
clip_norm=None,
best_num=1,
best_metric='mAP',
history=History(),
num_class=2):
'''
对RetinaNet进行训练，
args：
net，实例化的RetinaNet网络，在训练的过程中其发生了改变；
criterion，loss，即实例化的FocalLoss对象；
optimizer，优化器；
dataloaders，dataloaders组成的dict，其key必须有'train'，可以有'valid'，
'test'，如果有test则会在最后使用训练过程中最好的net进行预测；
epoch，int，一共需要进行多少个epoch的训练；
lr_schedular，使用的学习率策略；
clip_norm, 使用的梯度截断的参数，如果是None则不进行梯度截断；
best_num，要保存的最好的模型的数目；
best_metric，来判断模型好坏的指标，可以选择mAP或loss；
history，History对象，用于储存训练时期的结果，默认是History()的默认配置；
returns：
history，History对象，储存每个epoch train的loss和valid的loss、mAP和最好
的几个模型；
（另外，实际上输入的net会发生改变，成为在训练过程中valid上最好的net）
test_loss, test_mAP，如果有dataloaders中还有test，则会返回；
'''
for e in range(epoch):
if 'valid' in dataloaders.keys():
phases = ['train', 'valid']
else:
phases = ['train']
for phase in phases:
# 计算每个epoch的loss
epoch_loss = 0.
epoch_cls_loss = 0.
epoch_loc_loss = 0.
# 分不同的phase进行处理
if phase == 'train':
net.train()
if lr_schedular is not None:
lr_schedular.step()
widgets = [
'epoch: %d' % e,
'| ',
pb.Counter(),
pb.Bar(),
pb.AdaptiveETA(),
]
iterator = pb.progressbar(dataloaders[phase], widgets=widgets)
else:
net.eval()
iterator = dataloaders[phase]
# 进行一个epoch中所有batches的迭代
y_encoder = dataloaders[phase].dataset.y_encoder
all_label_preds = []
all_marker_preds = []
all_label_trues = []
all_marker_trues = []
for imgs, labels, markers in iterator:
imgs = imgs.cuda()
if phase == 'train':
cls_trues = labels.cuda()
loc_trues = markers.cuda()
else:
labels, cls_trues = labels
cls_trues = cls_trues.cuda()
markers, loc_trues = markers
loc_trues = loc_trues.cuda()
if phase == 'train':
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
cls_preds, loc_preds = net(imgs)
loss, cls_loss, loc_loss = criterion(
cls_trues, loc_trues, cls_preds, loc_preds)
if phase == 'train':
loss.backward()
if clip_norm != 0.0:
torch.nn.utils.clip_grad_norm_(
net.parameters(), clip_norm)
optimizer.step()
else:
label_preds, marker_preds = y_encoder.decode(
cls_preds, loc_preds)
all_label_preds.extend(label_preds)
all_marker_preds.extend(marker_preds)
all_label_trues.extend(labels)
all_marker_trues.extend(markers)
with torch.no_grad():
epoch_loss += loss.item()
epoch_cls_loss += cls_loss.item()
epoch_loc_loss += loc_loss.item()
# 每个epoch的所有batches都结束，计算在epoch上的loss平均
with torch.no_grad():
num_batches = len(dataloaders[phase].dataset)
epoch_loss /= num_batches
epoch_cls_loss /= num_batches
epoch_loc_loss /= num_batches
if phase == 'train':
history.update_hist(loss=epoch_loss,
cls_loss=epoch_cls_loss,
loc_loss=epoch_loc_loss)
print('%s, loss: %.4f, cls_loss: %.4f, loc_loss: %.4f' %
(phase, epoch_loss, epoch_cls_loss, epoch_loc_loss))
elif phase == 'valid':
_, map_score = mAP(all_label_trues,
all_marker_trues,
all_label_preds,
all_marker_preds,
num_class=num_class)
history.update_hist(val_loss=epoch_loss,
val_cls_loss=epoch_cls_loss,
val_loc_loss=epoch_loc_loss,
mAP=map_score)
if map_score is np.nan:
map_score = 0
history.update_best(mAP=map_score,
loss=epoch_loss,
cls_loss=epoch_cls_loss,
loc_loss=epoch_loc_loss,
epoch=e,
model_wts=copy.deepcopy(
net.state_dict()))
print(('%s, loss: %.4f, cls_loss: %.4f,'
' loc_loss: %.4f, mAP: %.4f') %
(phase, epoch_loss, epoch_cls_loss, epoch_loc_loss,
map_score))
if 'valid' in dataloaders.keys():
net.load_state_dict(history.best[0]['model_wts'])
print('valid best loss: %.4f, best mAP: %.4f' %
(history.best[0]['loss'], history.best[0]['mAP']))
# 如果有test，则再进行test的预测
if 'test' in dataloaders.keys():
test_loss, test_map = test(net,
dataloaders['test'],
criterion=criterion,
evaluate=True,
predict=False,
num_class=num_class)
print('test loss: %.4f, test mAP: %.4f' % (test_loss[0], test_map[1]))
return history, (test_loss, test_map)
return history
def train(net,
criterion,
optimizer,
dataloaders,
epoch,
device=torch.device('cuda:0'),
hist_args={}):
history = History(**hist_args)
for e in range(epoch):
epoch_loss = 0.
for phase in ['train', 'valid']:
y_encoder = dataloaders[phase].dataset.y_encoder
if phase == 'train':
net.train()
prefix = '%d, Training: ' % e
else:
net.eval()
prefix = '%d, Validation: ' % e
# validation时需要记录所有的真实标签和预测
epoch_label, epoch_loc = [], []
epoch_res_c, epoch_res_s, epoch_res_l = [], [], []
for batch in pb.progressbar(dataloaders[phase], prefix=prefix):
imgs, targets = batch[0].to(device), batch[-1].to(device)
with torch.set_grad_enabled(phase == 'train'):
preds = net(imgs)
loss = criterion(preds, targets)
if phase == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
# 如果是eval，需要记录所有的预测，用于计算mAP
img_size = list(imgs.shape[-1:-3:-1])
epoch_label.extend(batch[1])
epoch_loc.extend(batch[2])
for pred in preds:
res_c, res_s, res_l = y_encoder.decode(
pred, img_size)
epoch_res_c.append(res_c)
epoch_res_s.append(res_s)
epoch_res_l.append(res_l)
with torch.no_grad():
epoch_loss += loss.item()
# 整个epoch的所有batches循环结束
with torch.no_grad():
num_batches = len(dataloaders[phase].dataset)
epoch_loss /= num_batches
if phase == 'train':
history.update_hist(loss=epoch_loss)
print('%d, %s, loss: %.4f' % (e, phase, epoch_loss))
else:
aps, map_score = mAP(epoch_label, epoch_loc, epoch_res_c,
epoch_res_s, epoch_res_l)
history.update_hist(val_loss=epoch_loss, mAP=map_score)
if map_score is np.nan:
map_score = 0
history.update_best(mAP=map_score,
loss=epoch_loss,
epoch=e,
model_wts=copy.deepcopy(
net.state_dict()))
print('%d, %s, loss: %.4f, mAP: %.4f' %
(e, phase, epoch_loss, map_score))
if 'valid' in dataloaders.keys():
net.load_state_dict(history.best[0]['model_wts'])
print('valid best loss: %.4f, best mAP: %.4f' %
(history.best[0]['loss'], history.best[0]['mAP']))
return history
def test(net,
dataloader,
criterion=None,
evaluate=True,
predict=True,
device=torch.device('cuda:0')):
'''
使用训练好的net进行预测或评价
args：
net，训练好的RetinaNet对象；
dataloader，需要进行预测或评价的数据集的dataloader；
criterion，计算损失的函数，如果是None则不计算loss；
evaluate，如果是True则进行模型的评价（即计算mAP），如果
是False，则只进行预测或计算loss；
predict，是否进行预测，如果True则会返回预测框的类别和坐标，如果False，
则不会返回；
returns：
all_labels_pred, all_markers_pred，如果predict=True，则返回预测的每张图
片的预测框的标签和loc；
losses, cls_losses, loc_losses，如果criterion不是None，则得到其3个loss；
APs, mAP_score，如果evaluate=True，则返回每一类的AP值和其mAP；
'''
y_encoder = dataloader.dataset.y_encoder
y_encoder_mode = dataloader.dataset.out
assert (evaluate and y_encoder_mode in ['obj', 'all']) or not evaluate
assert (criterion is not None and y_encoder_mode in ['all', 'encode']) or \
criterion is None
print('Testing ...')
results = []
with torch.no_grad():
losses = 0.
all_labels_pred = [] # 预测的labels，
all_scores_pred = []
all_markers_pred = []
all_labels_true = []
all_markers_true = []
for batch in pb.progressbar(dataloader):
imgs = batch[0].to(device)
if y_encoder_mode in ['all', 'obj']:
labels_true, locs_true = batch[1:3]
if y_encoder_mode in ['all', 'encode']:
targets = batch[-1].to(device)
preds = net(imgs)
if criterion is not None:
loss = criterion(preds, targets)
losses += loss.item()
if predict or evaluate:
img_size = list(imgs.shape[-1:-3:-1])
for pred in preds:
res_c, res_s, res_l = y_encoder.decode(pred, img_size)
all_labels_pred.append(res_c)
all_scores_pred.append(res_s)
all_markers_pred.append(res_l)
if evaluate:
all_labels_true.extend(labels_true)
all_markers_true.extend(locs_true)
if predict:
results.append((all_labels_pred, all_markers_pred))
if criterion is not None:
losses = losses / len(dataloader.dataset)
results.append(losses)
if evaluate:
# 使用chain将两层的嵌套list变成一层，符合mAP函数的输出要求
APs, mAP_score = mAP(
all_labels_true,
all_markers_true,
all_labels_pred,
all_scores_pred,
all_markers_pred,
)
results.append((APs, mAP_score))
return tuple(results)
def evaluate_tagging(self,
experiment_path: str,
tag_file='tagging_predictions_{}.txt',
**kwargs):
exppath = Path(experiment_path)
if exppath.is_file(): # Best model passed!
model_parameters = torch.load(
str(exppath),
map_location=lambda storage, loc: storage)
experiment_path = exppath.parent # Just set upper path as default
else:
model_parameters = torch.load(
glob.glob("{}/run_model*".format(experiment_path))[0],
map_location=lambda storage, loc: storage)
config = torch.load(glob.glob(
"{}/run_config*".format(experiment_path))[0],
map_location=lambda storage, loc: storage)
logger = utils.getfile_outlogger(None)
# Use previous config, but update data such as kwargs
config_parameters = dict(config, **kwargs)
# Default columns to search for in data
config_parameters.setdefault('colname', ('filename', 'encoded'))
encoder = torch.load(glob.glob(
'{}/run_encoder*'.format(experiment_path))[0],
map_location=lambda storage, loc: storage)
test_data_filename = os.path.splitext(
os.path.basename(config_parameters['label']))[0]
strong_labels_df = pd.read_csv(config_parameters['label'], sep='\s+')
# Evaluation is done via the filenames, not full paths
if not np.issubdtype(strong_labels_df['filename'].dtype, np.number):
strong_labels_df['filename'] = strong_labels_df['filename'].apply(
os.path.basename)
if 'audiofilepath' in strong_labels_df.columns: # In case of ave dataset, the audiofilepath column is the main column
strong_labels_df['audiofilepath'] = strong_labels_df[
'audiofilepath'].apply(os.path.basename)
colname = 'audiofilepath' # AVE
else:
colname = 'filename' # Dcase etc.
weak_labels_df = strong_labels_df.groupby(
colname)['event_label'].unique().apply(
tuple).to_frame().reset_index()
if "event_labels" in strong_labels_df.columns:
assert False, "Data with the column event_labels are used to train not to evaluate"
weak_labels_array, encoder = utils.encode_labels(
labels=weak_labels_df['event_label'], encoder=encoder)
# assert (weak_labels_df['encoded'].apply(lambda x: sum(x)) >
# 0).all(), "No targets found, is the encoder maybe not right?"
for k, v in config_parameters.items():
logger.info(f"{k}:{v}")
dataloader = dataset.getdataloader(
{
'filename': weak_labels_df['filename'].values,
'encoded': weak_labels_array
},
config_parameters['data'],
batch_size=1,
shuffle=False,
colname=config_parameters[
'colname'], # For other datasets with different key names
num_workers=3,
)
model = getattr(models, config_parameters['model'])(
inputdim=dataloader.dataset.datadim,
outputdim=len(encoder.classes_),
**config_parameters['model_args'])
model.load_state_dict(model_parameters)
model = model.to(DEVICE).eval()
y_pred, y_true = [], []
with torch.no_grad():
for batch in tqdm(dataloader, unit='file', leave=False):
_, target, filenames = batch
clip_pred, _, _ = self._forward(model, batch)
clip_pred = clip_pred.cpu().detach().numpy()
y_pred.append(clip_pred)
y_true.append(target.numpy())
y_pred = np.concatenate(y_pred)
y_true = np.concatenate(y_true)
mAP = np.nan_to_num(metrics.mAP(y_true, y_pred))
auc = np.nan_to_num(metrics.roc(y_true, y_pred))
with open(
os.path.join(experiment_path,
tag_file.format(test_data_filename)), 'w') as wp:
print(f"mAP:{mAP.mean():.3f}", file=wp)
print(f"mAP:\n{mAP.mean():.3f}")
print(f"AuC:{auc.mean():.3f}", file=wp)
print(f"AuC:\n{auc.mean():.3f}")
def evaluate(
self,
experiment_path: str,
pred_file='hard_predictions_{}.txt',
tag_file='tagging_predictions_{}.txt',
event_file='event_{}.txt',
segment_file='segment_{}.txt',
class_result_file='class_result_{}.txt',
time_ratio=10. / 500,
postprocessing='double',
threshold=None,
window_size=None,
save_seq=False,
sed_eval=True, # Do evaluation on sound event detection ( time stamps, segemtn/evaluation based)
**kwargs):
"""evaluate
:param experiment_path: Path to already trained model using train
:type experiment_path: str
:param pred_file: Prediction output file, put into experiment dir
:param time_resolution: Resolution in time (1. represents the model resolution)
:param **kwargs: Overwrite standard args, please pass `data` and `label`
"""
# Update config parameters with new kwargs
config = torch.load(list(Path(f'{experiment_path}').glob("run_config*"))[0], map_location='cpu')
# Use previous config, but update data such as kwargs
config_parameters = dict(config, **kwargs)
# Default columns to search for in data
config_parameters.setdefault('colname', ('filename', 'encoded'))
model_parameters = torch.load(
glob.glob("{}/run_model*".format(experiment_path))[0],
map_location=lambda storage, loc: storage)
encoder = torch.load(glob.glob(
'{}/run_encoder*'.format(experiment_path))[0],
map_location=lambda storage, loc: storage)
strong_labels_df = pd.read_csv(config_parameters['label'], sep='\t')
# Evaluation is done via the filenames, not full paths
if not np.issubdtype(strong_labels_df['filename'].dtype, np.number):
strong_labels_df['filename'] = strong_labels_df['filename'].apply(
os.path.basename)
if 'audiofilepath' in strong_labels_df.columns: # In case of ave dataset, the audiofilepath column is the main column
strong_labels_df['audiofilepath'] = strong_labels_df[
'audiofilepath'].apply(os.path.basename)
colname = 'audiofilepath' # AVE
else:
colname = 'filename' # Dcase etc.
# Problem is that we iterate over the strong_labels_df, which is ambigious
# In order to conserve some time and resources just reduce strong_label to weak_label format
weak_labels_df = strong_labels_df.groupby(
colname)['event_label'].unique().apply(
tuple).to_frame().reset_index()
if "event_labels" in strong_labels_df.columns:
assert False, "Data with the column event_labels are used to train not to evaluate"
weak_labels_array, encoder = utils.encode_labels(
labels=weak_labels_df['event_label'], encoder=encoder)
dataloader = dataset.getdataloader(
{
'filename': weak_labels_df['filename'].values,
'encoded': weak_labels_array,
},
config_parameters['data'],
batch_size=1,
shuffle=False,
colname=config_parameters[
'colname'] # For other datasets with different key names
)
model = getattr(models, config_parameters['model'])(
inputdim=dataloader.dataset.datadim,
outputdim=len(encoder.classes_),
**config_parameters['model_args'])
model.load_state_dict(model_parameters)
model = model.to(DEVICE).eval()
time_predictions, clip_predictions = [], []
sequences_to_save = []
mAP_pred, mAP_tar = [], []
with torch.no_grad():
for batch in tqdm(dataloader, unit='file', leave=False):
_, target, filenames = batch
clip_pred, pred, _ = self._forward(model, batch)
clip_pred = clip_pred.cpu().detach().numpy()
mAP_tar.append(target.numpy().squeeze(0))
mAP_pred.append(clip_pred.squeeze(0))
pred = pred.cpu().detach().numpy()
if postprocessing == 'median':
if threshold is None:
thres = 0.5
else:
thres = threshold
if window_size is None:
window_size = 1
filtered_pred = utils.median_filter(
pred, window_size=window_size, threshold=thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
elif postprocessing == 'cATP-SDS':
# cATP-SDS postprocessing uses an "Optimal" configurations, assumes we have a prior
# Values are taken from the Surface Disentange paper
# Classes are (DCASE2018 only)
# ['Alarm_bell_ringing' 'Blender' 'Cat' 'Dishes' 'Dog'
# 'Electric_shaver_toothbrush' 'Frying' 'Running_water' 'Speech'
# 'Vacuum_cleaner']
assert pred.shape[
-1] == 10, "Only supporting DCASE2018 for now"
if threshold is None:
thres = 0.5
else:
thres = threshold
if window_size is None:
window_size = [17, 42, 17, 9, 16, 74, 85, 64, 18, 87]
# P(y|x) > alpha
clip_pred = utils.binarize(clip_pred, threshold=thres)
pred = pred * clip_pred
filtered_pred = np.zeros_like(pred)
# class specific filtering via median filter
for cl in range(pred.shape[-1]):
# Median filtering also applies thresholding
filtered_pred[..., cl] = utils.median_filter(
pred[..., cl],
window_size=window_size[cl],
threshold=thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
elif postprocessing == 'double':
# Double thresholding as described in
# https://arxiv.org/abs/1904.03841
if threshold is None:
hi_thres, low_thres = (0.75, 0.2)
else:
hi_thres, low_thres = threshold
filtered_pred = utils.double_threshold(pred,
high_thres=hi_thres,
low_thres=low_thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
elif postprocessing == 'triple':
# Triple thresholding as described in
# Using frame level + clip level predictions
if threshold is None:
clip_thres, hi_thres, low_thres = (0.5, 0.75, 0.2)
else:
clip_thres, hi_thres, low_thres = threshold
clip_pred = utils.binarize(clip_pred, threshold=clip_thres)
# Apply threshold to
pred = clip_pred * pred
filtered_pred = utils.double_threshold(pred,
high_thres=hi_thres,
low_thres=low_thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
for num_batch in range(len(decoded_pred)):
filename = filenames[num_batch]
cur_pred = pred[num_batch]
cur_clip = clip_pred[num_batch].reshape(1, -1)
# Clip predictions, independent of per-frame predictions
bin_clips = utils.binarize(cur_clip)
# Binarize with default threshold 0.5 For clips
bin_clips = encoder.inverse_transform(
bin_clips.reshape(1,
-1))[0] # 0 since only single sample
# Add each label individually into list
for clip_label in bin_clips:
clip_predictions.append({
'filename': filename,
'event_label': clip_label,
})
# Save each frame output, for later visualization
if save_seq:
labels = weak_labels_df.loc[weak_labels_df['filename']
== filename]['event_label']
to_save_df = pd.DataFrame(pred[num_batch],
columns=encoder.classes_)
# True labels
to_save_df.rename({'variable': 'event'},
axis='columns',
inplace=True)
to_save_df['filename'] = filename
to_save_df['pred_labels'] = np.array(labels).repeat(
len(to_save_df))
sequences_to_save.append(to_save_df)
label_prediction = decoded_pred[num_batch]
for event_label, onset, offset in label_prediction:
time_predictions.append({
'filename': filename,
'event_label': event_label,
'onset': onset,
'offset': offset
})
assert len(time_predictions) > 0, "No outputs, lower threshold?"
pred_df = pd.DataFrame(
time_predictions,
columns=['filename', 'event_label', 'onset', 'offset'])
clip_pred_df = pd.DataFrame(
clip_predictions,
columns=['filename', 'event_label', 'probability'])
test_data_filename = os.path.splitext(
os.path.basename(config_parameters['label']))[0]
if save_seq:
pd.concat(sequences_to_save).to_csv(os.path.join(
experiment_path, 'probabilities.csv'),
index=False,
sep='\t',
float_format="%.4f")
pred_df = utils.predictions_to_time(pred_df, ratio=time_ratio)
if pred_file:
pred_df.to_csv(os.path.join(experiment_path,
pred_file.format(test_data_filename)),
index=False,
sep="\t")
tagging_df = metrics.audio_tagging_results(strong_labels_df, pred_df)
clip_tagging_df = metrics.audio_tagging_results(
strong_labels_df, clip_pred_df)
print("Tagging Classwise Result: \n{}".format(
tabulate(clip_tagging_df,
headers='keys',
showindex=False,
tablefmt='github')))
print("mAP: {}".format(
metrics.mAP(np.array(mAP_tar), np.array(mAP_pred))))
if tag_file:
clip_tagging_df.to_csv(os.path.join(
experiment_path, tag_file.format(test_data_filename)),
index=False,
sep='\t')
if sed_eval:
event_result, segment_result = metrics.compute_metrics(
strong_labels_df, pred_df, time_resolution=1.0)
print("Event Based Results:\n{}".format(event_result))
event_results_dict = event_result.results_class_wise_metrics()
class_wise_results_df = pd.DataFrame().from_dict({
f: event_results_dict[f]['f_measure']
for f in event_results_dict.keys()
}).T
class_wise_results_df.to_csv(os.path.join(
experiment_path, class_result_file.format(test_data_filename)),
sep='\t')
print("Class wise F1-Macro:\n{}".format(
tabulate(class_wise_results_df,
headers='keys',
tablefmt='github')))
if event_file:
with open(
os.path.join(experiment_path,
event_file.format(test_data_filename)),
'w') as wp:
wp.write(event_result.__str__())
print("=" * 100)
print(segment_result)
if segment_file:
with open(
os.path.join(experiment_path,
segment_file.format(test_data_filename)),
'w') as wp:
wp.write(segment_result.__str__())
event_based_results = pd.DataFrame(
event_result.results_class_wise_average_metrics()['f_measure'],
index=['event_based'])
segment_based_results = pd.DataFrame(
segment_result.results_class_wise_average_metrics()
['f_measure'],
index=['segment_based'])
result_quick_report = pd.concat((
event_based_results,
segment_based_results,
))
# Add two columns
tagging_macro_f1, tagging_macro_pre, tagging_macro_rec = tagging_df.loc[
tagging_df['label'] == 'macro'].values[0][1:]
static_tagging_macro_f1, static_tagging_macro_pre, static_tagging_macro_rec = clip_tagging_df.loc[
clip_tagging_df['label'] == 'macro'].values[0][1:]
result_quick_report.loc['Time Tagging'] = [
tagging_macro_f1, tagging_macro_pre, tagging_macro_rec
]
result_quick_report.loc['Clip Tagging'] = [
static_tagging_macro_f1, static_tagging_macro_pre,
static_tagging_macro_rec
]
with open(
os.path.join(
experiment_path,
'quick_report_{}.md'.format(test_data_filename)),
'w') as wp:
print(tabulate(result_quick_report,
headers='keys',
tablefmt='github'),
file=wp)
print("Quick Report: \n{}".format(
tabulate(result_quick_report,
headers='keys',
tablefmt='github')))
predictions = []
masks = []
with tqdm(total=len(val), leave=False) as pbar:
for id in val:
_, mask, _ = dataset.get_by_id(id)
test_prediction = np.concatenate([predictions[id] for predictions in test_predictions_experiment], axis=0)
prediction = torch.FloatTensor(test_prediction)
mask = torch.FloatTensor(mask)
predictions.append(prediction)
masks.append(mask)
predictions = torch.stack(predictions, dim=0).cuda()
masks = torch.stack(masks, dim=0).cuda()
predictions = torch.sigmoid(predictions)
predictions = torch.mean(predictions, dim=1)
test_predictions = utils.TestPredictions(output, mode='val')
test_predictions.add_predictions(zip(predictions.cpu().numpy(), train_samples))
if output:
test_predictions.save()
predictions = (predictions > 0.5).float()
map = metrics.mAP(predictions, masks)
split_map.append(map)
print(np.mean(split_map), split_map)
