def eval(self, mode):
# Validate the network on the val_loader (during training) or test_loader (for checking result)
# During training use this function for validation data.
self.set_mode(mode)
eval_loss = 0.0
avg_auc1 = []
avg_ap1 = []
avg_auc2 = []
avg_ap2 = []
for i, data in enumerate(self.dataloader):
audio = data['audio'].to(device)
label = data['label'].to(device)
outputs = self.samplecnn(audio)
loss = self.bce(outputs, label)
auc1, aprec1 = utils.tagwise_aroc_ap(label.cpu().detach().numpy(), outputs.cpu().detach().numpy())
avg_auc1.append(np.mean(auc1))
avg_ap1.append(np.mean(aprec1))
auc2, aprec2 = utils.itemwise_aroc_ap(label.cpu().detach().numpy(), outputs.cpu().detach().numpy())
avg_auc2.append(np.mean(auc2))
avg_ap2.append(np.mean(aprec2))
eval_loss += loss.item() # loss.data[0]
avg_loss = eval_loss/len(self.dataloader)
print ("Retrieval : Average AROC = %.3f, AP = %.3f / "%(np.mean(avg_auc1), np.mean(avg_ap1)), "Annotation : Average AROC = %.3f, AP = %.3f"%(np.mean(avg_auc2), np.mean(avg_ap2)))
print ('Average loss: {:.4f} \n'. format(avg_loss))
return avg_loss
def test():
avg_auc1 = []
avg_ap1 = []
avg_auc2 = []
avg_ap2 = []
acc = 0
n = 0
phar = tqdm(test_loader, unit='valid', unit_scale=test_loader.batch_size)
for audio, label in phar:
audio = audio.to(device)
label = label.to(device)
pred = sampleModel(audio)
auc1, aprec1 = utils.tagwise_aroc_ap(label.cpu().data.numpy(),
pred.cpu().data.numpy())
avg_auc1.append(np.mean(auc1))
avg_ap1.append(np.mean(aprec1))
auc2, aprec2 = utils.itemwise_aroc_ap(label.cpu().data.numpy(),
pred.cpu().data.numpy())
avg_auc2.append(np.mean(auc2))
avg_ap2.append(np.mean(aprec2))
acc += torch.sum(
torch.argmax(pred, dim=1) == torch.argmax(label, dim=1))
n += audio.size()[0]
phar.set_postfix({
'acc': acc.item() / n,
'Retrieval: AROC': np.mean(avg_auc1),
'Retrieval: AP ': np.mean(avg_ap1),
'Annotation: AROC ': np.mean(avg_auc2),
'Annotation: AP ': np.mean(avg_ap2),
})
print('acc', acc.item() / n)
def eval():
self.set_mode('valid')
eval_loss = 0.0
avg_auc1 = []
avg_ap1 = []
avg_auc2 = []
avg_ap2 = []
for i, data in enumerate(self.dataloader):
audio = data['audio'].to(device)
label = data['label'].to(device)
outputs = self.samplecnn(audio)
loss = self.bce(outputs, label)
auc1, aprec1 = utils.tagwise_aroc_ap(label.cpu().detach().numpy(),
outputs.cpu().detach.numpy())
avg_auc1.append(np.mean(auc1))
avg_ap1.append(np.mean(aprec1))
auc2, aprec2 = utils.itemwise_aroc_ap(label.cpu().detach.numpy(),
outputs.cpu().detach.numpy())
avg_auc2.append(np.mean(auc2))
avg_ap2.append(np.mean(aprec2))
eval_loss += loss.data[0]
avg_loss = eval_loss / len(val_loader)
print(
"Retrieval : Average AROC = %.3f, AP = %.3f / " %
(np.mean(avg_auc1), np.mean(avg_ap1)),
"Annotation : Average AROC = %.3f, AP = %.3f" %
(np.mean(avg_auc2), np.mean(avg_ap2)))
print('Average loss: {:.4f} \n'.format(avg_loss))
return avg_loss
def train(epoch):
avg_auc1 = []
avg_ap1 = []
avg_auc2 = []
avg_ap2 = []
i = 0
n = 0
acc = 0
phar = tqdm(train_loader, unit='train', unit_scale=train_loader.batch_size)
for audio, label in phar:
audio = torch.unsqueeze(audio, dim=1)
audio = audio.to(device)
label = label.to(device)
pred = sampleModel(audio)
loss = criterion(pred, label)
optimizer.zero_grad()
loss.backward()
optimizer.step()
# n += audio.size()[0].item()
if (i + 1) % 10 == 0:
# print("Epoch [%d/%d], Iter [%d/%d] loss : %.4f" % (
# epoch + 1, args.epochs, i + 1, len(train_loader), loss.item()))
# retrieval
auc1, ap1 = utils.tagwise_aroc_ap(label.cpu().detach().numpy(),
pred.cpu().detach().numpy())
avg_auc1.append(np.mean(auc1))
avg_ap1.append(np.mean(ap1))
# annotation
auc2, ap2 = utils.itemwise_aroc_ap(label.cpu().detach().numpy(),
pred.cpu().detach().numpy())
avg_auc2.append(np.mean(auc2))
avg_ap2.append(np.mean(ap2))
print(
"Retrieval : AROC = %.3f, AP = %.3f / " %
(np.mean(auc1), np.mean(ap1)),
"Annotation : AROC = %.3f, AP = %.3f" %
(np.mean(auc2), np.mean(ap2)))
i += 1
acc += torch.sum(
torch.argmax(pred, dim=1) == torch.argmax(label, dim=1))
n += audio.size()[0]
phar.set_postfix({
'loss': loss.item(),
'acc': acc.item() / n,
'Retrieval: AROC': np.mean(avg_auc1),
'Retrieval: AP ': np.mean(avg_ap1),
'Annotation: AROC ': np.mean(avg_auc2),
'Annotation: AP ': np.mean(avg_ap2),
})
# print("Retrieval : Average AROC = %.3f, AP = %.3f / " % (np.mean(avg_auc1), np.mean(avg_ap1)),
# "Annotation :Average AROC = %.3f, AP = %.3f" % (np.mean(avg_auc2), np.mean(avg_ap2)))
# print('Evaluating...')
torch.save(sampleModel.state_dict(),
'checkpoints/speechcommand/sampleCNN_' + str(epoch) + '.pth')
def train(self) :
# Train the network
for epoch in range(config.NUM_EPOCHS):
self.set_mode('train')
avg_auc1 = []
avg_ap1 = []
avg_auc2 = []
avg_ap2 = []
for i, data in enumerate(self.dataloader):
audio = data['audio'].to(device)
label = data['label'].to(device)
outputs = self.samplecnn(audio)
loss = self.bce(outputs, label)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
if (i+1) % 10 == 0:
print ("Epoch [%d/%d], Iter [%d/%d] loss : %.4f" % (epoch+1, config.NUM_EPOCHS, i+1, len(self.dataloader), loss.item()))
# retrieval
auc1, ap1 = utils.tagwise_aroc_ap(label.cpu().detach().numpy(), outputs.cpu().detach().numpy())
avg_auc1.append(np.mean(auc1))
avg_ap1.append(np.mean(ap1))
# annotation
auc2, ap2 = utils.itemwise_aroc_ap(label.cpu().detach().numpy(), outputs.cpu().detach().numpy())
avg_auc2.append(np.mean(auc2))
avg_ap2.append(np.mean(ap2))
print ("Retrieval : AROC = %.3f, AP = %.3f / "%(np.mean(auc1), np.mean(ap1)), "Annotation : AROC = %.3f, AP = %.3f"%(np.mean(auc2), np.mean(ap2)))
self.curr_epoch +=1
print ("Retrieval : Average AROC = %.3f, AP = %.3f / "%(np.mean(avg_auc1), np.mean(avg_ap1)), "Annotation :Average AROC = %.3f, AP = %.3f"%(np.mean(avg_auc2), np.mean(avg_ap2)))
print ('Evaluating...')
eval_loss = self.eval('valid')
self.scheduler.step(eval_loss) # use the learning rate scheduler
curr_lr = self.optimizer.param_groups[0]['lr']
print ('Learning rate : {}'.format(curr_lr))
if curr_lr < 1e-7:
print ("Early stopping")
break
torch.save(self.samplecnn.state_dict(), os.path.join(self.model_savepath, self.samplecnn.__class__.__name__ + '_' + str(self.curr_epoch) + '.pth'))
def eval(epoch):
eval_loss = 0.0
avg_auc1 = []
avg_ap1 = []
avg_auc2 = []
avg_ap2 = []
acc = 0
n = 0
phar = tqdm(valid_loader, unit='valid', unit_scale=valid_loader.batch_size)
for audio, label in phar:
audio = torch.unsqueeze(audio, dim=1)
audio = audio.to(device)
label = label.to(device)
pred = sampleModel(audio)
loss = criterion(pred, label)
optimizer.zero_grad()
loss.backward()
optimizer.step()
auc1, aprec1 = utils.tagwise_aroc_ap(label.cpu().data.numpy(),
pred.cpu().data.numpy())
avg_auc1.append(np.mean(auc1))
avg_ap1.append(np.mean(aprec1))
auc2, aprec2 = utils.itemwise_aroc_ap(label.cpu().data.numpy(),
pred.cpu().data.numpy())
avg_auc2.append(np.mean(auc2))
avg_ap2.append(np.mean(aprec2))
acc += torch.sum(
torch.argmax(pred, dim=1) == torch.argmax(label, dim=1))
eval_loss += loss.item()
n += audio.size()[0]
avg_loss = eval_loss / len(valid_loader)
phar.set_postfix({
'loss': avg_loss,
'acc': acc.item() / n,
'Retrieval: AROC': np.mean(avg_auc1),
'Retrieval: AP ': np.mean(avg_ap1),
'Annotation: AROC ': np.mean(avg_auc2),
'Annotation: AP ': np.mean(avg_ap2),
})
# print("Retrieval : Average AROC = %.3f, AP = %.3f / " % (np.mean(avg_auc1), np.mean(avg_ap1)),
# "Annotation : Average AROC = %.3f, AP = %.3f" % (np.mean(avg_auc2), np.mean(avg_ap2)))
# print('Average loss: {:.4f} \n'.format(avg_loss))
return avg_loss