def train_loop(train):
# pdb.set_trace()
train_dataset = TrainDataset(train,
transform=get_transforms(opt, data='train'),
opt=opt)
train_loader = DataLoader(train_dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
print('Creating model...')
model = create_model(opt, pretrained=True)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
scheduler = get_scheduler(optimizer)
trainer = setiTrainer(opt, model, optimizer, scheduler=scheduler)
trainer.set_device(opt.gpus, opt.device)
start_epoch = 0
best_loss = np.inf
model_dir = os.path.join(opt.save_dir, 'weights')
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
trainer.optimizer,
opt.resume, opt.lr,
opt.lr_step)
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
# train
log_dict_train, outputs, labels = trainer.train(epoch, train_loader)
avg_loss = log_dict_train['loss']
score = get_score(labels, outputs)
# print loss info and visualization loss
epoch_info = 'Epoch: {} |'.format(epoch)
for k, v in log_dict_train.items():
logger.writer.add_scalar('train_{}'.format(k), v, epoch)
epoch_info += '{} {:8f} | '.format(k, v)
epoch_info += 'score {:8f} '.format(score)
logger.log.info(epoch_info)
logger.writer.add_scalar('lr',
trainer.scheduler.get_last_lr()[0], epoch)
# save model
save_model(os.path.join(model_dir, f'{opt.arch}_model_last.pth'),
epoch, model)
if avg_loss < best_loss:
best_loss = avg_loss
logger.log.info(
f'Epoch {epoch} - Save Best Loss: {best_loss:.4f} Model')
save_model(os.path.join(model_dir, f'{opt.arch}_best_loss.pth'),
epoch, model)
def test_model(state,
reference_tsv_path,
reduced_number_of_data=None,
strore_predicitions_fname=None):
dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace),
base_feature_dir=os.path.join(
cfg.workspace, "dataset", "features"),
save_log_feature=False)
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
LOG.info(reference_tsv_path)
df = dataset.initialize_and_get_df(reference_tsv_path,
reduced_number_of_data)
strong_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
strong_dataload,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=strore_predicitions_fname)
compute_strong_metrics(predictions, df)
weak_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes), DataLoader(weak_dataload,
batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
def test_loop(test_data):
# pdb.set_trace()
test_dataset = TrainDataset(test_data,
transform=get_transforms(opt, data='valid'),
opt=opt,
is_train=False)
test_loader = DataLoader(test_dataset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=False)
print('Creating model...')
model = create_model(opt, pretrained=False)
set_device(model, opt.gpus, opt.device)
model.eval()
if opt.load_model != '':
model = load_model(model, opt.load_model)
result = []
for batch in tqdm(test_loader):
for k in batch:
if k != 'meta':
batch[k] = batch[k].to(device=opt.device, non_blocking=True)
with torch.no_grad():
output = model(batch['input']).cpu().sigmoid().numpy()
result.append(output)
del batch, output
results = np.concatenate(result)
res_df = test_data.copy()
res_df['target'] = results
res_df.to_csv()
res_df.to_csv(osp.join(osp.dirname(opt.load_model), 'submit.csv'),
index=False,
columns=['id', 'target'])
def train_loop(folds, fold):
# pdb.set_trace()
logger.log.info(f"========== fold: {fold} training ==========")
trn_idx = folds[folds['fold'] != fold].index
val_idx = folds[folds['fold'] == fold].index
train_folds = folds.loc[trn_idx].reset_index(drop=True)
valid_folds = folds.loc[val_idx].reset_index(drop=True)
valid_labels = valid_folds[opt.target_col].values
train_dataset = TrainDataset(train_folds,
transform=get_transforms(opt, data='train'), opt=opt)
valid_dataset = TrainDataset(valid_folds,
transform=get_transforms(opt, data='valid'), opt=opt)
train_loader = DataLoader(train_dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers, pin_memory=True, drop_last=True)
val_loader = DataLoader(valid_dataset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.num_workers, pin_memory=True, drop_last=False)
print('Creating model...')
model = create_model(opt, pretrained=True)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
trainer = setiTrainer(opt, model, optimizer, scheduler=None)
trainer.set_device(opt.gpus, opt.device)
start_epoch = 0
best_score = 0.
best_loss = np.inf
model_dir = os.path.join(opt.save_dir, 'weights')
if opt.load_model != '':
model, optimizer, start_epoch = load_model(
model, opt.load_model, trainer.optimizer, opt.resume, opt.lr, opt.lr_step)
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
# train
log_dict_train, outputs, labels = trainer.train(epoch, train_loader)
train_score = get_score(labels, outputs)
# print loss info and visualization loss
epoch_info = 'epoch: {} |'.format(epoch)
for k, v in log_dict_train.items():
logger.writer.add_scalar('train_fold{}_{}'.format(fold, k), v, epoch)
epoch_info += '{} {:8f} | '.format(k, v)
epoch_info += 'score {:8f} '.format(train_score)
logger.log.info(epoch_info)
# eval
ret, preds, _ = trainer.val(epoch, val_loader)
# get val scoring
score = get_score(valid_labels, preds)
avg_val_loss = ret['loss']
logger.log.info(f'Epoch {epoch} - val Score: {score:.4f}')
# save model
save_model(os.path.join(model_dir, f'{opt.arch}_fold{fold}_model_last.pth'),
epoch, model, optimizer, preds)
if epoch in opt.lr_step:
lr = opt.lr * (0.1 ** (opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if score > best_score:
best_score = score
logger.log.info(f'Epoch {epoch} - Save Best Score: {best_score:.4f} Model')
save_model(os.path.join(model_dir, f'{opt.arch}_fold{fold}_best_score.pth'),
epoch, model, optimizer, preds)
if avg_val_loss < best_loss:
best_loss = avg_val_loss
logger.log.info(f'Epoch {epoch} - Save Best Loss: {best_loss:.4f} Model')
save_model(os.path.join(model_dir, f'{opt.arch}_fold{fold}_best_loss.pth'),
epoch, model, optimizer, preds)
valid_folds['preds'] = torch.load(os.path.join(model_dir, f'{opt.arch}_fold{fold}_best_loss.pth'),
map_location=torch.device('cpu'))['preds']
return valid_folds
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None):
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
# ##############
# Validation
# ##############
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
# # 2018
# LOG.info("Eval 2018")
# eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data)
# # Strong
# eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong)
# compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio)
# # Weak
# eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# Validation 2019
# LOG.info("Validation 2019 (original code)")
# b_dataset = B_DatasetDcase2019Task4(cfg.workspace,
# base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'),
# save_log_feature=False)
# b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data)
# b_validation_df.to_csv('old.csv')
# b_validation_strong = B_DataLoadDf(b_validation_df,
# b_dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong,
# save_predictions=strore_predicitions_fname)
# compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio)
# b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# ============================================================================================
# ============================================================================================
# ============================================================================================
dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir,
local_path=cfg.workspace,
exp_tag=cfg.exp_tag,
save_log_feature=False)
# Validation 2019
LOG.info("Validation 2019")
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data)
validation_strong = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
validation_strong,
many_hot_encoder.decode_strong,
save_predictions=strore_predicitions_fname)
vdf = validation_df.copy()
vdf.filename = vdf.filename.str.replace('.npy', '.wav')
pdf = predictions.copy()
pdf.filename = pdf.filename.str.replace('.npy', '.wav')
compute_strong_metrics(pdf, vdf, pooling_time_ratio)
validation_weak = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(validation_weak, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
weak_df = dataset.initialize_and_get_df(cfg.weak, reduced_number_of_data)
synthetic_df = dataset.initialize_and_get_df(cfg.synthetic,
reduced_number_of_data,
download=False)
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data)
classes = DatasetDcase2019Task4.get_classes(
[weak_df, validation_df, synthetic_df])
# Be careful, frames is max_frames // pooling_time_ratio because max_pooling is applied on time axis in the model
many_hot_encoder = ManyHotEncoder(classes,
n_frames=cfg.max_frames //
pooling_time_ratio)
transforms = get_transforms(cfg.max_frames)
# Divide weak in train and valid
train_weak_df = weak_df.sample(frac=0.8, random_state=26)
valid_weak_df = weak_df.drop(train_weak_df.index).reset_index(drop=True)
train_weak_df = train_weak_df.reset_index(drop=True)
LOG.debug(valid_weak_df.event_labels.value_counts())
train_weak_data = DataLoadDf(train_weak_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms)
# Divide synthetic in train and valid
filenames_train = synthetic_df.filename.drop_duplicates().sample(
frac=0.8, random_state=26)
train_synth_df = synthetic_df[synthetic_df.filename.isin(filenames_train)]
f = output_file.open('w', encoding='utf-8')
for filename, result, detail in zip(testset.im_list, results, details):
if not isinstance(filename, Path):
filename = Path(filename)
f.write(f'{filename.name},{scripts[result]}\n')
f.close()
print(f'finish. writen as {args.output}')
if __name__ == "__main__":
ckpt = load_ckpt(os.path.join(config['ckpt_dir'], args.ckpt))
if not args.output:
args.output = f"{ckpt['params']['model']}_{ckpt['config']['transform']['val']}_{ckpt['config']['input_size']}.txt"
transform = get_transforms(ckpt['config']['transform']['val'])
print(f'transform: {ckpt["config"]["transform"]["val"]}')
print(f'scripts: {get_scripts()}')
testset = TestSet(args.test_path, transform)
print(f'testset: {args.test_path}')
dataloader = DataLoader(testset, batch_size=args.batch_size, shuffle=False,
num_workers=config['num_workers'])
num_classes = len(get_scripts())
model = get_model(ckpt['params']['model'], num_classes, False)
model = model.cuda()
model.load_state_dict(ckpt['model'])
model = nn.DataParallel(model, device_ids=list(range(len(args.gpu_ids))))
save_log_feature=False)
train_df = dataset.initialize_and_get_df(cfg.train,
reduced_number_of_data,
training=True)
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data,
training=True)
test_df = dataset.initialize_and_get_df(cfg.test,
reduced_number_of_data,
training=True)
many_hot_encoder = ManyHotEncoder(cfg.classes,
n_frames=cfg.max_frames //
pooling_time_ratio)
transforms = get_transforms(cfg.max_frames)
train_data = DataLoadDf(train_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms)
validation_data = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms)
test_data = DataLoadDf(test_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms)
list_dataset = [train_data]