def train(args): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace taxonomy_level: 'fine' | 'coarse' model_type: string, e.g. 'Cnn_9layers_MaxPooling' holdout_fold: '1' | 'None', where '1' indicates using validation and 'None' indicates using full data for training batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace taxonomy_level = args.taxonomy_level model_type = args.model_type holdout_fold = args.holdout_fold batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename seq_len = 640 mel_bins = config.mel_bins frames_per_second = config.frames_per_second max_iteration = 10 # Number of mini-batches to evaluate on training data reduce_lr = True labels = get_labels(taxonomy_level) classes_num = len(labels) # Paths if mini_data: prefix = 'minidata_' else: prefix = '' train_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train.h5') validate_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'validate.h5') scalar_path = os.path.join( workspace, 'scalars', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train.h5') checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(checkpoints_dir) _temp_submission_path = os.path.join( workspace, '_temp_submissions', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type, '_submission.csv') create_folder(os.path.dirname(_temp_submission_path)) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) annotation_path = os.path.join(dataset_dir, 'annotations.csv') yaml_path = os.path.join(dataset_dir, 'dcase-ust-taxonomy.yaml') logs_dir = os.path.join( workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, 'w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Load scalar scalar = load_scalar(scalar_path) # Model Model = eval(model_type) model = Model(classes_num, seq_len, mel_bins, cuda) if cuda: model.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) print('cliqueNet parameters:', sum(param.numel() for param in model.parameters())) # Data generator data_generator = DataGenerator(train_hdf5_path=train_hdf5_path, validate_hdf5_path=validate_hdf5_path, holdout_fold=holdout_fold, scalar=scalar, batch_size=batch_size) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, taxonomy_level=taxonomy_level, cuda=cuda, verbose=False) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate if iteration % 200 == 0: logging.info('------------------------------------') logging.info('Iteration: {}, {} level statistics:'.format( iteration, taxonomy_level)) train_fin_time = time.time() # Evaluate on training data if mini_data: raise Exception('`mini_data` flag must be set to False to use ' 'the official evaluation tool!') train_statistics = evaluator.evaluate(data_type='train', max_iteration=None) # Evaluate on validation data if holdout_fold != 'none': validate_statistics = evaluator.evaluate( data_type='validate', submission_path=_temp_submission_path, annotation_path=annotation_path, yaml_path=yaml_path, max_iteration=None) validate_statistics_container.append_and_dump( iteration, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 1000 == 0 and iteration > 0: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 # Move data to GPU for key in batch_data_dict.keys(): if key in ['feature', 'fine_target', 'coarse_target']: batch_data_dict[key] = move_data_to_gpu( batch_data_dict[key], cuda) # Train model.train() batch_output = model(batch_data_dict['feature']) # loss batch_target = batch_data_dict['{}_target'.format(taxonomy_level)] loss = binary_cross_entropy(batch_output, batch_target) # Backward optimizer.zero_grad() loss.backward() optimizer.step() # Stop learning if iteration == 3000: break iteration += 1
def train(args): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace subtask: 'a' | 'b' | 'c', corresponds to 3 subtasks in DCASE2019 Task1 data_type: 'development' | 'evaluation' holdout_fold: '1' | 'none', set 1 for development and none for training on all data without validation model_type: string, e.g. 'Cnn_9layers_AvgPooling' batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace subtask = args.subtask data_type = args.data_type holdout_fold = args.holdout_fold model_type = args.model_type batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename fixed = args.fixed finetune = args.finetune ite_train = args.ite_train ite_eva = args.ite_eva ite_store = args.ite_store mel_bins = config.mel_bins frames_per_second = config.frames_per_second max_iteration = None # Number of mini-batches to evaluate on training data reduce_lr = True sources_to_evaluate = get_sources(subtask) in_domain_classes_num = len(config.labels) - 1 # Paths if mini_data: prefix = 'minidata_' else: prefix = '' sub_dir = get_subdir(subtask, data_type) train_csv = os.path.join(dataset_dir, sub_dir, 'meta.csv') validate_csv = os.path.join(dataset_dir, sub_dir, 'evaluation_setup', 'fold1_evaluate.csv') feature_hdf5_path = os.path.join(workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) scalar_path = os.path.join(workspace, 'scalars', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) checkpoints_dir = os.path.join(workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(sub_dir), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(checkpoints_dir) validate_statistics_path = os.path.join(workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(sub_dir), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) logs_dir = os.path.join(workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(sub_dir), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, 'w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Load scalar scalar = load_scalar(scalar_path) # Model Model = eval(model_type) if subtask in ['a', 'b']: if fixed=='True': model = Model(in_domain_classes_num, activation='logsoftmax', fixed=True) else : model = Model(in_domain_classes_num, activation='logsoftmax', fixed=False) loss_func = nll_loss elif subtask == 'c': model = Model(in_domain_classes_num, activation='sigmoid') loss_func = F.binary_cross_entropy if cuda: model.cuda() # Optimizer if fixed=='True': optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) else : optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) if finetune=='True': model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/'+model_type+'/2000_iterations.pth' #model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/Logmel_Res38/2000_iterations.pth' #model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/Logmel_Cnn14/2000_iterations.pth' #model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/Logmel_Cnn10/2000_iterations.pth' #model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/Logmel_MobileNetV2/2000_iterations.pth' #model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/Logmel_MobileNetV1/2000_iterations.pth' #model_path='/home/cdd/code2/dcase2020_task1/workspace/checkpoints/main/logmel_86frames_40melbins/TAU-urban-acoustic-scenes-2020-mobile-development/holdout_fold=1/Logmel_Wavegram_Cnn14/2000_iterations.pth' device = torch.device('cuda') checkpoint = torch.load(model_path, map_location=device) model.load_state_dict(checkpoint['model']) # Data generator data_generator = DataGenerator( feature_hdf5_path=feature_hdf5_path, train_csv=train_csv, validate_csv=validate_csv, holdout_fold=holdout_fold, scalar=scalar, batch_size=batch_size) # Evaluator evaluator = Evaluator( model=model, data_generator=data_generator, subtask=subtask, cuda=cuda) # Statistics validate_statistics_container = StatisticsContainer(validate_statistics_path) train_bgn_time = time.time() iteration = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate #1800 if iteration % 200 == 0 and iteration > ite_eva: logging.info('------------------------------------') logging.info('Iteration: {}'.format(iteration)) train_fin_time = time.time() for source in sources_to_evaluate: train_statistics = evaluator.evaluate( data_type='train', source=source, max_iteration=None, verbose=False) if holdout_fold != 'none': for source in sources_to_evaluate: validate_statistics = evaluator.evaluate( data_type='validate', source=source, max_iteration=None, verbose=False) validate_statistics_container.append_and_dump( iteration, source, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info( 'Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 200 == 0 and iteration > ite_store: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict()} checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.93 # Move data to GPU for key in batch_data_dict.keys(): if key in ['feature', 'feature_gamm', 'feature_mfcc', 'feature_panns', 'target']: batch_data_dict[key] = move_data_to_gpu(batch_data_dict[key], cuda) # Train # batch_output,batch_loss = model(batch_data_dict['feature'], batch_data_dict['feature_gamm'], batch_data_dict['feature_mfcc'], batch_data_dict['feature_panns']) # loss = loss_func(batch_output, batch_data_dict['target']) # Using Mixup model.train() mixed_x1, mixed_x2, mixed_x3, mixed_x4, y_a, y_b, lam = mixup_data(x1=batch_data_dict['feature'], x2=batch_data_dict['feature_gamm'], x3=batch_data_dict['feature_mfcc'], x4=batch_data_dict['feature_panns'], y=batch_data_dict['target'], alpha=0.2) batch_output,batch_loss = model(mixed_x1, mixed_x2, mixed_x3, mixed_x4) if batch_output.shape[1] == 10: # single scale models loss = mixup_criterion(loss_func, batch_output, y_a, y_b, lam) else: # multi scale models losses = [] for ite in range(batch_output.shape[1]-1): loss = mixup_criterion(loss_func, batch_output[:,ite,:], y_a, y_b, lam) losses.append(loss) loss = sum(losses) # Backward optimizer.zero_grad() loss.backward() optimizer.step() # Stop learning # 12000 for scratch if iteration == ite_train: break iteration += 1
def train(args, i): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace holdout_fold: '1' | 'none', set 1 for development and none for training on all data without validation model_type: string, e.g. 'Cnn_9layers_AvgPooling' batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace holdout_fold = args.holdout_fold model_type = args.model_type batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename audio_num = config.audio_num mel_bins = config.mel_bins frames_per_second = config.frames_per_second max_iteration = None # Number of mini-batches to evaluate on training data reduce_lr = True in_domain_classes_num = len(config.labels) # Paths if mini_data: prefix = 'minidata_' else: prefix = '' train_csv = os.path.join(sys.path[0], 'fold' + str(i) + '_train.csv') validate_csv = os.path.join(sys.path[0], 'fold' + str(i) + '_test.csv') feature_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins.h5'.format(prefix, frames_per_second, mel_bins)) checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins.h5'.format(prefix, frames_per_second, mel_bins), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(checkpoints_dir) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) logs_dir = os.path.join( workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, 'w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Model Model = eval(model_type) model = Model(in_domain_classes_num, activation='logsoftmax') loss_func = nll_loss if cuda: model.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) # optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-5) # Data generator data_generator = DataGenerator(feature_hdf5_path=feature_hdf5_path, train_csv=train_csv, validate_csv=validate_csv, holdout_fold=holdout_fold, batch_size=batch_size) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, cuda=cuda) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate if iteration % 100 == 0 and iteration >= 1500: logging.info('------------------------------------') logging.info('Iteration: {}'.format(iteration)) train_fin_time = time.time() train_statistics = evaluator.evaluate(data_type='train', iteration=iteration, max_iteration=None, verbose=False) if holdout_fold != 'none': validate_statistics = evaluator.evaluate(data_type='validate', iteration=iteration, max_iteration=None, verbose=False) validate_statistics_container.append_and_dump( iteration, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 100 == 0 and iteration > 0: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 100 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 # Move data to GPU for key in batch_data_dict.keys(): if key in ['feature', 'target']: batch_data_dict[key] = move_data_to_gpu( batch_data_dict[key], cuda) # Train for i in range(audio_num): model.train() data, target_a, target_b, lam = mixup_data( x=batch_data_dict['feature'][:, i, :, :], y=batch_data_dict['target'], alpha=0.2) batch_output = model(data) # batch_output = model(batch_data_dict['feature']) # loss loss = loss_func(batch_output, batch_data_dict['target']) loss = mixup_criterion(loss_func, batch_output, target_a, target_b, lam) # Backward optimizer.zero_grad() loss.backward() optimizer.step() # Stop learning if iteration == 4000: break iteration += 1
def train(args): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace subtask: 'a' | 'b' | 'c', corresponds to 3 subtasks in DCASE2019 Task1 data_type: 'development' | 'evaluation' holdout_fold: '1' | 'none', set 1 for development and none for training on all data without validation model_type: string, e.g. 'Cnn_9layers_AvgPooling' batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace subtask = args.subtask data_type = args.data_type holdout_fold = args.holdout_fold model_type = args.model_type batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename mel_bins = config.mel_bins frames_per_second = config.frames_per_second max_iteration = None # Number of mini-batches to evaluate on training data reduce_lr = True sources_to_evaluate = get_sources(subtask) in_domain_classes_num = len(config.labels) - 1 # Paths if mini_data: prefix = 'minidata_' else: prefix = '' sub_dir = get_subdir(subtask, data_type) train_csv = os.path.join(dataset_dir, sub_dir, 'evaluation_setup', 'fold1_train.csv') validate_csv = os.path.join(dataset_dir, sub_dir, 'evaluation_setup', 'fold1_evaluate.csv') feature_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) feature_hdf5_path_left = os.path.join( workspace, 'features_left', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) feature_hdf5_path_right = os.path.join( workspace, 'features_right', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) feature_hdf5_path_side = os.path.join( workspace, 'features_side', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) scalar_path = os.path.join( workspace, 'scalars', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) scalar_path_left = os.path.join( workspace, 'scalars_left', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) scalar_path_right = os.path.join( workspace, 'scalars_right', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) scalar_path_side = os.path.join( workspace, 'scalars_side', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(sub_dir)) checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(sub_dir), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(checkpoints_dir) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(sub_dir), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) logs_dir = os.path.join( workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(sub_dir), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, 'w') logging.info(args) # Load scalar scalar = load_scalar(scalar_path) scalar_left = load_scalar(scalar_path_left) scalar_right = load_scalar(scalar_path_right) scalar_side = load_scalar(scalar_path_side) # Model Model = eval(model_type) if subtask in ['a', 'b']: model = Model(in_domain_classes_num, activation='logsoftmax') loss_func = nll_loss elif subtask == 'c': model = Model(in_domain_classes_num, activation='sigmoid') loss_func = F.binary_cross_entropy if cuda: model.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) # Data generator data_generator = DataGenerator( feature_hdf5_path=feature_hdf5_path, feature_hdf5_path_left=feature_hdf5_path_left, feature_hdf5_path_right=feature_hdf5_path_right, feature_hdf5_path_side=feature_hdf5_path_side, train_csv=train_csv, validate_csv=validate_csv, scalar=scalar, scalar_left=scalar_left, scalar_right=scalar_right, scalar_side=scalar_side, batch_size=batch_size) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, subtask=subtask, cuda=cuda) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 # Train on mini batches for batch_data_dict, batch_data_dict_left, batch_data_dict_right, batch_data_dict_side in data_generator.generate_train( ): # Evaluate if iteration % 200 == 0: logging.info('------------------------------------') logging.info('Iteration: {}'.format(iteration)) train_fin_time = time.time() for source in sources_to_evaluate: train_statistics = evaluator.evaluate(data_type='train', source=source, max_iteration=None, verbose=False) for source in sources_to_evaluate: validate_statistics = evaluator.evaluate(data_type='validate', source=source, max_iteration=None, verbose=False) validate_statistics_container.append_and_dump( iteration, source, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 1000 == 0 and iteration > 0: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 # Move data to GPU for key in batch_data_dict.keys(): if key in ['feature', 'target']: batch_data_dict[key] = move_data_to_gpu( batch_data_dict[key], cuda) for key in batch_data_dict_left.keys(): if key in ['feature_left', 'target']: batch_data_dict_left[key] = move_data_to_gpu( batch_data_dict_left[key], cuda) for key in batch_data_dict_right.keys(): if key in ['feature_right', 'target']: batch_data_dict_right[key] = move_data_to_gpu( batch_data_dict_right[key], cuda) for key in batch_data_dict_side.keys(): if key in ['feature_side', 'target']: batch_data_dict_side[key] = move_data_to_gpu( batch_data_dict_side[key], cuda) # Train model.train() batch_output = model(data=batch_data_dict['feature'], data_left=batch_data_dict_left['feature_left'], data_right=batch_data_dict_right['feature_right'], data_side=batch_data_dict_side['feature_side']) # loss loss = loss_func(batch_output, batch_data_dict['target']) # Backward optimizer.zero_grad() loss.backward() optimizer.step() # Stop learning if iteration == 15000: break iteration += 1
def train(args): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace taxonomy_level: 'fine' | 'coarse' model_type: string, e.g. 'Cnn_9layers_MaxPooling' holdout_fold: '1' | 'None', where '1' indicates using validation and 'None' indicates using full data for training batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace taxonomy_level = args.taxonomy_level model_type = args.model_type holdout_fold = args.holdout_fold batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename plt_x = [] plt_y = [] mel_bins = config.mel_bins frames_per_second = config.frames_per_second max_iteration = 10 # Number of mini-batches to evaluate on training data reduce_lr = True labels = get_labels(taxonomy_level) classes_num = len(labels) # Paths if mini_data: prefix = 'minidata_' else: prefix = '' train_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train.h5') validate_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'validate.h5') scalar_path = os.path.join( workspace, 'scalars', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train.h5') checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(checkpoints_dir) _temp_submission_path = os.path.join( workspace, '_temp_submissions', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type, '_submission.csv') create_folder(os.path.dirname(_temp_submission_path)) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) annotation_path = os.path.join(dataset_dir, 'annotations.csv') yaml_path = os.path.join(dataset_dir, 'dcase-ust-taxonomy.yaml') logs_dir = os.path.join( workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, 'w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Load scalar scalar = load_scalar(scalar_path) # Model Model = eval(model_type) model = Model(classes_num) if cuda: model.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) # Data generator data_generator = DataGenerator(train_hdf5_path=train_hdf5_path, validate_hdf5_path=validate_hdf5_path, holdout_fold=holdout_fold, scalar=scalar, batch_size=batch_size) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, taxonomy_level=taxonomy_level, cuda=cuda, verbose=False) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 best_inde = {} best_inde['micro_auprc'] = np.array([0.0]) best_inde['micro_f1'] = np.array([0.0]) best_inde['macro_auprc'] = np.array([0.0]) best_inde['average_precision'] = np.array([0.0]) best_inde['sum'] = best_inde['micro_auprc'] + best_inde[ 'micro_f1'] + best_inde['macro_auprc'] best_map = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate if iteration % 200 == 0: logging.info('------------------------------------') logging.info('Iteration: {}, {} level statistics:'.format( iteration, taxonomy_level)) train_fin_time = time.time() # Evaluate on training data if mini_data: raise Exception('`mini_data` flag must be set to False to use ' 'the official evaluation tool!') train_statistics = evaluator.evaluate(data_type='train', max_iteration=None) if iteration > 5000: if best_map < np.mean(train_statistics['average_precision']): best_map = np.mean(train_statistics['average_precision']) logging.info('best_map= {}'.format(best_map)) # logging.info('iter= {}'.format(iteration)) checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'indicators': train_statistics } checkpoint_path = os.path.join(checkpoints_dir, 'best2.pth') torch.save(checkpoint, checkpoint_path) logging.info( 'best_models saved to {}'.format(checkpoint_path)) # Evaluate on validation data if holdout_fold != 'none': validate_statistics = evaluator.evaluate( data_type='validate', submission_path=_temp_submission_path, annotation_path=annotation_path, yaml_path=yaml_path, max_iteration=None) validate_statistics_container.append_and_dump( iteration, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 1000 == 0 and iteration > 0: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 # Move data to GPU for key in batch_data_dict.keys(): if key in ['feature', 'fine_target', 'coarse_target', 'spacetime']: batch_data_dict[key] = move_data_to_gpu( batch_data_dict[key], cuda) feature, spacetime, targets_a, targets_b, lam = mixup_data( batch_data_dict['feature'], batch_data_dict['spacetime'], batch_data_dict['{}_target'.format(taxonomy_level)], alpha=1.0, use_cuda=True) # Train model.train() criterion = nn.BCELoss().cuda() batch_output = model(feature, spacetime) # loss #batch_target = batch_data_dict['{}_target'.format(taxonomy_level)] loss = mixup_criterion(criterion, batch_output, targets_a, targets_b, lam) #loss = binary_cross_entropy(batch_output, batch_target) # Backward optimizer.zero_grad() loss.backward() optimizer.step() if iteration % 100 == 0: plt_x.append(iteration) plt_y.append(loss.item()) if iteration % 10000 == 0 and iteration != 0: plt.figure(1) plt.suptitle('test result ', fontsize='18') plt.plot(plt_x, plt_y, 'r-', label='loss') plt.legend(loc='best') plt.savefig( '/home/fangjunyan/count/' + time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) + '{}'.format(holdout_fold) + '{}.jpg'.format(taxonomy_level)) # Stop learning if iteration == 10000: break iteration += 1
def train(args): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace taxonomy_level: 'fine' | 'coarse' model_type: string, e.g. 'Cnn_9layers_MaxPooling' holdout_fold: '1' | 'None', where '1' indicates using validation and 'None' indicates using full data for training batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace taxonomy_level = args.taxonomy_level model_type = args.model_type holdout_fold = args.holdout_fold batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename plt_x = [] plt_y = [] T_max = 300 mel_bins = config.mel_bins frames_per_second = config.frames_per_second max_iteration = 10 # Number of mini-batches to evaluate on training data reduce_lr = True labels = get_labels(taxonomy_level) classes_num = len(labels) def mixup_data(x1, x2, y, alpha=1.0, use_cuda=True): # 数据增强,看下那个博客 '''Returns mixed inputs, pairs of targets, and lambda''' if alpha > 0: lam = np.random.beta(alpha, alpha) # 随机生成一个(1,1)的张量 else: lam = 1 # batch_size = x1.size()[0] if use_cuda: index = torch.randperm( batch_size).cuda() # 给定参数n,返回一个从0到n-1的随机整数序列 else: index = torch.randperm(batch_size) # 使用cpu还是gpu mixed_x1 = lam * x1 + (1 - lam) * x1[index, :] mixed_x2 = lam * x2 + (1 - lam) * x2[index, :] # 混合数据 y_a, y_b = y, y[index] return mixed_x1, mixed_x2, y_a, y_b, lam def mixup_criterion(criterion, pred, y_a, y_b, lam): return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) # Paths if mini_data: prefix = 'minidata_' else: prefix = '' train_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train.h5') validate_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'validate.h5') scalar_path = os.path.join( workspace, 'scalars', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train.h5') checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(checkpoints_dir) _temp_submission_path = os.path.join( workspace, '_temp_submissions', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type, '_submission.csv') create_folder(os.path.dirname(_temp_submission_path)) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) loss_path = os.path.join( workspace, 'loss', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_folder(loss_path) annotation_path = os.path.join(dataset_dir, 'annotations.csv') yaml_path = os.path.join(dataset_dir, 'dcase-ust-taxonomy.yaml') logs_dir = os.path.join( workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'taxonomy_level={}'.format(taxonomy_level), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, 'w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Load scalar scalar = load_scalar(scalar_path) # Model Model = eval(model_type) model = Model(classes_num) logging.info( " Space_Duo_Cnn_9_Avg 多一层 258*258 不共用FC,必须带时空标签 用loss 监测,使用去零one hot " ) if cuda: model.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) logging.info('model parm:{} '.format( sum(param.numel() for param in model.parameters()))) #计算模型参数量 # Data generator data_generator = DataGenerator(train_hdf5_path=train_hdf5_path, validate_hdf5_path=validate_hdf5_path, holdout_fold=holdout_fold, scalar=scalar, batch_size=batch_size) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, taxonomy_level=taxonomy_level, cuda=cuda, verbose=False) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 best_inde = {} best_inde['micro_auprc'] = np.array([0.0]) best_inde['micro_f1'] = np.array([0.0]) best_inde['macro_auprc'] = np.array([0.0]) best_inde['average_precision'] = np.array([0.0]) best_inde['sum'] = best_inde['micro_auprc'] + best_inde[ 'micro_f1'] + best_inde['macro_auprc'] last_loss1 = [] last_loss2 = [] last_loss = [] best_map = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate if iteration % 200 == 0: logging.info('------------------------------------') logging.info('Iteration: {}, {} level statistics:'.format( iteration, taxonomy_level)) train_fin_time = time.time() # Evaluate on training data if mini_data: raise Exception('`mini_data` flag must be set to False to use ' 'the official evaluation tool!') train_statistics = evaluator.evaluate(data_type='train', max_iteration=None) if iteration > 5000: if best_map < np.mean(train_statistics['average_precision']): best_map = np.mean(train_statistics['average_precision']) logging.info('best_map= {}'.format(best_map)) # logging.info('iter= {}'.format(iteration)) checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'indicators': train_statistics } checkpoint_path = os.path.join(checkpoints_dir, 'best7.pth') torch.save(checkpoint, checkpoint_path) logging.info( 'best_models saved to {}'.format(checkpoint_path)) # Evaluate on validation data if holdout_fold != 'none': validate_statistics = evaluator.evaluate( data_type='validate', submission_path=_temp_submission_path, annotation_path=annotation_path, yaml_path=yaml_path, max_iteration=None) validate_statistics_container.append_and_dump( iteration, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 batch_data2_dict = batch_data_dict.copy() n = [] for i, l in enumerate(batch_data2_dict['coarse_target']): k = 0 for j in range(0, 8): if l[j] > 0.6: l[j] = 1 else: l[j] = 0 k += 1 if k == 8: if taxonomy_level == 'coarse': n.append(i) for i, l in enumerate(batch_data2_dict['fine_target']): k = 0 for j in range(0, 29): if l[j] > 0.6: l[j] = 1 else: l[j] = 0 k += 1 if k == 29: if taxonomy_level == 'fine': n.append(i) batch_data2_dict['fine_target'] = np.delete( batch_data2_dict['fine_target'], n, axis=0) batch_data2_dict['coarse_target'] = np.delete( batch_data2_dict['coarse_target'], n, axis=0) batch_data2_dict['audio_name'] = np.delete( batch_data2_dict['audio_name'], n, axis=0) batch_data2_dict['feature'] = np.delete(batch_data2_dict['feature'], n, axis=0) batch_data2_dict['spacetime'] = np.delete( batch_data2_dict['spacetime'], n, axis=0) if batch_data2_dict['audio_name'].size == 0: iteration += 1 continue #使用 概率数据请注释下行,使用去零onehot数据不用注释 batch_data_dict = batch_data2_dict # if iteration <8655: # batch_data_dict = batch_data2_dict # elif iteration >=8655 and iteration % 2 == 0: # batch_data_dict = batch_data2_dict # Move data to GPU ,'external_target','external_feature' for key in batch_data_dict.keys(): if key in ['feature', 'fine_target', 'coarse_target', 'spacetime']: batch_data_dict[key] = move_data_to_gpu( batch_data_dict[key], cuda) # Train model.train() # 使用mix_up 数据增强 feature1, spacetime1, targets1_a, targets1_b, lam1 = mixup_data( batch_data_dict['feature'], batch_data_dict['spacetime'], batch_data_dict['fine_target'], alpha=1.0, use_cuda=True) feature2, spacetime2, targets2_a, targets2_b, lam2 = mixup_data( batch_data_dict['feature'], batch_data_dict['spacetime'], batch_data_dict['coarse_target'], alpha=1.0, use_cuda=True) batch_output1 = model.forward1(feature1, spacetime1) batch_output2 = model.forward2(feature2, spacetime2) lam1 = int(lam1) lam2 = int(lam2) loss1 = (lam1 * binary_cross_entropy(batch_output1, targets1_a) + (1 - lam1) * binary_cross_entropy(batch_output1, targets1_b)) loss2 = (lam2 * binary_cross_entropy(batch_output2, targets2_a) + (1 - lam2) * binary_cross_entropy(batch_output2, targets2_b)) #不使用mix_up 数据增强,请使用以下代码 # batch_target1 = batch_data_dict['fine_target'] # batch_output1 = model.forward1(batch_data_dict['feature'], batch_data_dict['spacetime']) # batch_target2 = batch_data_dict['coarse_target'] # batch_output2 = model.forward2(batch_data_dict['feature'], batch_data_dict['spacetime']) # loss1 = binary_cross_entropy(batch_output1, batch_target1) # loss2 = binary_cross_entropy(batch_output2, batch_target2) loss = loss1 + loss2 #使用loss监测请使用以下代码否者注释 if iteration > 4320: new_loss = loss.item() if len(last_loss) < 5: last_loss.append(new_loss) else: cha = 0 for i in range(4): cha += abs(last_loss[i + 1] - last_loss[i]) if new_loss > last_loss[4] and cha >= (new_loss - last_loss[4]) > cha / 2: for i in range(4): last_loss[i] = last_loss[i + 1] last_loss[4] = new_loss logging.info(' drop iteration:{}'.format(iteration)) iteration += 1 continue elif new_loss > last_loss[4] and (new_loss - last_loss[4]) > cha / 2.75: for i in range(4): last_loss[i] = last_loss[i + 1] last_loss[4] = new_loss logging.info(' low weightiteration:{}'.format(iteration)) loss = loss / 2 else: for i in range(4): last_loss[i] = last_loss[i + 1] last_loss[4] = new_loss # # Backward optimizer.zero_grad() loss.backward() optimizer.step() if iteration % 50 == 0: plt_x.append(iteration) plt_y.append(loss) if iteration % 13000 == 0 and iteration != 0: plt.figure(1) plt.suptitle('test result ', fontsize='18') plt.plot(plt_x, plt_y, 'r-', label='loss') plt.legend(loc='best') plt.savefig( loss_path + '/' + time.strftime('%m%d_%H%M%S', time.localtime(time.time())) + 'loss.jpg') plt.savefig(loss_path + '/loss.jpg') # Stop learning if iteration == 13000: # logging.info("best_micro_auprc:{:.3f}".format(best_inde['micro_auprc'])) # logging.info("best_micro_f1:{:.3f}".format(best_inde['micro_f1'])) # logging.info("best_macro_auprc:{:.3f}".format(best_inde['macro_auprc'])) # labels = get_labels(taxonomy_level) # for k, label in enumerate(labels): # logging.info(' {:<40}{:.3f}'.format(label, best_inde['average_precision'][k])) break iteration += 1
def train(args): '''Train. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace audio_type: 'foa' | 'mic' holdout_fold: '1' | '2' | '3' | '4' | 'none', set to none if using all data without validation to train model_type: string, e.g. 'Cnn_9layers_AvgPooling' batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace audio_type = args.audio_type holdout_fold = args.holdout_fold model_type = args.model_type batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename mel_bins = config.mel_bins frames_per_second = config.frames_per_second classes_num = config.classes_num max_validate_num = None # Number of audio recordings to validate reduce_lr = True # Reduce learning rate after several iterations # Paths if mini_data: prefix = 'minidata_' else: prefix = '' metadata_dir = os.path.join(dataset_dir, 'metadata_dev') features_dir = os.path.join( workspace, 'features', '{}{}_{}_logmel_{}frames_{}melbins'.format(prefix, audio_type, 'dev', frames_per_second, mel_bins)) scalar_path = os.path.join( workspace, 'scalars', '{}{}_{}_logmel_{}frames_{}melbins'.format(prefix, audio_type, 'dev', frames_per_second, mel_bins), 'scalar.h5') checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}{}_{}_logmel_{}frames_{}melbins'.format(prefix, audio_type, 'dev', frames_per_second, mel_bins), model_type, 'holdout_fold={}'.format(holdout_fold)) create_folder(checkpoints_dir) # All folds result should write to the same directory temp_submissions_dir = os.path.join( workspace, '_temp', 'submissions', filename, '{}{}_{}_logmel_{}frames_{}melbins'.format(prefix, audio_type, 'dev', frames_per_second, mel_bins), model_type) create_folder(temp_submissions_dir) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}{}_{}_logmel_{}frames_{}melbins'.format(prefix, audio_type, 'dev', frames_per_second, mel_bins), 'holdout_fold={}'.format(holdout_fold), model_type, 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) logs_dir = os.path.join( args.workspace, 'logs', filename, args.mode, '{}{}_{}_logmel_{}frames_{}melbins'.format(prefix, audio_type, 'dev', frames_per_second, mel_bins), 'holdout_fold={}'.format(holdout_fold), model_type) create_logging(logs_dir, filemode='w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Load scalar scalar = load_scalar(scalar_path) # Model Model = eval(model_type) model = Model(classes_num) if cuda: model.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) # Data generator data_generator = DataGenerator(features_dir=features_dir, scalar=scalar, batch_size=batch_size, holdout_fold=holdout_fold) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, cuda=cuda) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate if iteration % 200 == 0: logging.info('------------------------------------') logging.info('Iteration: {}'.format(iteration)) train_fin_time = time.time() ''' # Uncomment for evaluating on training dataset train_statistics = evaluator.evaluate( data_type='train', metadata_dir=metadata_dir, submissions_dir=temp_submissions_dir, max_validate_num=max_validate_num) ''' if holdout_fold != 'none': validate_statistics = evaluator.evaluate( data_type='validate', metadata_dir=metadata_dir, submissions_dir=temp_submissions_dir, max_validate_num=max_validate_num) validate_statistics_container.append_and_dump( iteration, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 1000 == 0 and iteration > 0: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 # Move data to GPU for key in batch_data_dict.keys(): batch_data_dict[key] = move_data_to_gpu(batch_data_dict[key], cuda) # Train model.train() batch_output_dict = model(batch_data_dict['feature']) loss = event_spatial_loss(batch_output_dict, batch_data_dict) # Backward optimizer.zero_grad() loss.backward() optimizer.step() # Stop learning if iteration == 5000: break iteration += 1
def train(args): '''Training. Model will be saved after several iterations. Args: dataset_dir: string, directory of dataset workspace: string, directory of workspace data_type: 'train_weak' | 'train_synthetic' holdout_fold: '1' | 'none', set 1 for development and none for training on all data without validation.' model_type: string, e.g. 'Cnn_9layers_AvgPooling' loss_type: 'clipwise_binary_crossentropy' | 'framewise_binary_crossentropy' batch_size: int cuda: bool mini_data: bool, set True for debugging on a small part of data ''' # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace data_type = args.data_type holdout_fold = args.holdout_fold model_type = args.model_type loss_type = args.loss_type batch_size = args.batch_size cuda = args.cuda and torch.cuda.is_available() mini_data = args.mini_data filename = args.filename mel_bins = config.mel_bins frames_per_second = config.frames_per_second classes_num = config.classes_num max_iteration = None # Number of mini-batches to evaluate on training data reduce_lr = True # Paths if mini_data: prefix = 'minidata_' else: prefix = '' if loss_type == 'clipwise_binary_crossentropy': strong_target_training = False elif loss_type == 'framewise_binary_crossentropy': strong_target_training = True else: raise Exception('Incorrect argument!') train_relative_name = get_relative_path_no_extension(data_type) validate_relative_name = get_relative_path_no_extension('validation') train_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(train_relative_name)) validate_hdf5_path = os.path.join( workspace, 'features', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}.h5'.format(validate_relative_name)) scalar_path = os.path.join( workspace, 'scalars', '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), 'train/weak.h5') train_metadata_path = os.path.join(dataset_dir, 'metadata', '{}.csv'.format(train_relative_name)) validate_metadata_path = os.path.join( dataset_dir, 'metadata', 'validation', '{}.csv'.format(validate_relative_name)) checkpoints_dir = os.path.join( workspace, 'checkpoints', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(train_relative_name), 'holdout_fold={}'.format(holdout_fold), model_type, 'loss_type={}'.format(loss_type)) create_folder(checkpoints_dir) temp_submission_path = os.path.join( workspace, '_temp', 'submissions', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(train_relative_name), 'holdout_fold={}'.format(holdout_fold), model_type, 'loss_type={}'.format(loss_type), '_temp_submission.csv') create_folder(os.path.dirname(temp_submission_path)) validate_statistics_path = os.path.join( workspace, 'statistics', filename, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(train_relative_name), 'holdout_fold={}'.format(holdout_fold), model_type, 'loss_type={}'.format(loss_type), 'validate_statistics.pickle') create_folder(os.path.dirname(validate_statistics_path)) logs_dir = os.path.join( args.workspace, 'logs', filename, args.mode, '{}logmel_{}frames_{}melbins'.format(prefix, frames_per_second, mel_bins), '{}'.format(train_relative_name), 'holdout_fold={}'.format(holdout_fold), model_type, 'loss_type={}'.format(loss_type)) create_logging(logs_dir, filemode='w') logging.info(args) if cuda: logging.info('Using GPU.') else: logging.info('Using CPU. Set --cuda flag to use GPU.') # Load scalar scalar = load_scalar(scalar_path) # Model Model = eval(model_type) model = Model(classes_num, strong_target_training) if cuda: model.cuda() loss_func = eval(loss_type) # Optimizer optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08, weight_decay=0., amsgrad=True) # Data generator data_generator = DataGenerator(train_hdf5_path=train_hdf5_path, validate_hdf5_path=validate_hdf5_path, holdout_fold=holdout_fold, scalar=scalar, batch_size=batch_size) # Evaluator evaluator = Evaluator(model=model, data_generator=data_generator, cuda=cuda, verbose=False) # Statistics validate_statistics_container = StatisticsContainer( validate_statistics_path) train_bgn_time = time.time() iteration = 0 # Train on mini batches for batch_data_dict in data_generator.generate_train(): # Evaluate if iteration % 200 == 0: logging.info('------------------------------------') logging.info('Iteration: {}'.format(iteration)) train_fin_time = time.time() train_statistics = evaluator.evaluate( data_type='train', metadata_path=train_metadata_path, submission_path=temp_submission_path, max_iteration=max_iteration) if holdout_fold != 'none': validate_statistics = evaluator.evaluate( data_type='validate', metadata_path=validate_metadata_path, submission_path=temp_submission_path, max_iteration=max_iteration) validate_statistics_container.append_and_dump( iteration, validate_statistics) train_time = train_fin_time - train_bgn_time validate_time = time.time() - train_fin_time logging.info('Train time: {:.3f} s, validate time: {:.3f} s' ''.format(train_time, validate_time)) train_bgn_time = time.time() # Save model if iteration % 1000 == 0 and iteration > 0: checkpoint = { 'iteration': iteration, 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } checkpoint_path = os.path.join( checkpoints_dir, '{}_iterations.pth'.format(iteration)) torch.save(checkpoint, checkpoint_path) logging.info('Model saved to {}'.format(checkpoint_path)) # Reduce learning rate if reduce_lr and iteration % 200 == 0 and iteration > 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.9 # Move data to GPU for key in batch_data_dict.keys(): if key in ['feature', 'weak_target', 'strong_target']: batch_data_dict[key] = move_data_to_gpu( batch_data_dict[key], cuda) # Train model.train() batch_output_dict = model(batch_data_dict['feature']) # loss loss = loss_func(batch_output_dict, batch_data_dict) # Backward optimizer.zero_grad() loss.backward() optimizer.step() # Stop learning if iteration == 5000: break iteration += 1