def test(args): data_dir = args.data_dir workspace = args.workspace #mini_data = args.mini_data balance_type = args.balance_type #learning_rate = args.learning_rate filename = args.filename model_type = args.model_type model = args.model #batch_size = args.batch_size # Test data test_hdf5_path = os.path.join(data_dir, "eval1.h5") (test_x, test_y, test_id_list) = utilities.load_data(test_hdf5_path) # Output directories sub_dir = os.path.join(filename, 'balance_type={}'.format(balance_type), 'model_type={}'.format(model_type)) models_dir = os.path.join(workspace, "models", sub_dir) utilities.create_folder(models_dir) stats_dir = os.path.join(workspace, "stats", sub_dir) utilities.create_folder(stats_dir) probs_dir = os.path.join(workspace, "probs", sub_dir) utilities.create_folder(probs_dir) iteration = 1200 # Optimization method optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-07) #Loading model.. PATH = os.path.join(models_dir, "md_{}_iters_300batchsize.tar".format(iteration)) checkpoint = torch.load(PATH) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logging.info("Training data shape: {}".format(test_x.shape)) logging.info("Training data shape: {}".format(test_y.shape)) logging.info("Test statistics:") evaluate(model=model, input=test_x, target=test_y, stats_dir=os.path.join(stats_dir, "test"), probs_dir=os.path.join(probs_dir, "test"), iteration=iteration) print('ready')
def organise_data(config, logger, file, database, min_samples, learning_procedure, mode_label='train'): amcs = config.EXPERIMENT_DETAILS['AUDIO_MODE_IS_CONCAT_NOT_SHORTEN'] freq_bins = config.EXPERIMENT_DETAILS['FREQ_BINS'] feature_exp = config.EXPERIMENT_DETAILS['FEATURE_EXP'] convert_to_im = config.EXPERIMENT_DETAILS['CONVERT_TO_IMAGE'] feature_dim = config.EXPERIMENT_DETAILS['FEATURE_DIMENSIONS'] window_size = config.WINDOW_SIZE hop = config.HOP_SIZE labels = util.load_labels(file) features = util.load_data(database, labels) seconds_segment = determine_seconds_segment( config.EXPERIMENT_DETAILS['SECONDS_TO_SEGMENT'], feature_dim, window_size, hop, learning_procedure, feature_exp) features, labels, loc = process_data(amcs, freq_bins, features, labels, mode_label, min_samples, logger, seconds_segment, convert_to_im, feature_exp) if learning_procedure == 'chunked_file': feat_shape = features.shape features, loc = group_data(features, feat_shape, feature_dim, freq_bins, convert_to_im) index = [0, 0] class_weights = [1, 1] zeros = ones = 0 if mode_label == 'dev': _, zero_index, _, one_index, class_weights = data_info( labels, mode_label, logger, config) index = [zero_index, one_index] return features, labels, index, loc, (zeros, ones, class_weights)
def train(args): """Train a model. """ data_dir = args.data_dir workspace = args.workspace mini_data = args.mini_data balance_type = args.balance_type learning_rate = args.learning_rate filename = args.filename model_type = args.model_type model = args.model batch_size = args.batch_size # Path of hdf5 data bal_train_hdf5_path = os.path.join(data_dir, "bal_train.h5") unbal_train_hdf5_path = os.path.join(data_dir, "unbal_train.h5") test_hdf5_path = os.path.join(data_dir, "eval.h5") # Load data load_time = time.time() if mini_data: # Only load balanced data (bal_train_x, bal_train_y, bal_train_id_list) = utilities.load_data(bal_train_hdf5_path) train_x = bal_train_x train_y = bal_train_y train_id_list = bal_train_id_list else: # Load both balanced and unbalanced data (bal_train_x, bal_train_y, bal_train_id_list) = utilities.load_data(bal_train_hdf5_path) (unbal_train_x, unbal_train_y, unbal_train_id_list) = utilities.load_data(unbal_train_hdf5_path) train_x = np.concatenate((bal_train_x, unbal_train_x)) train_y = np.concatenate((bal_train_y, unbal_train_y)) train_id_list = bal_train_id_list + unbal_train_id_list # Test data (test_x, test_y, test_id_list) = utilities.load_data(test_hdf5_path) logging.info("Loading data time: {:.3f} s".format(time.time() - load_time)) logging.info("Training data shape: {}".format(train_x.shape)) # Optimization method optimizer = Adam(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=optimizer) # Output directories sub_dir = os.path.join(filename, 'balance_type={}'.format(balance_type), 'model_type={}'.format(model_type)) models_dir = os.path.join(workspace, "models", sub_dir) utilities.create_folder(models_dir) stats_dir = os.path.join(workspace, "stats", sub_dir) utilities.create_folder(stats_dir) probs_dir = os.path.join(workspace, "probs", sub_dir) utilities.create_folder(probs_dir) # Data generator if balance_type == 'no_balance': DataGenerator = data_generator.VanillaDataGenerator elif balance_type == 'balance_in_batch': DataGenerator = data_generator.BalancedDataGenerator else: raise Exception("Incorrect balance_type!") train_gen = DataGenerator(x=train_x, y=train_y, batch_size=batch_size, shuffle=True, seed=1234) iteration = 0 call_freq = 1000 train_time = time.time() for (batch_x, batch_y) in train_gen.generate(): # Compute stats every several interations if iteration % call_freq == 0: logging.info("------------------") logging.info("Iteration: {}, train time: {:.3f} s".format( iteration, time.time() - train_time)) logging.info("Balance train statistics:") evaluate(model=model, input=bal_train_x, target=bal_train_y, stats_dir=os.path.join(stats_dir, 'bal_train'), probs_dir=os.path.join(probs_dir, 'bal_train'), iteration=iteration) logging.info("Test statistics:") evaluate(model=model, input=test_x, target=test_y, stats_dir=os.path.join(stats_dir, "test"), probs_dir=os.path.join(probs_dir, "test"), iteration=iteration) train_time = time.time() # Update params (batch_x, batch_y) = utilities.transform_data(batch_x, batch_y) model.train_on_batch(x=batch_x, y=batch_y) iteration += 1 # Save model save_out_path = os.path.join(models_dir, "md_{}_iters.h5".format(iteration)) model.save(save_out_path) # Stop training when maximum iteration achieves if iteration == 50001: break
def train(args): """Train a model. """ data_dir = args.data_dir workspace = args.workspace mini_data = args.mini_data balance_type = args.balance_type learning_rate = args.learning_rate filename = args.filename model_type = args.model_type model = args.model batch_size = args.batch_size cuda = True # Move model to gpu if cuda: model.cuda() # Path of hdf5 data bal_train_hdf5_path = os.path.join(data_dir, "bal_train.h5") unbal_train_hdf5_path = os.path.join(data_dir, "unbal_train.h5") test_hdf5_path = os.path.join(data_dir, "eval.h5") # Load data load_time = time.time() if mini_data: # Only load balanced data (bal_train_x, bal_train_y, bal_train_id_list) = utilities.load_data(bal_train_hdf5_path) train_x = bal_train_x train_y = bal_train_y train_id_list = bal_train_id_list else: # Load both balanced and unbalanced data (bal_train_x, bal_train_y, bal_train_id_list) = utilities.load_data(bal_train_hdf5_path) (unbal_train_x, unbal_train_y, unbal_train_id_list) = utilities.load_data(unbal_train_hdf5_path) train_x = np.concatenate((bal_train_x, unbal_train_x)) train_y = np.concatenate((bal_train_y, unbal_train_y)) train_id_list = bal_train_id_list + unbal_train_id_list # Test data (test_x, test_y, test_id_list) = utilities.load_data(test_hdf5_path) logging.info("Loading data time: {:.3f} s".format(time.time() - load_time)) logging.info("Training data shape: {}".format(train_x.shape)) # Optimization method optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-07) # Output directories sub_dir = os.path.join(filename, 'balance_type={}'.format(balance_type), 'model_type={}'.format(model_type)) models_dir = os.path.join(workspace, "models", sub_dir) utilities.create_folder(models_dir) stats_dir = os.path.join(workspace, "stats", sub_dir) utilities.create_folder(stats_dir) probs_dir = os.path.join(workspace, "probs", sub_dir) utilities.create_folder(probs_dir) # Data generator if balance_type == 'no_balance': DataGenerator = data_generator.VanillaDataGenerator elif balance_type == 'balance_in_batch': DataGenerator = data_generator.BalancedDataGenerator else: raise Exception("Incorrect balance_type!") train_gen = DataGenerator(x=train_x, y=train_y, batch_size=batch_size, shuffle=True, seed=1234) iteration = 0 call_freq = 1000 train_time = time.time() for (batch_x, batch_y) in train_gen.generate(): # Compute stats every several interations if iteration % call_freq == 0 and iteration > 1: logging.info("------------------") logging.info("Iteration: {}, train time: {:.3f} s".format( iteration, time.time() - train_time)) logging.info("Balance train statistics:") evaluate(model=model, input=bal_train_x, target=bal_train_y, stats_dir=os.path.join(stats_dir, 'bal_train'), probs_dir=os.path.join(probs_dir, 'bal_train'), iteration=iteration) logging.info("Test statistics:") evaluate(model=model, input=test_x, target=test_y, stats_dir=os.path.join(stats_dir, "test"), probs_dir=os.path.join(probs_dir, "test"), iteration=iteration) train_time = time.time() (batch_x, batch_y) = utilities.transform_data(batch_x, batch_y) batch_x = move_data_to_gpu(batch_x, cuda) batch_y = move_data_to_gpu(batch_y, cuda) # Forward. model.train() output = model(batch_x) # Loss. loss = F.binary_cross_entropy(output, batch_y) # Backward. optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 # Save model. if iteration % 5000 == 0: save_out_dict = { 'iteration': iteration, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), } save_out_path = os.path.join(models_dir, "md_{}_iters.tar".format(iteration)) torch.save(save_out_dict, save_out_path) logging.info("Save model to {}".format(save_out_path)) # Stop training when maximum iteration achieves if iteration == 20001: break