def __init__(self, batch_size, seed=1234): """ Inputs: hdf5_path: str batch_size: int dev_train_csv: str | None, if None then use all data for training dev_validate_csv: str | None, if None then use all data for training seed: int, random seed """ self.batch_size = batch_size self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) # Load data (self.train_x, self.train_y, self.validate_x, self.validate_y, _, _) = load_data() print(self.train_x.shape, self.validate_x.shape) self.train_audio_names = np.arange(len(self.train_x)) self.validate_audio_names = np.arange(len(self.validate_x)) # Calculate scalar (self.mean, self.std) = calculate_scalar(self.train_x)
def __init__(self, hdf5_path, batch_size, train_txt=None, validate_txt=None, seed=1234): """Data generator. Args: hdf5_path: string, path of development hdf5 batch_size: int train_txt: string | None, if None then all audios are used for training validate_txt: string | None, if None then all audios are used for training seed: int, random seed """ self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) lb_to_ix = config.lb_to_ix self.batch_size = batch_size # Load data load_time = time.time() hf = h5py.File(hdf5_path, 'r') self.audio_names = np.array( [name.decode() for name in hf['audio_name']]) self.x = hf['feature'][:] target_labels = [label.decode() for label in hf['label']] self.y = np.array([lb_to_ix[label] for label in target_labels]) logging.info("Load data time: {}".format(time.time() - load_time)) # Get audio ids for training and validation if train_txt is None or validate_txt is None: self.train_audio_indexes = np.arange(len(self.audio_names)) self.validate_audio_indexes = np.array([]) else: get_index_time = time.time() self.train_audio_indexes = self.get_audio_indexes(train_txt) self.validate_audio_indexes = self.get_audio_indexes(validate_txt) logging.info('Get indexes time: {:.3f} s'.format(time.time() - get_index_time)) logging.info('Training audios: {}'.format(len( self.train_audio_indexes))) logging.info('Validation audios: {}'.format( len(self.validate_audio_indexes))) # Scalar (self.mean, self.std) = calculate_scalar(self.x[self.train_audio_indexes]) hf.close()
def __init__(self, hdf5_path, batch_size, dev_train_csv=None, dev_validate_csv=None, seed=1234): """ Inputs: hdf5_path: str batch_size: int dev_train_csv: str | None, if None then use all data for training dev_validate_csv: str | None, if None then use all data for training seed: int, random seed """ self.batch_size = batch_size self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) lb_to_ix = config.lb_to_ix # 加载数据 load_time = time.time() hf = h5py.File(hdf5_path, 'r') self.audio_names = np.array([s.decode() for s in hf['filename'][:]]) self.x = hf['feature'][:] self.scene_labels = [s.decode() for s in hf['scene_label'][:]] self.identifiers = [s.decode() for s in hf['identifier'][:]] self.source_labels = [s.decode() for s in hf['source_label']] self.y = np.array([lb_to_ix[lb] for lb in self.scene_labels]) hf.close() logging.info('Loading data time: {:.3f} s'.format(time.time() - load_time)) # 使用所有的数据进行训练,也就是不分训练集和验证集 if dev_train_csv is None and dev_validate_csv is None: self.train_audio_indexes = np.arange(len(self.audio_names)) logging.info('Use all development data for training. ') # 使用部分数据进行训练,就是把所有数据分为训练集和验证集 else: self.train_audio_indexes = self.get_audio_indexes_from_csv( dev_train_csv) self.validate_audio_indexes = self.get_audio_indexes_from_csv( dev_validate_csv) logging.info('Split development data to {} training and {} ' 'validation data. '.format( len(self.train_audio_indexes), len(self.validate_audio_indexes))) # Calculate scalar (self.mean, self.std) = calculate_scalar(self.x[self.train_audio_indexes])
def __init__(self, hdf5_path, batch_size, validation_csv=None, holdout_fold=None, seed=1234): """Data generator. Args: hdf5_path: str, path of hdf5 file batch_size: int validation_csv: None | str, path of validation csv, if None then all data is used for training. holdout_fold: int, fold for validation seed: int """ self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) self.batch_size = batch_size self.holdout_fold = holdout_fold # Load data load_time = time.time() hf = h5py.File(hdf5_path, 'r') self.itemids = np.array([e.decode() for e in hf['itemid'][:]]) self.x = hf['feature'][:] self.y = hf['hasbird'][:][:, np.newaxis] self.datasetids = [e.decode() for e in hf['datasetid'][:]] hf.close() logging.info('Load hdf5 time: {} s'.format(time.time() - load_time)) # Get training & validation indexes (self.train_audio_indexes, self.valid_audio_indexes) = \ self.get_train_validate_audio_indexes(validation_csv, holdout_fold) logging.info('Training audios: {}'.format(len( self.train_audio_indexes))) logging.info('Validation audios: {}'.format( len(self.valid_audio_indexes))) # Calculate scalar (self.mean, self.std) = calculate_scalar(self.x[self.train_audio_indexes])
def fit(args): """ Calculate scalar. Args: feature_dir: feature path data_type: 'dev' | 'eval' audio_type: 'foa' | 'mic' """ if args.data_type == 'train/weak': hdf5_dir = os.path.join(args.feature_dir, args.feature_type, hdf5_folder_name, 'train/weak/') elif args.data_type == 'train/unlabel_in_domain': hdf5_dir = os.path.join(args.feature_dir, args.feature_type, hdf5_folder_name, 'train/unlabel_in_domain/') elif args.data_type == 'validation': hdf5_dir = os.path.join(args.feature_dir, args.feature_type, hdf5_folder_name, 'validation/') elif args.data_type == 'eval': hdf5_dir = os.path.join(args.feature_dir, args.feature_type, hdf5_folder_name, 'eval/') scalar_path = os.path.join(args.feature_dir, args.feature_type, hdf5_folder_name, 'scalar.h5') os.makedirs(os.path.dirname(scalar_path), exist_ok=True) print('\n============> Start Calculating Scalar.\n') load_time = timer() features = [] for hdf5_fn in os.listdir(hdf5_dir): hdf5_path = os.path.join(hdf5_dir, hdf5_fn) with h5py.File(hdf5_path, 'r') as hf: features.append(hf['feature'][:]) print('Load feature time: {:.3f} s'.format(timer() - load_time)) features = np.concatenate(features, axis=1) (mean, std) = calculate_scalar(features) with h5py.File(scalar_path, 'w') as hf_scalar: hf_scalar.create_dataset('mean', data=mean, dtype=np.float32) hf_scalar.create_dataset('std', data=std, dtype=np.float32) print('Features shape: {}'.format(features.shape)) print('mean {}:\n{}'.format(mean.shape, mean)) print('std {}:\n{}'.format(std.shape, std)) print('Write out scalar to {}'.format(scalar_path))
def calculate_training_data_scalar(self): """Concatenate all training data and calculate scalar. """ train_bgn_fin_indices = self.bgn_fin_indices[self.train_audio_indexes] train_x_concat = [] for [bgn, fin] in train_bgn_fin_indices: train_x_concat.append(self.x[bgn : fin]) train_x_concat = np.concatenate(train_x_concat, axis=0) (mean, std) = calculate_scalar(train_x_concat) return mean, std
def __init__(self, hdf5_path, batch_size, fold_for_validation, scale, seed=1234): self.random_state = np.random.RandomState(seed) # Load data hf = h5py.File(hdf5_path, 'r') self.x = hf['feature'][:] self.y = hf['hasbird'][:][:, np.newaxis] self.itemids = [e.decode() for e in hf['itemid'][:]] self.datasetids = [e.decode() for e in hf['datasetid'][:]] self.folds = hf['fold'][:] hf.close() self.batch_size = batch_size self.fold_for_validation = fold_for_validation self.scale = scale (self.mean, self.std) = calculate_scalar(self.x) self.tr_samples = np.sum(self.folds != fold_for_validation) self.va_samples = np.sum(self.folds == fold_for_validation) (self.tr_indexes, self.va_indexes) = self.get_init_indexes()
def __init__(self, hdf5_path, batch_size, holdout_fold, seed=1234): """ Inputs: hdf5_path: str batch_size: int holdout_fold: int seed: int, random seed """ self.batch_size = batch_size self.holdout_fold = holdout_fold self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) # Load data load_time = time.time() hf = h5py.File(hdf5_path, 'r') self.audio_names = np.array([s.decode() for s in hf['audio_name'][:]]) self.x = hf['logmel'][:] self.y = hf['target'][:] self.folds = hf['fold'][:] hf.close() logging.info('Loading data time: {:.3f} s'.format( time.time() - load_time)) # Split data to training and validation self.train_audio_indexes, self.validate_audio_indexes = \ self.get_train_validate_audio_indexes() # Calculate scalar (self.mean, self.std) = calculate_scalar( self.x[self.train_audio_indexes])
def __init__(self, train_hdf5_path, validate_hdf5_path, batch_size, validate, seed=1234): """Data generator. Args: train_hdf5_path: str, path of train hdf5 file validate_hdf5_path: str, path of validate hdf5 path batch_size: int validate: bool seed: int """ self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) lb_to_ix = config.lb_to_ix self.batch_size = batch_size self.validate = validate # Load data load_time = time.time() hf = h5py.File(train_hdf5_path, 'r') self.train_audio_names = np.array( [s.decode() for s in hf['audio_name'][:]]) self.train_x = hf['feature'][:] self.train_y = hf['target'][:] hf.close() hf = h5py.File(validate_hdf5_path, 'r') self.validate_audio_names = np.array( [s.decode() for s in hf['audio_name']]) self.validate_x = hf['feature'][:] self.validate_y = hf['target'][:] hf.close() logging.info('Loading data time: {:.3f} s' ''.format(time.time() - load_time)) # Get train & validate audio indexes self.audio_names = np.concatenate( (self.train_audio_names, self.validate_audio_names), axis=0) self.x = np.concatenate((self.train_x, self.validate_x), axis=0) self.y = np.concatenate((self.train_y, self.validate_y), axis=0) if validate: self.train_audio_indexes = np.arange(len(self.train_audio_names)) self.validate_audio_indexes = np.arange( len(self.train_audio_names), len(self.train_audio_names) + len(self.validate_audio_names)) else: self.train_audio_indexes = np.arange(len(self.audio_names)) self.validate_audio_indexes = np.array([]) logging.info("Training audios: {}".format(len( self.train_audio_indexes))) logging.info("Validation audios: {}".format( len(self.validate_audio_indexes))) # Calculate scalar (self.mean, self.std) = calculate_scalar(self.x[self.train_audio_indexes])
def __init__(self, hdf5_path, target_device, train_house_list, validate_house_list, batch_size, seq_len, width, random_seed=1234, binary_threshold=None, balance_threshold=None, balance_positive=0.5): """Data generator. Args: hdf5_path: string, path of hdf5 file. target_device: string, e.g. 'washingmachine' train_house_list: list of string, e.g. ['house1', 'house2'] validate_house_list: list of string, e.g. ['house3'] batch_size: int seq_len: int width: int random_seed: int binary_threshold: None or float """ self.target_device = target_device self.batch_size = batch_size assert seq_len % 2 == 1, "seq_len has to be odd, otherwise padding will be off by 1" self.seq_len = seq_len self.width = width self.random_state = np.random.RandomState(random_seed) self.validate_random_state = np.random.RandomState(1) self.validate_house_list = validate_house_list self.binary_threshold = binary_threshold self.balance_threshold = balance_threshold self.balance_positive = balance_positive if self.balance_threshold is not None: logging.info('----balance threshold is not none and use balance generation----') self.generate = self._generate_balanced else: logging.info('----balance threshold is none and normal generation is used----') self.generate = self._generate assert len(train_house_list) > 0 # Load hdf5 file load_time = time.time() self.hf = h5py.File(hdf5_path, 'r') # Load training data # NOTE read_data will return binarized data if binary_threshold is set to a float (self.train_x, self.train_y) = self.read_data( self.hf, target_device, train_house_list) # Load validation data # NOTE read_data will return binarized data if binary_threshold is set to a float (self.validate_x, self.validate_y) = self.read_data( self.hf, target_device, validate_house_list) logging.info("Load data time: {} s".format(time.time() - load_time)) # Calculate scalar (self.mean_x, self.std_x, self.max_x) = calculate_scalar(self.train_x) (self.mean_y, self.std_y, self.max_y) = calculate_scalar(self.train_y) logging.info('mean_x: {}, std_x: {}, max_x: {}'.format(self.mean_x, self.std_x, self.max_x)) logging.info('mean_y: {}, std_y: {}, max_y: {}:'.format(self.mean_y, self.std_y, self.max_y)) # Training indexes self.train_indexes = np.arange( 0, len(self.train_x) - seq_len - width + 2, width) self.train_indexes = self.valid_data(self.train_x, self.train_y, self.train_indexes) # Validation indexes self.validate_indexes = np.arange( 0, len(self.validate_x) - seq_len - width + 2, width) self.validate_indexes = self.valid_data(self.validate_x, self.validate_y, self.validate_indexes) logging.info("Number of indexes: {}".format(len(self.train_indexes)))
def __init__(self, hdf5_path, batch_size, dev_train_csv=None, dev_validate_csv=None, seed=1234): """ Inputs: hdf5_path: str batch_size: int dev_train_csv: str | None, if None then use all data for training dev_validate_csv: str | None, if None then use all data for training seed: int, random seed """ self.batch_size = batch_size self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) lb_to_ix = config.lb_to_ix # Load data load_time = time.time() hf = h5py.File(hdf5_path, 'r') self.audio_names = np.array([s.decode() for s in hf['filename'][:]]) self.x = hf['feature'][:] self.emotion_labels = [s.decode() for s in hf['emotion_label'][:]] self.y = np.array([lb_to_ix[lb] for lb in self.emotion_labels]) hf.close() logging.info('Loading data time: {:.3f} s'.format(time.time() - load_time)) # Split data to training and validation self.train_audio_indexes = self.get_audio_indexes_from_csv( dev_train_csv) self.validate_audio_indexes = self.get_audio_indexes_from_csv( dev_validate_csv) # upsample # y_train = self.y[self.train_audio_indexes] # num = [list(y_train).count(0), list(y_train).count(1), list(y_train).count(2)] # num_max = np.argmax(np.array(num)) # # for i in range(0, len(num)): # if i == num_max: # continue # else: # y_train_temp = np.argwhere(y_train == i) # y_train_temp = [j[0] for j in y_train_temp] # number = num[num_max] - num[i] # train_audio_indexes_temp = [self.train_audio_indexes[j] for j in y_train_temp] # train_audio_indexes_choice = np.random.choice(train_audio_indexes_temp, number) # for j in train_audio_indexes_choice: # self.train_audio_indexes.append(j) # # self.train_audio_indexes = list(itertools.chain.from_iterable(self.train_audio_indexes)) logging.info('Split development data to {} training and {} ' 'validation data. '.format( len(self.train_audio_indexes), len(self.validate_audio_indexes))) # Calculate scalar (self.mean, self.std) = calculate_scalar(self.x[self.train_audio_indexes])
def __init__(self, hdf5_path, batch_size, dev_train_csv=None, dev_validate_csv=None, seed=1234): """ Inputs: hdf5_path: str batch_size: int dev_train_csv: str | None, if None then use all data for training dev_validate_csv: str | None, if None then use all data for training seed: int, random seed """ self.batch_size = batch_size self.random_state = np.random.RandomState(seed) self.validate_random_state = np.random.RandomState(0) lb_to_ix = config.lb_to_ix lb_to_ix_domain = config.lb_to_ix_domain # Load data load_time = time.time() hf = h5py.File(hdf5_path, 'r') self.audio_names = np.array([s.decode() for s in hf['filename'][:]]) self.x = hf['feature'][:] self.scene_labels = [s.decode() for s in hf['scene_label'][:]] self.identifiers = [s.decode() for s in hf['identifier'][:]] self.source_labels = [s.decode() for s in hf['source_label']] self.y = np.array([lb_to_ix[lb] for lb in self.scene_labels]) self.device = np.array( [lb_to_ix_domain[lbd] for lbd in self.source_labels]) self.device = self.device.reshape(len(self.device), -1) onehot = OneHotEncoder() onehot.fit(self.device) self.device = onehot.transform(self.device).toarray() hf.close() logging.info('Loading data time: {:.3f} s'.format(time.time() - load_time)) # Use all data for training if dev_train_csv is None and dev_validate_csv is None: self.train_audio_indexes = np.arange(len(self.audio_names)) logging.info('Use all development data for training. ') # Split data to training and validation else: self.train_audio_indexes = self.get_audio_indexes_from_csv( dev_train_csv) self.validate_audio_indexes = self.get_audio_indexes_from_csv( dev_validate_csv) logging.info('Split development data to {} training and {} ' 'validation data. '.format( len(self.train_audio_indexes), len(self.validate_audio_indexes))) # Calculate scalar (self.mean, self.std) = calculate_scalar(self.x[self.train_audio_indexes])