def evaluate_latent(fn, feeder, title): y_true = [] Z = [] for outputs in Progbar(feeder.set_batch(batch_mode='file'), name=title, print_report=True, print_summary=False, count_func=lambda x: x[-1].shape[0]): name = str(outputs[0]) idx = int(outputs[1]) data = outputs[2:] assert idx == 0 y_true.append(name) Z.append(fn(*data)) Z = np.concatenate(Z, axis=0) # ====== visualize spectrogram ====== # if Z.ndim >= 3: sample = np.random.choice(range(len(Z)), size=3, replace=False) spec = Z[sample.astype('int32')] y = [y_true[int(i)] for i in sample] plot_figure(nrow=6, ncol=6) for i, (s, tit) in enumerate(zip(spec, y)): s = s.reshape(len(s), -1) plot_spectrogram(s.T, ax=(1, 3, i + 1), title=tit) # ====== visualize each point ====== # # flattent to 2D Z = np.reshape(Z, newshape=(len(Z), -1)) # tsne if necessary if Z.shape[-1] > 3: Z = fast_tsne(Z, n_components=3, n_jobs=8, random_state=K.get_rng().randint(0, 10e8)) # color and marker Z_color = [digit_color_map[i.split('_')[-1]] for i in y_true] Z_marker = [gender_marker_map[i.split('_')[1]] for i in y_true] plot_figure(nrow=6, ncol=20) for i, azim in enumerate((15, 60, 120)): plot_scatter(x=Z[:, 0], y=Z[:, 1], z=Z[:, 2], ax=(1, 3, i + 1), size=4, color=Z_color, marker=Z_marker, azim=azim, legend=legends if i == 1 else None, legend_ncol=11, fontsize=10, title=title) plot_save(os.path.join(FIG_PATH, '%s.pdf' % title))
# =========================================================================== update_ops = K.optimizers.Adam(lr=0.001).minimize(loss) K.initialize_all_variables() # ====== intitalize ====== # record_train_loss = [] record_valid_loss = [] patience = 3 epoch = 0 # We want the rate to go up but the distortion to go down while True: # ====== training ====== # train_losses = [] prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch) start_time = timeit.default_timer() for start, end in batching(batch_size=args.bs, n=X_train.shape[0], seed=K.get_rng().randint(10e8)): _ = K.eval(loss, feed_dict={X: X_train[start:end]}, update_after=update_ops) prog.add(end - start) train_losses.append(_) # ====== training log ====== # print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time)) print("[Training set] Loss: %.4f" % np.mean(train_losses)) # ====== validation set ====== # code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid}) print("[Valid set] Loss: %.4f" % lo) # ====== record the history ====== # record_train_loss.append(np.mean(train_losses)) record_valid_loss.append(lo) # ====== plotting ====== # if args.dim > 2:
functions=[f_pred_proba, f_z1, f_z2, f_z3], X=X_test_data, title='TEST') print("Test Latent:", Z1_test.shape, Z2_test.shape, Z3_test.shape) y_pred = np.argmax(y_pred_proba, axis=-1) evaluate(y_true=X_test_true, y_pred_proba=y_pred_proba, labels=labels, title="Test set (Deep prediction)", path=os.path.join(EXP_DIR, 'test_deep.pdf')) # ====== make a streamline classifier ====== # # training PLDA Z3_train, y_train = make_dnn_prediction(f_z3, X=train, title="TRAIN") print("Z3_train:", Z3_train.shape, y_train.shape) Z3_valid, y_valid = make_dnn_prediction(f_z3, X=valid, title="VALID") print("Z3_valid:", Z3_valid.shape, y_valid.shape) plda = PLDA(n_phi=200, random_state=K.get_rng().randint(10e8), n_iter=12, labels=labels, verbose=0) plda.fit(np.concatenate([Z3_train, Z3_valid], axis=0), np.concatenate([y_train, y_valid], axis=0)) y_pred_log_proba = plda.predict_log_proba(Z3_test) evaluate(y_true=X_test_true, y_pred_log_proba=y_pred_log_proba, labels=labels, title="Test set (PLDA - Latent prediction)", path=os.path.join(EXP_DIR, 'test_latent.pdf')) # ====== visualize ====== # visualize_latent_space(X_org=X_test_data, X_latent=Z1_test, name=X_test_name,
# =========================================================================== update_ops = K.optimizers.Adam(lr=0.001).minimize(loss) K.initialize_all_variables() # ====== intitalize ====== # record_train_loss = [] record_valid_loss = [] patience = 3 epoch = 0 # We want the rate to go up but the distortion to go down while True: # ====== training ====== # train_losses = [] prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch) start_time = timeit.default_timer() for start, end in batching(batch_size=args.bs, n=X_train.shape[0], seed=K.get_rng().randint(10e8)): _ = K.eval(loss, feed_dict={X: X_train[start:end]}, update_after=update_ops) prog.add(end - start) train_losses.append(_) # ====== training log ====== # print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time)) print("[Training set] Loss: %.4f" % np.mean(train_losses)) # ====== validation set ====== # code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid}) print("[Valid set] Loss: %.4f" % lo) # ====== record the history ====== # record_train_loss.append(np.mean(train_losses)) record_valid_loss.append(lo) # ====== plotting ====== # if args.dim > 2:
from odin.ml import PLDA from sklearn.decomposition import PCA from sklearn.manifold import TSNE from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.mixture import GaussianMixture from sklearn.neural_network import BernoulliRBM # =========================================================================== # Const # =========================================================================== TRAINING_PERCENT = 0.5 POINT_SIZE = 8 NUM_DIM = 3 colors = ['r', 'b', 'g'] markers = ["o", "^", "s"] SEED = K.get_rng().randint(0, 10e8) # =========================================================================== # Load dataset # =========================================================================== ds = F.IRIS.load() print(ds) nb_samples = ds['X'].shape[0] ids = K.get_rng().permutation(nb_samples) X = ds['X'][ids] y = ds['y'][ids] labels = ds['name'] print("Labels:", ctext(labels)) assert len(colors) == len(labels) and len(markers) == len(labels) X_train = X[:int(TRAINING_PERCENT * nb_samples)]
def prepare_data(feat, label, utt_length=0.4, for_ivec=False): """ Returns (i-vector) ------------------ ds[feat] train_files y_train test_files y_test labels Returns (x-vector) ------------------ train : Feeder feeder for training data for iterating over pair of (X, y) valid : Feeder feeder for validating data for iterating over pair of (X, y) X_test_name : list of file names file names are append with '.%d' for cut segment ID X_test_true : list of integer label of each sample X_test_data : array list of test data same length as X_test_name labels : list of string list of labels for classification task Example ------- (train, valid, X_test_name, X_test_true, X_test_data, labels) = prepare_data_dnn(feat=FEAT, label='gender') """ label = str(label).lower() assert label in _support_label, "No support for label: %s" % label assert 0 < utt_length <= 1. # ====== load dataset ====== # if not os.path.exists(PATH_ACOUSTIC): raise RuntimeError( "Cannot find extracted acoustic features at path: '%s'," "run the code speech_features_extraction.py!" % PATH_ACOUSTIC) ds = F.Dataset(PATH_ACOUSTIC, read_only=True) assert feat in ds, "Cannot find feature with name: %s" % feat indices = list(ds['indices'].items()) K.get_rng().shuffle(indices) # ====== helper ====== # def is_train(x): return x.split('_')[0] == 'train' def extract_label(x): return x.split('_')[_support_label[label]] print("Task:", ctext(label, 'cyan')) fn_label, labels = unique_labels([i[0] for i in indices], key_func=extract_label, return_labels=True) print("Labels:", ctext(labels, 'cyan')) # ====== training and test data ====== # train_files = [] # (name, (start, end)) ... test_files = [] for name, (start, end) in indices: if is_train(name): train_files.append((name, (start, end))) else: test_files.append((name, (start, end))) # name for each dataset, useful for later print("#Train:", ctext(len(train_files), 'cyan')) print("#Test:", ctext(len(test_files), 'cyan')) # ====== for i-vectors ====== # y_train = np.array([fn_label(i[0]) for i in train_files]) y_test = np.array([fn_label(i[0]) for i in test_files]) if bool(for_ivec): return ds[feat], train_files, y_train, test_files, y_test, labels # ====== length ====== # length = [(end - start) for _, (start, end) in indices] max_length = max(length) frame_length = int(max_length * utt_length) step_length = frame_length print("Max length :", ctext(max_length, 'yellow')) print("Frame length:", ctext(frame_length, 'yellow')) print("Step length :", ctext(step_length, 'yellow')) # ====== split dataset ====== # # split by speaker ID train_files, valid_files = train_valid_test_split( x=train_files, train=0.8, cluster_func=None, idfunc=lambda x: x[0].split('_')[4], # splited by speaker inc_test=False) print("#File train:", ctext(len(train_files), 'cyan')) print("#File valid:", ctext(len(valid_files), 'cyan')) print("#File test :", ctext(len(test_files), 'cyan')) recipes = [ F.recipes.Sequencing(frame_length=frame_length, step_length=step_length, end='pad', pad_mode='post', pad_value=0), F.recipes.Name2Label(converter_func=fn_label), F.recipes.LabelOneHot(nb_classes=len(labels), data_idx=-1) ] feeder_train = F.Feeder(F.IndexedData(ds[feat], indices=train_files), ncpu=6, batch_mode='batch') feeder_valid = F.Feeder(F.IndexedData(ds[feat], indices=valid_files), ncpu=4, batch_mode='batch') feeder_test = F.Feeder(F.IndexedData(ds[feat], indices=test_files), ncpu=4, batch_mode='file') feeder_train.set_recipes(recipes) feeder_valid.set_recipes(recipes) feeder_test.set_recipes(recipes) print(feeder_train) # ====== process X_test, y_test in advance for faster evaluation ====== # @cache_disk def _extract_test_data(feat, label, utt_length): prog = Progbar(target=len(feeder_test), print_summary=True, name="Preprocessing test set") X_test = defaultdict(list) for name, idx, X, y in feeder_test: # validate everything as expected assert fn_label(name) == np.argmax(y), name # label is right # save to list X_test[name].append((idx, X)) prog.add(X.shape[0]) # ====== create 1 array for data and dictionary for indices ====== # X_test_name = [] X_test_data = [] for name, X in X_test.items(): X = np.concatenate([x[1] for x in sorted(X, key=lambda i: i[0])], axis=0).astype('float16') X_test_name += [name + '.%d' % i for i in range(len(X))] X_test_data.append(X) X_test_name = np.array(X_test_name) X_test_data = np.concatenate(X_test_data, axis=0) return X_test_name, X_test_data # convert everything back to float32 X_test_name, X_test_data = _extract_test_data(feat, label, utt_length) X_test_true = np.array([fn_label(i.split('.')[0]) for i in X_test_name]) return feeder_train, feeder_valid, \ X_test_name, X_test_true, X_test_data, labels
from odin.ml import PLDA from sklearn.decomposition import PCA from sklearn.manifold import TSNE from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.mixture import GaussianMixture from sklearn.neural_network import BernoulliRBM # =========================================================================== # Const # =========================================================================== TRAINING_PERCENT = 0.5 POINT_SIZE = 8 NUM_DIM = 3 colors = ['r', 'b', 'g'] markers = ["o", "^", "s"] SEED = K.get_rng().randint(0, 10e8) # =========================================================================== # Load dataset # =========================================================================== ds = F.IRIS.load() print(ds) nb_samples = ds['X'].shape[0] ids = K.get_rng().permutation(nb_samples) X = ds['X'][ids] y = ds['y'][ids] labels = ds['name'] print("Labels:", ctext(labels)) assert len(colors) == len(labels) and len(markers) == len(labels) X_train = X[:int(TRAINING_PERCENT * nb_samples)]
# Prediction # =========================================================================== y_pred_proba, Z1_test, Z2_test, Z3_test = make_dnn_prediction( functions=[f_pred_proba, f_z1, f_z2, f_z3], X=X_test_data, title='TEST') print("Test Latent:", Z1_test.shape, Z2_test.shape, Z3_test.shape) y_pred = np.argmax(y_pred_proba, axis=-1) evaluate(y_true=X_test_true, y_pred_proba=y_pred_proba, labels=labels, title="Test set (Deep prediction)", path=os.path.join(EXP_DIR, 'test_deep.pdf')) # ====== make a streamline classifier ====== # # training PLDA Z3_train, y_train = make_dnn_prediction(f_z3, X=train, title="TRAIN") print("Z3_train:", Z3_train.shape, y_train.shape) Z3_valid, y_valid = make_dnn_prediction(f_z3, X=valid, title="VALID") print("Z3_valid:", Z3_valid.shape, y_valid.shape) plda = PLDA(n_phi=200, random_state=K.get_rng().randint(10e8), n_iter=12, labels=labels, verbose=0) plda.fit(np.concatenate([Z3_train, Z3_valid], axis=0), np.concatenate([y_train, y_valid], axis=0)) y_pred_log_proba = plda.predict_log_proba(Z3_test) evaluate(y_true=X_test_true, y_pred_log_proba=y_pred_log_proba, labels=labels, title="Test set (PLDA - Latent prediction)", path=os.path.join(EXP_DIR, 'test_latent.pdf')) # ====== visualize ====== # visualize_latent_space(X_org=X_test_data, X_latent=Z1_test, name=X_test_name, labels=X_test_true, title="latent1") visualize_latent_space(X_org=X_test_data, X_latent=Z2_test, name=X_test_name, labels=X_test_true, title="latent2") V.plot_save(os.path.join(EXP_DIR, 'latent.pdf'))
BATCH_SIZE = 32 NB_EPOCH = 20 NB_SAMPLES = 8 VALID_PERCENTAGE = 0.4 # =========================================================================== # Load dataset # =========================================================================== path = get_datasetpath(name='TIDIGITS_feats', override=False) assert os.path.isdir(path), \ "Cannot find preprocessed feature at: %s, try to run 'odin/examples/features.py'" % path ds = F.Dataset(path, read_only=True) assert all(f in ds for f in FEAT), "Cannot find features with name: %s" % FEAT # ====== get all the indices of single digit ====== # indices = [(name, (s, e)) for name, (s, e) in list(ds['indices'].items()) if len(name.split('_')[-1]) == 1] K.get_rng().shuffle(indices) print("Found %s utterances of single digit" % ctext(len(indices), 'cyan')) # =========================================================================== # Load and visual the dataset # =========================================================================== train = [] test = [] max_length = 0 min_length = np.inf for name, (start, end) in indices: assert end - start > 0 if name.split('_')[0] == 'train': train.append((name, (start, end))) else: test.append((name, (start, end))) max_length = max(end - start, max_length)
def prepare_data(feat, label, utt_length=0.4, for_ivec=False): """ Returns (i-vector) ------------------ ds[feat] train_files y_train test_files y_test labels Returns (x-vector) ------------------ train : Feeder feeder for training data for iterating over pair of (X, y) valid : Feeder feeder for validating data for iterating over pair of (X, y) X_test_name : list of file names file names are append with '.%d' for cut segment ID X_test_true : list of integer label of each sample X_test_data : array list of test data same length as X_test_name labels : list of string list of labels for classification task Example ------- (train, valid, X_test_name, X_test_true, X_test_data, labels) = prepare_data_dnn(feat=FEAT, label='gender') """ label = str(label).lower() assert label in _support_label, "No support for label: %s" % label assert 0 < utt_length <= 1. # ====== load dataset ====== # if not os.path.exists(PATH_ACOUSTIC): raise RuntimeError("Cannot find extracted acoustic features at path: '%s'," "run the code speech_features_extraction.py!" % PATH_ACOUSTIC) ds = F.Dataset(PATH_ACOUSTIC, read_only=True) assert feat in ds, "Cannot find feature with name: %s" % feat indices = list(ds['indices'].items()) K.get_rng().shuffle(indices) # ====== helper ====== # def is_train(x): return x.split('_')[0] == 'train' def extract_label(x): return x.split('_')[_support_label[label]] print("Task:", ctext(label, 'cyan')) fn_label, labels = unique_labels([i[0] for i in indices], key_func=extract_label, return_labels=True) print("Labels:", ctext(labels, 'cyan')) # ====== training and test data ====== # train_files = [] # (name, (start, end)) ... test_files = [] for name, (start, end) in indices: if is_train(name): train_files.append((name, (start, end))) else: test_files.append((name, (start, end))) # name for each dataset, useful for later print("#Train:", ctext(len(train_files), 'cyan')) print("#Test:", ctext(len(test_files), 'cyan')) # ====== for i-vectors ====== # y_train = np.array([fn_label(i[0]) for i in train_files]) y_test = np.array([fn_label(i[0]) for i in test_files]) if bool(for_ivec): return ds[feat], train_files, y_train, test_files, y_test, labels # ====== length ====== # length = [(end - start) for _, (start, end) in indices] max_length = max(length) frame_length = int(max_length * utt_length) step_length = frame_length print("Max length :", ctext(max_length, 'yellow')) print("Frame length:", ctext(frame_length, 'yellow')) print("Step length :", ctext(step_length, 'yellow')) # ====== split dataset ====== # # split by speaker ID train_files, valid_files = train_valid_test_split( x=train_files, train=0.8, cluster_func=None, idfunc=lambda x: x[0].split('_')[4], # splited by speaker inc_test=False) print("#File train:", ctext(len(train_files), 'cyan')) print("#File valid:", ctext(len(valid_files), 'cyan')) print("#File test :", ctext(len(test_files), 'cyan')) recipes = [ F.recipes.Sequencing(frame_length=frame_length, step_length=step_length, end='pad', pad_mode='post', pad_value=0), F.recipes.Name2Label(converter_func=fn_label), F.recipes.LabelOneHot(nb_classes=len(labels), data_idx=-1) ] feeder_train = F.Feeder(F.IndexedData(ds[feat], indices=train_files), ncpu=6, batch_mode='batch') feeder_valid = F.Feeder(F.IndexedData(ds[feat], indices=valid_files), ncpu=4, batch_mode='batch') feeder_test = F.Feeder(F.IndexedData(ds[feat], indices=test_files), ncpu=4, batch_mode='file') feeder_train.set_recipes(recipes) feeder_valid.set_recipes(recipes) feeder_test.set_recipes(recipes) print(feeder_train) # ====== process X_test, y_test in advance for faster evaluation ====== # @cache_disk def _extract_test_data(feat, label, utt_length): prog = Progbar(target=len(feeder_test), print_summary=True, name="Preprocessing test set") X_test = defaultdict(list) for name, idx, X, y in feeder_test: # validate everything as expected assert fn_label(name) == np.argmax(y), name # label is right # save to list X_test[name].append((idx, X)) prog.add(X.shape[0]) # ====== create 1 array for data and dictionary for indices ====== # X_test_name = [] X_test_data = [] for name, X in X_test.items(): X = np.concatenate([x[1] for x in sorted(X, key=lambda i: i[0])], axis=0).astype('float16') X_test_name += [name + '.%d' % i for i in range(len(X))] X_test_data.append(X) X_test_name = np.array(X_test_name) X_test_data = np.concatenate(X_test_data, axis=0) return X_test_name, X_test_data # convert everything back to float32 X_test_name, X_test_data = _extract_test_data(feat, label, utt_length) X_test_true = np.array([fn_label(i.split('.')[0]) for i in X_test_name]) return feeder_train, feeder_valid, \ X_test_name, X_test_true, X_test_data, labels