def evaluate_latent(fn, feeder, title):
y_true = []
Z = []
for outputs in Progbar(feeder.set_batch(batch_mode='file'),
name=title,
print_report=True,
print_summary=False,
count_func=lambda x: x[-1].shape[0]):
name = str(outputs[0])
idx = int(outputs[1])
data = outputs[2:]
assert idx == 0
y_true.append(name)
Z.append(fn(*data))
Z = np.concatenate(Z, axis=0)
# ====== visualize spectrogram ====== #
if Z.ndim >= 3:
sample = np.random.choice(range(len(Z)), size=3, replace=False)
spec = Z[sample.astype('int32')]
y = [y_true[int(i)] for i in sample]
plot_figure(nrow=6, ncol=6)
for i, (s, tit) in enumerate(zip(spec, y)):
s = s.reshape(len(s), -1)
plot_spectrogram(s.T, ax=(1, 3, i + 1), title=tit)
# ====== visualize each point ====== #
# flattent to 2D
Z = np.reshape(Z, newshape=(len(Z), -1))
# tsne if necessary
if Z.shape[-1] > 3:
Z = fast_tsne(Z,
n_components=3,
n_jobs=8,
random_state=K.get_rng().randint(0, 10e8))
# color and marker
Z_color = [digit_color_map[i.split('_')[-1]] for i in y_true]
Z_marker = [gender_marker_map[i.split('_')[1]] for i in y_true]
plot_figure(nrow=6, ncol=20)
for i, azim in enumerate((15, 60, 120)):
plot_scatter(x=Z[:, 0],
y=Z[:, 1],
z=Z[:, 2],
ax=(1, 3, i + 1),
size=4,
color=Z_color,
marker=Z_marker,
azim=azim,
legend=legends if i == 1 else None,
legend_ncol=11,
fontsize=10,
title=title)
plot_save(os.path.join(FIG_PATH, '%s.pdf' % title))
# ===========================================================================
update_ops = K.optimizers.Adam(lr=0.001).minimize(loss)
K.initialize_all_variables()
# ====== intitalize ====== #
record_train_loss = []
record_valid_loss = []
patience = 3
epoch = 0
# We want the rate to go up but the distortion to go down
while True:
# ====== training ====== #
train_losses = []
prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch)
start_time = timeit.default_timer()
for start, end in batching(batch_size=args.bs, n=X_train.shape[0],
seed=K.get_rng().randint(10e8)):
_ = K.eval(loss, feed_dict={X: X_train[start:end]},
update_after=update_ops)
prog.add(end - start)
train_losses.append(_)
# ====== training log ====== #
print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time))
print("[Training set] Loss: %.4f" % np.mean(train_losses))
# ====== validation set ====== #
code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid})
print("[Valid set] Loss: %.4f" % lo)
# ====== record the history ====== #
record_train_loss.append(np.mean(train_losses))
record_valid_loss.append(lo)
# ====== plotting ====== #
if args.dim > 2:
functions=[f_pred_proba, f_z1, f_z2, f_z3], X=X_test_data, title='TEST')
print("Test Latent:", Z1_test.shape, Z2_test.shape, Z3_test.shape)
y_pred = np.argmax(y_pred_proba, axis=-1)
evaluate(y_true=X_test_true,
y_pred_proba=y_pred_proba,
labels=labels,
title="Test set (Deep prediction)",
path=os.path.join(EXP_DIR, 'test_deep.pdf'))
# ====== make a streamline classifier ====== #
# training PLDA
Z3_train, y_train = make_dnn_prediction(f_z3, X=train, title="TRAIN")
print("Z3_train:", Z3_train.shape, y_train.shape)
Z3_valid, y_valid = make_dnn_prediction(f_z3, X=valid, title="VALID")
print("Z3_valid:", Z3_valid.shape, y_valid.shape)
plda = PLDA(n_phi=200,
random_state=K.get_rng().randint(10e8),
n_iter=12,
labels=labels,
verbose=0)
plda.fit(np.concatenate([Z3_train, Z3_valid], axis=0),
np.concatenate([y_train, y_valid], axis=0))
y_pred_log_proba = plda.predict_log_proba(Z3_test)
evaluate(y_true=X_test_true,
y_pred_log_proba=y_pred_log_proba,
labels=labels,
title="Test set (PLDA - Latent prediction)",
path=os.path.join(EXP_DIR, 'test_latent.pdf'))
# ====== visualize ====== #
visualize_latent_space(X_org=X_test_data,
X_latent=Z1_test,
name=X_test_name,
# ===========================================================================
update_ops = K.optimizers.Adam(lr=0.001).minimize(loss)
K.initialize_all_variables()
# ====== intitalize ====== #
record_train_loss = []
record_valid_loss = []
patience = 3
epoch = 0
# We want the rate to go up but the distortion to go down
while True:
# ====== training ====== #
train_losses = []
prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch)
start_time = timeit.default_timer()
for start, end in batching(batch_size=args.bs, n=X_train.shape[0],
seed=K.get_rng().randint(10e8)):
_ = K.eval(loss, feed_dict={X: X_train[start:end]},
update_after=update_ops)
prog.add(end - start)
train_losses.append(_)
# ====== training log ====== #
print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time))
print("[Training set] Loss: %.4f" % np.mean(train_losses))
# ====== validation set ====== #
code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid})
print("[Valid set] Loss: %.4f" % lo)
# ====== record the history ====== #
record_train_loss.append(np.mean(train_losses))
record_valid_loss.append(lo)
# ====== plotting ====== #
if args.dim > 2:
from odin.ml import PLDA
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.mixture import GaussianMixture
from sklearn.neural_network import BernoulliRBM
# ===========================================================================
# Const
# ===========================================================================
TRAINING_PERCENT = 0.5
POINT_SIZE = 8
NUM_DIM = 3
colors = ['r', 'b', 'g']
markers = ["o", "^", "s"]
SEED = K.get_rng().randint(0, 10e8)
# ===========================================================================
# Load dataset
# ===========================================================================
ds = F.IRIS.load()
print(ds)
nb_samples = ds['X'].shape[0]
ids = K.get_rng().permutation(nb_samples)
X = ds['X'][ids]
y = ds['y'][ids]
labels = ds['name']
print("Labels:", ctext(labels))
assert len(colors) == len(labels) and len(markers) == len(labels)
X_train = X[:int(TRAINING_PERCENT * nb_samples)]
def prepare_data(feat, label, utt_length=0.4, for_ivec=False):
"""
Returns (i-vector)
------------------
ds[feat]
train_files
y_train
test_files
y_test
labels
Returns (x-vector)
------------------
train : Feeder
feeder for training data for iterating over pair of (X, y)
valid : Feeder
feeder for validating data for iterating over pair of (X, y)
X_test_name : list of file names
file names are append with '.%d' for cut segment ID
X_test_true : list of integer
label of each sample
X_test_data : array
list of test data same length as X_test_name
labels : list of string
list of labels for classification task
Example
-------
(train, valid,
X_test_name, X_test_true, X_test_data,
labels) = prepare_data_dnn(feat=FEAT, label='gender')
"""
label = str(label).lower()
assert label in _support_label, "No support for label: %s" % label
assert 0 < utt_length <= 1.
# ====== load dataset ====== #
if not os.path.exists(PATH_ACOUSTIC):
raise RuntimeError(
"Cannot find extracted acoustic features at path: '%s',"
"run the code speech_features_extraction.py!" % PATH_ACOUSTIC)
ds = F.Dataset(PATH_ACOUSTIC, read_only=True)
assert feat in ds, "Cannot find feature with name: %s" % feat
indices = list(ds['indices'].items())
K.get_rng().shuffle(indices)
# ====== helper ====== #
def is_train(x):
return x.split('_')[0] == 'train'
def extract_label(x):
return x.split('_')[_support_label[label]]
print("Task:", ctext(label, 'cyan'))
fn_label, labels = unique_labels([i[0] for i in indices],
key_func=extract_label,
return_labels=True)
print("Labels:", ctext(labels, 'cyan'))
# ====== training and test data ====== #
train_files = [] # (name, (start, end)) ...
test_files = []
for name, (start, end) in indices:
if is_train(name):
train_files.append((name, (start, end)))
else:
test_files.append((name, (start, end)))
# name for each dataset, useful for later
print("#Train:", ctext(len(train_files), 'cyan'))
print("#Test:", ctext(len(test_files), 'cyan'))
# ====== for i-vectors ====== #
y_train = np.array([fn_label(i[0]) for i in train_files])
y_test = np.array([fn_label(i[0]) for i in test_files])
if bool(for_ivec):
return ds[feat], train_files, y_train, test_files, y_test, labels
# ====== length ====== #
length = [(end - start) for _, (start, end) in indices]
max_length = max(length)
frame_length = int(max_length * utt_length)
step_length = frame_length
print("Max length :", ctext(max_length, 'yellow'))
print("Frame length:", ctext(frame_length, 'yellow'))
print("Step length :", ctext(step_length, 'yellow'))
# ====== split dataset ====== #
# split by speaker ID
train_files, valid_files = train_valid_test_split(
x=train_files,
train=0.8,
cluster_func=None,
idfunc=lambda x: x[0].split('_')[4], # splited by speaker
inc_test=False)
print("#File train:", ctext(len(train_files), 'cyan'))
print("#File valid:", ctext(len(valid_files), 'cyan'))
print("#File test :", ctext(len(test_files), 'cyan'))
recipes = [
F.recipes.Sequencing(frame_length=frame_length,
step_length=step_length,
end='pad',
pad_mode='post',
pad_value=0),
F.recipes.Name2Label(converter_func=fn_label),
F.recipes.LabelOneHot(nb_classes=len(labels), data_idx=-1)
]
feeder_train = F.Feeder(F.IndexedData(ds[feat], indices=train_files),
ncpu=6,
batch_mode='batch')
feeder_valid = F.Feeder(F.IndexedData(ds[feat], indices=valid_files),
ncpu=4,
batch_mode='batch')
feeder_test = F.Feeder(F.IndexedData(ds[feat], indices=test_files),
ncpu=4,
batch_mode='file')
feeder_train.set_recipes(recipes)
feeder_valid.set_recipes(recipes)
feeder_test.set_recipes(recipes)
print(feeder_train)
# ====== process X_test, y_test in advance for faster evaluation ====== #
@cache_disk
def _extract_test_data(feat, label, utt_length):
prog = Progbar(target=len(feeder_test),
print_summary=True,
name="Preprocessing test set")
X_test = defaultdict(list)
for name, idx, X, y in feeder_test:
# validate everything as expected
assert fn_label(name) == np.argmax(y), name # label is right
# save to list
X_test[name].append((idx, X))
prog.add(X.shape[0])
# ====== create 1 array for data and dictionary for indices ====== #
X_test_name = []
X_test_data = []
for name, X in X_test.items():
X = np.concatenate([x[1] for x in sorted(X, key=lambda i: i[0])],
axis=0).astype('float16')
X_test_name += [name + '.%d' % i for i in range(len(X))]
X_test_data.append(X)
X_test_name = np.array(X_test_name)
X_test_data = np.concatenate(X_test_data, axis=0)
return X_test_name, X_test_data
# convert everything back to float32
X_test_name, X_test_data = _extract_test_data(feat, label, utt_length)
X_test_true = np.array([fn_label(i.split('.')[0]) for i in X_test_name])
return feeder_train, feeder_valid, \
X_test_name, X_test_true, X_test_data, labels
from odin.ml import PLDA
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.mixture import GaussianMixture
from sklearn.neural_network import BernoulliRBM
# ===========================================================================
# Const
# ===========================================================================
TRAINING_PERCENT = 0.5
POINT_SIZE = 8
NUM_DIM = 3
colors = ['r', 'b', 'g']
markers = ["o", "^", "s"]
SEED = K.get_rng().randint(0, 10e8)
# ===========================================================================
# Load dataset
# ===========================================================================
ds = F.IRIS.load()
print(ds)
nb_samples = ds['X'].shape[0]
ids = K.get_rng().permutation(nb_samples)
X = ds['X'][ids]
y = ds['y'][ids]
labels = ds['name']
print("Labels:", ctext(labels))
assert len(colors) == len(labels) and len(markers) == len(labels)
X_train = X[:int(TRAINING_PERCENT * nb_samples)]
# Prediction
# ===========================================================================
y_pred_proba, Z1_test, Z2_test, Z3_test = make_dnn_prediction(
functions=[f_pred_proba, f_z1, f_z2, f_z3], X=X_test_data, title='TEST')
print("Test Latent:", Z1_test.shape, Z2_test.shape, Z3_test.shape)
y_pred = np.argmax(y_pred_proba, axis=-1)
evaluate(y_true=X_test_true, y_pred_proba=y_pred_proba, labels=labels,
title="Test set (Deep prediction)",
path=os.path.join(EXP_DIR, 'test_deep.pdf'))
# ====== make a streamline classifier ====== #
# training PLDA
Z3_train, y_train = make_dnn_prediction(f_z3, X=train, title="TRAIN")
print("Z3_train:", Z3_train.shape, y_train.shape)
Z3_valid, y_valid = make_dnn_prediction(f_z3, X=valid, title="VALID")
print("Z3_valid:", Z3_valid.shape, y_valid.shape)
plda = PLDA(n_phi=200, random_state=K.get_rng().randint(10e8),
n_iter=12, labels=labels, verbose=0)
plda.fit(np.concatenate([Z3_train, Z3_valid], axis=0),
np.concatenate([y_train, y_valid], axis=0))
y_pred_log_proba = plda.predict_log_proba(Z3_test)
evaluate(y_true=X_test_true, y_pred_log_proba=y_pred_log_proba, labels=labels,
title="Test set (PLDA - Latent prediction)",
path=os.path.join(EXP_DIR, 'test_latent.pdf'))
# ====== visualize ====== #
visualize_latent_space(X_org=X_test_data, X_latent=Z1_test,
name=X_test_name, labels=X_test_true,
title="latent1")
visualize_latent_space(X_org=X_test_data, X_latent=Z2_test,
name=X_test_name, labels=X_test_true,
title="latent2")
V.plot_save(os.path.join(EXP_DIR, 'latent.pdf'))
BATCH_SIZE = 32
NB_EPOCH = 20
NB_SAMPLES = 8
VALID_PERCENTAGE = 0.4
# ===========================================================================
# Load dataset
# ===========================================================================
path = get_datasetpath(name='TIDIGITS_feats', override=False)
assert os.path.isdir(path), \
"Cannot find preprocessed feature at: %s, try to run 'odin/examples/features.py'" % path
ds = F.Dataset(path, read_only=True)
assert all(f in ds for f in FEAT), "Cannot find features with name: %s" % FEAT
# ====== get all the indices of single digit ====== #
indices = [(name, (s, e)) for name, (s, e) in list(ds['indices'].items())
if len(name.split('_')[-1]) == 1]
K.get_rng().shuffle(indices)
print("Found %s utterances of single digit" % ctext(len(indices), 'cyan'))
# ===========================================================================
# Load and visual the dataset
# ===========================================================================
train = []
test = []
max_length = 0
min_length = np.inf
for name, (start, end) in indices:
assert end - start > 0
if name.split('_')[0] == 'train':
train.append((name, (start, end)))
else:
test.append((name, (start, end)))
max_length = max(end - start, max_length)
def prepare_data(feat, label, utt_length=0.4, for_ivec=False):
"""
Returns (i-vector)
------------------
ds[feat]
train_files
y_train
test_files
y_test
labels
Returns (x-vector)
------------------
train : Feeder
feeder for training data for iterating over pair of (X, y)
valid : Feeder
feeder for validating data for iterating over pair of (X, y)
X_test_name : list of file names
file names are append with '.%d' for cut segment ID
X_test_true : list of integer
label of each sample
X_test_data : array
list of test data same length as X_test_name
labels : list of string
list of labels for classification task
Example
-------
(train, valid,
X_test_name, X_test_true, X_test_data,
labels) = prepare_data_dnn(feat=FEAT, label='gender')
"""
label = str(label).lower()
assert label in _support_label, "No support for label: %s" % label
assert 0 < utt_length <= 1.
# ====== load dataset ====== #
if not os.path.exists(PATH_ACOUSTIC):
raise RuntimeError("Cannot find extracted acoustic features at path: '%s',"
"run the code speech_features_extraction.py!" % PATH_ACOUSTIC)
ds = F.Dataset(PATH_ACOUSTIC, read_only=True)
assert feat in ds, "Cannot find feature with name: %s" % feat
indices = list(ds['indices'].items())
K.get_rng().shuffle(indices)
# ====== helper ====== #
def is_train(x):
return x.split('_')[0] == 'train'
def extract_label(x):
return x.split('_')[_support_label[label]]
print("Task:", ctext(label, 'cyan'))
fn_label, labels = unique_labels([i[0] for i in indices],
key_func=extract_label,
return_labels=True)
print("Labels:", ctext(labels, 'cyan'))
# ====== training and test data ====== #
train_files = [] # (name, (start, end)) ...
test_files = []
for name, (start, end) in indices:
if is_train(name):
train_files.append((name, (start, end)))
else:
test_files.append((name, (start, end)))
# name for each dataset, useful for later
print("#Train:", ctext(len(train_files), 'cyan'))
print("#Test:", ctext(len(test_files), 'cyan'))
# ====== for i-vectors ====== #
y_train = np.array([fn_label(i[0]) for i in train_files])
y_test = np.array([fn_label(i[0]) for i in test_files])
if bool(for_ivec):
return ds[feat], train_files, y_train, test_files, y_test, labels
# ====== length ====== #
length = [(end - start) for _, (start, end) in indices]
max_length = max(length)
frame_length = int(max_length * utt_length)
step_length = frame_length
print("Max length :", ctext(max_length, 'yellow'))
print("Frame length:", ctext(frame_length, 'yellow'))
print("Step length :", ctext(step_length, 'yellow'))
# ====== split dataset ====== #
# split by speaker ID
train_files, valid_files = train_valid_test_split(
x=train_files, train=0.8,
cluster_func=None,
idfunc=lambda x: x[0].split('_')[4], # splited by speaker
inc_test=False)
print("#File train:", ctext(len(train_files), 'cyan'))
print("#File valid:", ctext(len(valid_files), 'cyan'))
print("#File test :", ctext(len(test_files), 'cyan'))
recipes = [
F.recipes.Sequencing(frame_length=frame_length, step_length=step_length,
end='pad', pad_mode='post', pad_value=0),
F.recipes.Name2Label(converter_func=fn_label),
F.recipes.LabelOneHot(nb_classes=len(labels), data_idx=-1)
]
feeder_train = F.Feeder(F.IndexedData(ds[feat], indices=train_files),
ncpu=6, batch_mode='batch')
feeder_valid = F.Feeder(F.IndexedData(ds[feat], indices=valid_files),
ncpu=4, batch_mode='batch')
feeder_test = F.Feeder(F.IndexedData(ds[feat], indices=test_files),
ncpu=4, batch_mode='file')
feeder_train.set_recipes(recipes)
feeder_valid.set_recipes(recipes)
feeder_test.set_recipes(recipes)
print(feeder_train)
# ====== process X_test, y_test in advance for faster evaluation ====== #
@cache_disk
def _extract_test_data(feat, label, utt_length):
prog = Progbar(target=len(feeder_test),
print_summary=True, name="Preprocessing test set")
X_test = defaultdict(list)
for name, idx, X, y in feeder_test:
# validate everything as expected
assert fn_label(name) == np.argmax(y), name # label is right
# save to list
X_test[name].append((idx, X))
prog.add(X.shape[0])
# ====== create 1 array for data and dictionary for indices ====== #
X_test_name = []
X_test_data = []
for name, X in X_test.items():
X = np.concatenate([x[1] for x in sorted(X, key=lambda i: i[0])],
axis=0).astype('float16')
X_test_name += [name + '.%d' % i for i in range(len(X))]
X_test_data.append(X)
X_test_name = np.array(X_test_name)
X_test_data = np.concatenate(X_test_data, axis=0)
return X_test_name, X_test_data
# convert everything back to float32
X_test_name, X_test_data = _extract_test_data(feat, label, utt_length)
X_test_true = np.array([fn_label(i.split('.')[0])
for i in X_test_name])
return feeder_train, feeder_valid, \
X_test_name, X_test_true, X_test_data, labels
