def embedding_classifier_tsne(filename_feature, filename_list_key, filename_scaler):
"""calculate average precision of classifier embedding"""
list_feature_flatten_test, label_integer_test, le, scaler = \
load_data_embedding(filename_feature=filename_feature,
filename_list_key=filename_list_key,
filename_scaler=filename_scaler)
path_model = '/Users/gong/Documents/pycharmProjects/phoneticSimilarity/models/phone_embedding_classifier'
path_eval = '/Users/gong/Documents/pycharmProjects/phoneticSimilarity/eval/phone_embedding_classifier'
# configs = [[1, 0], [1, 1], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2], [3, 3]]
configs = [[1, 1]]
for config in configs:
model_name = config_select(config)
pickle.dump(le, open(os.path.join(path_eval, model_name + '_le.pkl'), 'wb'), protocol=2)
embedding_dim = 29
for ii in range(1):
filename_model = os.path.join(path_model, model_name + '_' + str(ii) + '.h5')
model = load_model(filepath=filename_model)
weights = model.get_weights()
input_shape = [1, None, 80]
model_1_batch = model_select(config=config, input_shape=input_shape)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros((len(list_feature_flatten_test), embedding_dim))
for ii_emb in range(len(list_feature_flatten_test)):
print('calculate', ii, 'run time', ii_emb, 'embedding', len(list_feature_flatten_test), 'total')
x_batch = np.expand_dims(scaler.transform(list_feature_flatten_test[ii_emb]), axis=0)
embeddings[ii_emb, :] = model_1_batch.predict_on_batch(x_batch)
np.save(file=os.path.join(path_eval, model_name + '_embedding_' + str(ii)), arr=embeddings)
np.save(file=os.path.join(path_eval, model_name + '_labels_'), arr=label_integer_test)
# # txt.set_path_effects([
# # PathEffects.Stroke(linewidth=5, foreground="w"),
# # PathEffects.Normal()])
# txts.append(txt)
plt.show()
if __name__ == '__main__':
MTL = True
config = [2, 0]
embedding_dim = 2
path_eval = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/eval/phone_embedding_classifier'
prefix = '_MTL' if MTL else '_2_class_teacher_student'
model_name = config_select(
config) + prefix if embedding_dim == 2 else config_select(config)
le = pickle.load(
open(os.path.join(path_eval, model_name + '_le.pkl'), 'rb'))
embedding_profess = np.load(
os.path.join(path_eval,
model_name + '_embedding_professionality0.npy'))
embedding_pronun = np.load(
os.path.join(path_eval, model_name + '_embedding_pronunciation0.npy'))
labels = np.load(
os.path.join(path_eval, model_name + '_embeddings_labels.npy'))
plot_tsne_pronun(embedding_pronun, labels, le)
# plot_tsne_profess(embedding_profess, labels, le)
# index_feature = range(len(list_feature_flatten))
# train_index, val_index, _, _ = train_test_split(index_feature, labels_integer, test_size=0.1, stratify=labels_integer)
#
# pickle.dump(le ,open(filename_label_encoder, 'wb'), protocol=2)
# pickle.dump([train_index, val_index], open(filename_data_splits, 'wb'), protocol=2)
train_index, val_index = pickle.load(open(filename_data_splits, 'rb'))
#
# for train_index, val_index in folds5_split_indices:
configs = [[1, 1], [1, 0], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2], [3, 3]]
for config in configs:
model_name = config_select(config=config)
if output_shape == 2:
model_name += '_2_class'
for ii in range(5):
file_path_model = os.path.join(path_model, model_name + '_teacher_student' + '_' + str(ii) + '.h5')
file_path_log = os.path.join(path_model, 'log', model_name + '_teacher_student' + '_' + str(ii) + '.csv')
list_feature_fold_train = [scaler.transform(list_feature_flatten[ii]) for ii in train_index]
labels_integer_fold_train = labels_integer[train_index]
labels_fold_train = to_categorical(labels_integer_fold_train)
list_feature_fold_val = [scaler.transform(list_feature_flatten[ii]) for ii in val_index]
labels_integer_fold_val = labels_integer[val_index]
labels_fold_val = to_categorical(labels_integer_fold_val)
def embedding_classifier_ap(filename_feature_teacher,
filename_list_key_teacher,
filename_feature_student,
filename_list_key_student, filename_scaler):
"""calculate average precision of classifier embedding"""
list_feature_flatten_val, label_integer_val, le, scaler = \
load_data_embedding_teacher_student(filename_feature_teacher=filename_feature_teacher,
filename_list_key_teacher=filename_list_key_teacher,
filename_feature_student=filename_feature_student,
filename_list_key_student=filename_list_key_student,
filename_scaler=filename_scaler)
path_model = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/models/phone_embedding_classifier'
path_eval = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/eval/phone_embedding_classifier'
configs = [[1, 0], [1, 1], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2],
[3, 3]]
# configs = [[1, 1]]
for config in configs:
model_name = config_select(config)
list_ap = []
embedding_dim = 54
for ii in range(5):
filename_model = os.path.join(
path_model,
model_name + '_teacher_student' + '_' + str(ii) + '.h5')
model = load_model(filepath=filename_model)
weights = model.get_weights()
input_shape = [1, None, 80]
model_1_batch = model_select(config=config,
input_shape=input_shape,
output_shape=embedding_dim)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros(
(len(list_feature_flatten_val), embedding_dim))
for ii_emb in range(len(list_feature_flatten_val)):
print('calculate', ii, 'run time', ii_emb, 'embedding',
len(list_feature_flatten_val), 'total')
x_batch = np.expand_dims(scaler.transform(
list_feature_flatten_val[ii_emb]),
axis=0)
embeddings[ii_emb, :] = model_1_batch.predict_on_batch(x_batch)
# dist_mat = distance_matrix_embedding_classifier(embeddings)
dist_mat = (2.0 - squareform(pdist(embeddings, 'cosine'))) / 2.0
gt_mat = ground_truth_matrix(label_integer_val)
np.save(file=os.path.join(
path_eval, 'dist_mat_' + 'teacher_student_' + str(ii)),
arr=dist_mat)
ap = eval_embeddings(dist_mat=dist_mat, gt_mat=gt_mat)
list_ap.append(ap)
filename_eval = os.path.join(path_eval,
model_name + '_teacher_student' + '.csv')
with open(filename_eval, 'w') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
)
csvwriter.writerow([np.mean(list_ap), np.std(list_ap)])
def run_process(path_dataset, path_model, exp):
batch_size = 64
input_shape = (batch_size, None, 80)
patience = 15
output_shape = 27
attention = False
conv = False
dropout = False
dense = False
if exp == "baseline":
pass
elif exp == "attention":
attention = True
elif exp == "32_embedding":
dense = True
elif exp == "cnn":
conv = True
elif exp == "dense":
dense = True
elif exp == "dropout":
dropout = 0.25
elif exp == "best_combination":
attention = True
conv = True
dropout = 0.25
else:
raise ValueError("exp {} is not a valid parameter".format(exp))
if attention and conv and dropout:
attention_dense_str = "attention_conv_dropout_"
elif attention:
attention_dense_str = "attention_"
elif dense:
attention_dense_str = "dense_"
elif conv:
attention_dense_str = "conv_"
elif dropout:
attention_dense_str = "dropout_"
else:
attention_dense_str = ""
# path_dataset = '/media/gong/ec990efa-9ee0-4693-984b-29372dcea0d1/Data/RongGong/phoneEmbedding'
filename_feature_teacher = os.path.join(
path_dataset, 'feature_phn_embedding_train_teacher.pkl')
filename_list_key_teacher = os.path.join(path_dataset,
'list_key_teacher.pkl')
filename_feature_student = os.path.join(
path_dataset, 'feature_phn_embedding_train_student.pkl')
filename_list_key_student = os.path.join(path_dataset,
'list_key_student.pkl')
filename_scaler_teacher_student = os.path.join(
path_dataset, 'scaler_phn_embedding_train_teacher_student.pkl')
filename_label_encoder = os.path.join(
path_dataset, 'le_phn_embedding_teacher_student.pkl')
filename_data_splits = os.path.join(path_dataset,
'data_splits_teacher_student.pkl')
# path_model = '../../temp'
list_feature_flatten, labels_integer, le, scaler = \
load_data_embedding_teacher_student(filename_feature_teacher=filename_feature_teacher,
filename_list_key_teacher=filename_list_key_teacher,
filename_feature_student=filename_feature_student,
filename_list_key_student=filename_list_key_student,
filename_scaler=filename_scaler_teacher_student)
# combine teacher and student label to the same one
labels = le.inverse_transform(labels_integer)
phn_set = list(set([l.split('_')[0] for l in labels]))
for ii in range(len(phn_set)):
indices_phn = [
i for i, s in enumerate(labels) if phn_set[ii] == s.split('_')[0]
]
labels_integer[indices_phn] = ii
train_index, val_index = pickle.load(open(filename_data_splits, 'rb'))
# for train_index, val_index in folds5_split_indices:
if attention or dense or conv or dropout:
configs = [[2, 0]]
else:
configs = [[1, 0], [1, 1], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1],
[3, 2], [3, 3]]
for config in configs:
model_name = config_select(config=config)
for ii in range(5):
file_path_model = os.path.join(
path_model, model_name + '_27_class' + '_' +
attention_dense_str + str(ii) + '.h5')
file_path_log = os.path.join(
path_model, 'log', model_name + '_27_class' + '_' +
attention_dense_str + str(ii) + '.csv')
list_feature_fold_train = [
scaler.transform(list_feature_flatten[ii])
for ii in train_index
]
labels_integer_fold_train = labels_integer[train_index]
labels_fold_train = to_categorical(labels_integer_fold_train)
list_feature_fold_val = [
scaler.transform(list_feature_flatten[ii]) for ii in val_index
]
labels_integer_fold_val = labels_integer[val_index]
labels_fold_val = to_categorical(labels_integer_fold_val)
train_embedding_RNN_batch(
list_feature_fold_train=list_feature_fold_train,
labels_fold_train=labels_fold_train,
list_feature_fold_val=list_feature_fold_val,
labels_fold_val=labels_fold_val,
batch_size=batch_size,
input_shape=input_shape,
output_shape=output_shape,
file_path_model=file_path_model,
filename_log=file_path_log,
patience=patience,
config=config,
attention=attention,
dense=dense,
conv=conv,
dropout=dropout)
def embedding_classifier_ap(filename_feature_teacher,
filename_list_key_teacher,
filename_feature_student,
filename_list_key_student,
filename_scaler,
config,
val_test,
attention=False,
dense=False,
conv=False,
dropout=False,
path_eval="./eval/phone_embedding_classifier",
path_model='./models/phone_embedding_classifier'):
"""calculate average precision of classification embedding"""
list_feature_flatten_val, label_integer_val, le, scaler = \
load_data_embedding_teacher_student(filename_feature_teacher=filename_feature_teacher,
filename_list_key_teacher=filename_list_key_teacher,
filename_feature_student=filename_feature_student,
filename_list_key_student=filename_list_key_student,
filename_scaler=filename_scaler)
labels = le.inverse_transform(label_integer_val)
index_teacher, index_student = get_index_teacher_student(
labels=labels, label_integer_val=label_integer_val)
# path_model =
# path_eval = './eval/phone_embedding_classifier'
# for config in configs:
model_name = config_select(config)
list_ap = []
embedding_dim = 27
input_shape = [1, None, 80]
prefix = '_27_class'
if attention and conv and dropout:
attention_dense_str = 'attention_conv_dropout_'
elif attention:
attention_dense_str = "attention_"
elif dense:
attention_dense_str = "dense_"
elif conv:
attention_dense_str = "conv_"
elif dropout:
attention_dense_str = "dropout_"
else:
attention_dense_str = ""
# 5 running times
for ii in range(5):
# embedding model filename
filename_model = os.path.join(
path_model,
model_name + prefix + '_' + attention_dense_str + str(ii) + '.h5')
# load model weights, create a new model with batch size 1, then set weights back
if attention:
model = load_model(
filepath=filename_model,
custom_objects={'Attention': Attention(return_attention=True)})
x, input, _ = model_select_attention(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
else:
model = load_model(filepath=filename_model)
x, input = model_select(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
weights = model.get_weights()
if dense:
outputs = Dense(units=32)(x)
else:
outputs = Dense(embedding_dim, activation='softmax')(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model_1_batch.summary())
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros((len(list_feature_flatten_val), embedding_dim))
for ii_emb in range(len(list_feature_flatten_val)):
print('calculate', ii, 'run time', ii_emb, 'embedding',
len(list_feature_flatten_val), 'total')
x_batch = np.expand_dims(scaler.transform(
list_feature_flatten_val[ii_emb]),
axis=0)
# evaluate the feature to get the embedding
out = model_1_batch.predict_on_batch(x_batch)
if attention:
embeddings[ii_emb, :] = out[0, :]
else:
embeddings[ii_emb, :] = out
ap = calculate_ap(embeddings=embeddings,
label_integer_val=label_integer_val,
index_student=index_student)
list_ap.append(ap)
post_fix = prefix if val_test == 'val' else prefix + '_extra_student'
# write results to .csv
filename_eval = os.path.join(
path_eval, model_name + post_fix + '_' + attention_dense_str + '.csv')
with open(filename_eval, 'w') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
)
csvwriter.writerow([np.mean(list_ap), np.std(list_ap)])
def embedding_classifier_ap(filename_feature_teacher,
filename_list_key_teacher,
filename_feature_student,
filename_list_key_student,
filename_scaler,
config,
val_test,
attention=False,
dense=False,
conv=False,
dropout=False,
path_eval='./eval/phone_embedding_classifier',
path_model='./models/phone_embedding_classifier'):
"""calculate teacher student pairs average precision of classifier embedding"""
list_feature_flatten_val, label_integer_val, le, scaler = \
load_data_embedding_teacher_student(filename_feature_teacher=filename_feature_teacher,
filename_list_key_teacher=filename_list_key_teacher,
filename_feature_student=filename_feature_student,
filename_list_key_student=filename_list_key_student,
filename_scaler=filename_scaler)
# path_model = './models/phone_embedding_classifier'
# path_eval = './eval/phone_embedding_classifier'
prefix = '_2_class_teacher_student'
if dense:
embedding_dim = 32
else:
embedding_dim = 2
model_name = config_select(config) + prefix
if dense and conv:
attention_dense_str = 'dense_conv_'
elif attention:
attention_dense_str = "attention_"
elif dense:
attention_dense_str = "dense_"
elif conv:
attention_dense_str = "conv_"
elif dropout:
attention_dense_str = "dropout_"
else:
attention_dense_str = ""
list_ap = []
input_shape = [1, None, 80]
# average precision of each phone
array_ap_phn_5_runs = np.zeros((5, 27))
for ii in range(5):
print('run time', ii)
filename_model = os.path.join(
path_model,
model_name + '_' + attention_dense_str + str(ii) + '.h5')
if attention:
model = load_model(
filepath=filename_model,
custom_objects={'Attention': Attention(return_attention=True)})
x, input, _ = model_select_attention(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
else:
model = load_model(filepath=filename_model)
x, input = model_select(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
weights = model.get_weights()
if dense:
outputs = Dense(embedding_dim)(x)
else:
outputs = Dense(embedding_dim, activation='softmax')(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros((len(list_feature_flatten_val), embedding_dim))
for ii_emb in range(len(list_feature_flatten_val)):
x_batch = np.expand_dims(scaler.transform(
list_feature_flatten_val[ii_emb]),
axis=0)
out = model_1_batch.predict_on_batch(x_batch)
if attention:
embeddings[ii_emb, :] = out[0, :]
else:
embeddings[ii_emb, :] = out
list_dist = []
list_gt = []
array_ap_phn = np.zeros((27, ))
cols = []
list_ratio_tea_stu = []
# calculate the AP for each phone
for ii_class in range(27):
# teacher student pair class index
idx_ii_class = np.where(
np.logical_or(label_integer_val == 2 * ii_class,
label_integer_val == 2 * ii_class + 1))[0]
idx_ii_class_stu = len(
np.where(label_integer_val == 2 * ii_class)[0])
idx_ii_class_tea = len(
np.where(label_integer_val == 2 * ii_class + 1)[0])
# ratio of teacher's samples
list_ratio_tea_stu.append(
idx_ii_class_tea / float(idx_ii_class_tea + idx_ii_class_stu))
dist_mat = (2.0 - squareform(
pdist(embeddings[idx_ii_class], 'cosine'))) / 2.0
labels_ii_class = [label_integer_val[idx] for idx in idx_ii_class]
gt_mat = ground_truth_matrix(labels_ii_class)
sample_num = dist_mat.shape[0]
iu1 = np.triu_indices(sample_num, 1) # trim the upper mat
list_dist.append(dist_mat[iu1])
list_gt.append(gt_mat[iu1])
# calculate the average precision of each phoneme
ap_phn = average_precision_score(y_true=np.abs(list_gt[ii_class]),
y_score=np.abs(
list_dist[ii_class]),
average='weighted')
cols.append(le.inverse_transform(2 * ii_class).split('_')[0])
array_ap_phn[ii_class] = ap_phn
array_dist = np.concatenate(list_dist)
array_gt = np.concatenate(list_gt)
ap = average_precision_score(y_true=np.abs(array_gt),
y_score=np.abs(array_dist),
average='weighted')
list_ap.append(ap)
array_ap_phn_5_runs[ii, :] = array_ap_phn
# save results to .csv
post_fix = prefix + '_2_class' if val_test == 'val' else prefix + '_2_class_extra_student'
filename_eval = os.path.join(
path_eval, model_name + post_fix + attention_dense_str + '.csv')
with open(filename_eval, 'w') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
)
csvwriter.writerow([np.mean(list_ap), np.std(list_ap)])
# organize the Dataframe, and save the individual phone results
ap_phn_mean = np.mean(array_ap_phn_5_runs, axis=0)
ap_phn_std = np.std(array_ap_phn_5_runs, axis=0)
ap_phn_mean_std = pd.DataFrame(np.transpose(
np.vstack((ap_phn_mean, ap_phn_std, list_ratio_tea_stu))),
columns=['mean', 'std', 'ratio'],
index=cols)
ap_phn_mean_std = ap_phn_mean_std.sort_values(by='mean')
ap_phn_mean_std.to_csv(
os.path.join(
path_eval,
model_name + post_fix + attention_dense_str + '_phn_mean_std.csv'))