def calculate_overall_quality_score(list_feature_teacher,
list_feature_student):
"""
Calculate overall quality similarity
:param list_feature_teacher:
:param list_feature_student:
:return:
"""
path_model = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/models/phone_embedding_classifier'
path_dataset = '/media/gong/ec990efa-9ee0-4693-984b-29372dcea0d1/Data/RongGong/phoneEmbedding'
config = [1, 0]
input_shape = [1, None, 80]
embedding_dim = 32
prefix = '_2_class_teacher_student'
attention_dense_str = "dense_32_"
model_name = config_select(config) + prefix
filename_model = os.path.join(
path_model, model_name + '_' + attention_dense_str + str(0) + '.h5')
filename_scaler = os.path.join(
path_dataset, 'scaler_phn_embedding_train_teacher_student.pkl')
model = load_model(
filepath=filename_model,
custom_objects={'Attention': Attention(return_attention=True)})
weights = model.get_weights()
x, input = model_select(config=config,
input_shape=input_shape,
conv=False,
dropout=False)
outputs = Dense(embedding_dim)(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
scaler = pickle.load(open(filename_scaler, 'rb'))
list_simi = []
for ii in range(len(list_feature_teacher)):
x_batch_teacher = np.expand_dims(scaler.transform(
list_feature_teacher[ii]),
axis=0)
x_batch_student = np.expand_dims(scaler.transform(
list_feature_student[ii]),
axis=0)
embedding_teacher = model_1_batch.predict_on_batch(x_batch_teacher)
embedding_student = model_1_batch.predict_on_batch(x_batch_student)
dist = (2.0 - cosine(embedding_teacher, embedding_student)) / 2.0
list_simi.append(dist)
return list_simi
def embedding_classifier_ap(filename_feature, filename_list_key,
filename_scaler):
"""calculate average precision of classifier embedding"""
list_feature_flatten_test, label_integer_test, le, scaler = \
load_data_embedding(filename_feature=filename_feature,
filename_list_key=filename_list_key,
filename_scaler=filename_scaler)
path_model = '/Users/gong/Documents/pycharmProjects/phoneticSimilarity/models/phone_embedding_classifier'
path_eval = '/Users/gong/Documents/pycharmProjects/phoneticSimilarity/eval/phone_embedding_classifier'
# configs = [[1, 0], [1, 1], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2], [3, 3]]
configs = [[1, 1]]
for config in configs:
model_name = config_select(config)
list_ap = []
embedding_dim = 29
for ii in range(5):
filename_model = os.path.join(path_model,
model_name + '_' + str(ii) + '.h5')
model = load_model(filepath=filename_model)
weights = model.get_weights()
input_shape = [1, None, 80]
model_1_batch = model_select(config=config,
input_shape=input_shape)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros(
(len(list_feature_flatten_test), embedding_dim))
for ii_emb in range(len(list_feature_flatten_test)):
print('calculate', ii, 'run time', ii_emb, 'embedding',
len(list_feature_flatten_test), 'total')
x_batch = np.expand_dims(scaler.transform(
list_feature_flatten_test[ii_emb]),
axis=0)
embeddings[ii_emb, :] = model_1_batch.predict_on_batch(x_batch)
# dist_mat = distance_matrix_embedding_classifier(embeddings)
dist_mat = (2.0 - squareform(pdist(embeddings, 'cosine'))) / 2.0
gt_mat = ground_truth_matrix(label_integer_test)
np.save(file=os.path.join(path_eval, 'dist_mat_' + str(ii)),
arr=dist_mat)
ap = eval_embeddings(dist_mat=dist_mat, gt_mat=gt_mat)
list_ap.append(ap)
filename_eval = os.path.join(path_eval, model_name + '.csv')
with open(filename_eval, 'w') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
)
csvwriter.writerow([np.mean(list_ap), np.std(list_ap)])
def embedding_classifier_ap(filename_feature_teacher,
filename_list_key_teacher,
filename_feature_student,
filename_list_key_student,
filename_scaler,
embedding_dim,
config,
val_test,
MTL=False,
attention=False,
dense=False,
conv=False,
dropout=0.25):
"""calculate teacher student pairs average precision of classifier embedding"""
list_feature_flatten_val, label_integer_val, le, scaler = \
load_data_embedding_teacher_student(filename_feature_teacher=filename_feature_teacher,
filename_list_key_teacher=filename_list_key_teacher,
filename_feature_student=filename_feature_student,
filename_list_key_student=filename_list_key_student,
filename_scaler=filename_scaler)
path_model = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/models/phone_embedding_classifier'
path_eval = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/eval/phone_embedding_classifier'
# configs = [[1, 0], [1, 1], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2], [3, 3]]
# configs = [[2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2], [3, 3]]
prefix = '_MTL' if MTL else '_2_class_teacher_student'
model_name = config_select(
config
) + prefix if embedding_dim == 2 or embedding_dim == 32 else config_select(
config)
if dense and conv:
attention_dense_str = 'dense_conv_'
elif attention:
attention_dense_str = "attention_"
elif dense:
attention_dense_str = "dense_"
elif conv:
attention_dense_str = "conv_"
elif dropout:
attention_dense_str = "dropout_"
else:
attention_dense_str = ""
list_ap = []
# average precision of each phone
array_ap_phn_5_runs = np.zeros((5, 27))
for ii in range(5):
print('run time', ii)
filename_model = os.path.join(
path_model,
model_name + '_' + attention_dense_str + str(ii) + '.h5')
if attention:
model = load_model(
filepath=filename_model,
custom_objects={'Attention': Attention(return_attention=True)})
else:
model = load_model(filepath=filename_model)
weights = model.get_weights()
input_shape = [1, None, 80]
if attention:
x, input, _ = model_select_attention(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
else:
x, input = model_select(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
if MTL:
if dense:
pronun_out = x
profess_out = Dense(32)(x)
else:
pronun_out = Dense(27,
activation='softmax',
name='pronunciation')(x)
profess_out = Dense(embedding_dim,
activation='softmax',
name='professionality')(x)
model_1_batch = Model(inputs=input,
outputs=[pronun_out, profess_out])
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
loss_weights=[0.5, 0.5])
else:
if dense:
outputs = Dense(embedding_dim)(x)
else:
outputs = Dense(embedding_dim, activation='softmax')(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros((len(list_feature_flatten_val), embedding_dim))
for ii_emb in range(len(list_feature_flatten_val)):
# print('calculate', ii, 'run time', ii_emb, 'embedding', len(list_feature_flatten_val), 'total')
x_batch = np.expand_dims(scaler.transform(
list_feature_flatten_val[ii_emb]),
axis=0)
if MTL:
_, out = model_1_batch.predict_on_batch(x_batch)
else:
out = model_1_batch.predict_on_batch(x_batch)
if attention:
embeddings[ii_emb, :] = out[0, :]
else:
embeddings[ii_emb, :] = out
# dist_mat = distance_matrix_embedding_classifier(embeddings)
list_dist = []
list_gt = []
array_ap_phn = np.zeros((27, ))
cols = []
list_ratio_tea_stu = []
for ii_class in range(27):
# teacher student pair class index
idx_ii_class = np.where(
np.logical_or(label_integer_val == 2 * ii_class,
label_integer_val == 2 * ii_class + 1))[0]
idx_ii_class_stu = len(
np.where(label_integer_val == 2 * ii_class)[0])
idx_ii_class_tea = len(
np.where(label_integer_val == 2 * ii_class + 1)[0])
# ratio of teacher's samples
list_ratio_tea_stu.append(
idx_ii_class_tea / float(idx_ii_class_tea + idx_ii_class_stu))
dist_mat = (2.0 - squareform(
pdist(embeddings[idx_ii_class], 'cosine'))) / 2.0
labels_ii_class = [label_integer_val[idx] for idx in idx_ii_class]
gt_mat = ground_truth_matrix(labels_ii_class)
sample_num = dist_mat.shape[0]
iu1 = np.triu_indices(sample_num, 1) # trim the upper mat
list_dist.append(dist_mat[iu1])
list_gt.append(gt_mat[iu1])
# calculate the average precision of each phoneme
ap_phn = average_precision_score(y_true=np.abs(list_gt[ii_class]),
y_score=np.abs(
list_dist[ii_class]),
average='weighted')
cols.append(le.inverse_transform(2 * ii_class).split('_')[0])
array_ap_phn[ii_class] = ap_phn
print(list_ratio_tea_stu)
array_dist = np.concatenate(list_dist)
array_gt = np.concatenate(list_gt)
ap = average_precision_score(y_true=np.abs(array_gt),
y_score=np.abs(array_dist),
average='weighted')
list_ap.append(ap)
array_ap_phn_5_runs[ii, :] = array_ap_phn
post_fix = prefix + '_2_class' if val_test == 'val' else prefix + '_2_class_extra_student'
filename_eval = os.path.join(
path_eval, model_name + post_fix + attention_dense_str + '.csv')
with open(filename_eval, 'w') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
)
csvwriter.writerow([np.mean(list_ap), np.std(list_ap)])
# organize the Dataframe
ap_phn_mean = np.mean(array_ap_phn_5_runs, axis=0)
ap_phn_std = np.std(array_ap_phn_5_runs, axis=0)
ap_phn_mean_std = pd.DataFrame(np.transpose(
np.vstack((ap_phn_mean, ap_phn_std, list_ratio_tea_stu))),
columns=['mean', 'std', 'ratio'],
index=cols)
ap_phn_mean_std = ap_phn_mean_std.sort_values(by='mean')
ap_phn_mean_std.to_csv(
os.path.join(
path_eval,
model_name + post_fix + attention_dense_str + '_phn_mean_std.csv'))
def embedding_classifier_helper(configs,
MTL,
path_eval,
path_model,
le,
list_feature_flatten,
scaler,
label,
dense=False,
emb_all=False):
embedding_dim = 32 if dense else 2
for config in configs:
input_shape = [1, None, 80]
prefix = '_MTL' if MTL else '_2_class_teacher_student'
model_name = config_select(config) + prefix if embedding_dim == 2 or embedding_dim == 32 else config_select(config)
dense_str = "dense_32_" if dense else ""
if emb_all and dense:
emb_all_str = "_dense_all"
elif emb_all:
emb_all_str = "_all"
else:
emb_all_str = ""
if le:
# label encoder
pickle.dump(le, open(os.path.join(path_eval, model_name + '_le.pkl'), 'wb'), protocol=2)
for ii in range(1):
filename_model = os.path.join(path_model, model_name + '_' + dense_str + str(ii) + '.h5')
model = load_model(filepath=filename_model)
weights = model.get_weights()
x, input = model_select(config=config, input_shape=input_shape, conv=False, dropout=False)
if MTL:
pronun_out = Dense(27, activation='softmax', name='pronunciation')(x)
profess_out = Dense(embedding_dim, activation='softmax', name='professionality')(x)
model_1_batch = Model(inputs=input, outputs=[pronun_out, profess_out])
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
loss_weights=[0.5, 0.5])
else:
if dense:
outputs = Dense(embedding_dim)(x)
else:
outputs = Dense(embedding_dim, activation='softmax')(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings_profess = np.zeros((len(list_feature_flatten), embedding_dim))
if MTL:
embeddings_pronun = np.zeros((len(list_feature_flatten), 27))
for ii_emb in range(len(list_feature_flatten)):
print('calculate', ii, 'run time', ii_emb, 'embedding', len(list_feature_flatten), 'total')
x_batch = np.expand_dims(scaler.transform(list_feature_flatten[ii_emb]), axis=0)
if MTL:
embeddings_pronun[ii_emb, :], embeddings_profess[ii_emb, :] = \
model_1_batch.predict_on_batch(x_batch)
else:
embeddings_profess[ii_emb, :] = model_1_batch.predict_on_batch(x_batch)
np.save(file=os.path.join(path_eval, model_name + '_embedding_professionality' + dense_str + emb_all_str + str(ii)),
arr=embeddings_profess)
np.save(file=os.path.join(path_eval, model_name + '_embeddings_labels' + dense_str + emb_all_str), arr=label)
if MTL:
np.save(file=os.path.join(path_eval, model_name + '_embedding_pronunciation' + dense_str + emb_all_str + str(ii)),
arr=embeddings_pronun)
def embedding_classifier_helper(configs,
path_eval,
path_model,
le,
list_feature_flatten,
scaler,
label,
dense=False,
emb_all=False):
"""
Output the classification model embedding for overall quality
:param configs: model architecture
:param path_eval: embedding output path
:param path_model: embedding model path
:param le: label encoder
:param list_feature_flatten: log-mel feature list
:param scaler: feature scaler
:param label:
:param dense: bool, use 32 embedding or not
:param emb_all: bool, use all the dataset, including professional, amateur train, validation and extra test
:return:
"""
embedding_dim = 32 if dense else 2
for config in configs:
input_shape = [1, None, 80]
prefix = '_2_class_teacher_student'
dense_str = "dense_32_" if dense else ""
model_name = config_select(config) + prefix
if emb_all and dense:
emb_all_str = "_dense_all"
elif emb_all:
emb_all_str = "_all"
else:
emb_all_str = ""
if le:
# label encoder
pickle.dump(le, open(os.path.join(path_eval, model_name + '_le.pkl'), 'wb'), protocol=2)
ii = 0 # only use the first model
filename_model = os.path.join(path_model, model_name + '_' + dense_str + str(ii) + '.h5')
model = load_model(filepath=filename_model)
weights = model.get_weights()
x, input = model_select(config=config, input_shape=input_shape, conv=False, dropout=False)
if dense:
outputs = Dense(embedding_dim)(x)
else:
outputs = Dense(embedding_dim, activation='softmax')(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_1_batch.set_weights(weights=weights)
embeddings_profess = np.zeros((len(list_feature_flatten), embedding_dim))
for ii_emb in range(len(list_feature_flatten)):
print('calculate', ii, 'run time', ii_emb, 'embedding', len(list_feature_flatten), 'total')
x_batch = np.expand_dims(scaler.transform(list_feature_flatten[ii_emb]), axis=0)
embeddings_profess[ii_emb, :] = model_1_batch.predict_on_batch(x_batch)
np.save(file=os.path.join(path_eval,
model_name + '_embedding_overall_quality' + dense_str + emb_all_str + str(ii)),
arr=embeddings_profess)
np.save(file=os.path.join(path_eval,
model_name + '_embeddings_labels' + dense_str + emb_all_str), arr=label)
def embedding_classifier_ap(filename_feature_teacher,
filename_list_key_teacher,
filename_feature_student,
filename_list_key_student,
filename_scaler,
config,
val_test,
MTL=False,
attention=False,
dense=False,
conv=False,
dropout=0.25):
"""calculate average precision of classifier embedding"""
list_feature_flatten_val, label_integer_val, le, scaler = \
load_data_embedding_teacher_student(filename_feature_teacher=filename_feature_teacher,
filename_list_key_teacher=filename_list_key_teacher,
filename_feature_student=filename_feature_student,
filename_list_key_student=filename_list_key_student,
filename_scaler=filename_scaler)
labels = le.inverse_transform(label_integer_val)
# combine teacher and student label to the same one
phn_set = list(set([l.split('_')[0] for l in labels]))
for ii in range(len(phn_set)):
indices_phn = [
i for i, s in enumerate(labels) if phn_set[ii] == s.split('_')[0]
]
label_integer_val[indices_phn] = ii
index_teacher = [
ii for ii in range(len(label_integer_val)) if 'teacher' in labels[ii]
]
index_student = [
ii for ii in range(len(label_integer_val)) if 'student' in labels[ii]
]
print('index_teacher min max', min(index_teacher), max(index_teacher))
print('index_student min max', min(index_student), max(index_student))
path_model = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/models/phone_embedding_classifier'
path_eval = '/home/gong/Documents/pycharmProjects/phoneticSimilarity/eval/phone_embedding_classifier'
# configs = [[1, 0], [1, 1], [2, 0], [2, 1], [2, 2], [3, 0], [3, 1], [3, 2], [3, 3]]
# configs = [[1, 1]]
# for config in configs:
model_name = config_select(config)
list_ap = []
embedding_dim = 27
prefix = '_MTL' if MTL else '_27_class'
if attention and conv and dropout:
attention_dense_str = 'attention_conv_dropout_'
elif attention:
attention_dense_str = "attention_"
elif dense:
attention_dense_str = "dense_"
elif conv:
attention_dense_str = "conv_"
elif dropout:
attention_dense_str = "dropout_"
else:
attention_dense_str = ""
for ii in range(5):
filename_model = os.path.join(
path_model,
model_name + prefix + '_' + attention_dense_str + str(ii) + '.h5')
if attention:
model = load_model(
filepath=filename_model,
custom_objects={'Attention': Attention(return_attention=True)})
else:
model = load_model(filepath=filename_model)
weights = model.get_weights()
input_shape = [1, None, 80]
if attention:
x, input, _ = model_select_attention(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
else:
x, input = model_select(config=config,
input_shape=input_shape,
conv=conv,
dropout=dropout)
if MTL:
pronun_out = Dense(embedding_dim,
activation='softmax',
name='pronunciation')(x)
if dense:
x = Dense(32)(x)
profess_out = Dense(2,
activation='softmax',
name='professionality')(x)
model_1_batch = Model(inputs=input,
outputs=[pronun_out, profess_out])
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
loss_weights=[0.5, 0.5])
else:
if dense:
outputs = Dense(units=32)(x)
else:
outputs = Dense(embedding_dim, activation='softmax')(x)
model_1_batch = Model(inputs=input, outputs=outputs)
model_1_batch.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model_1_batch.summary())
model_1_batch.set_weights(weights=weights)
embeddings = np.zeros((len(list_feature_flatten_val), embedding_dim))
for ii_emb in range(len(list_feature_flatten_val)):
print('calculate', ii, 'run time', ii_emb, 'embedding',
len(list_feature_flatten_val), 'total')
x_batch = np.expand_dims(scaler.transform(
list_feature_flatten_val[ii_emb]),
axis=0)
if MTL:
out, _ = model_1_batch.predict_on_batch(x_batch)
else:
out = model_1_batch.predict_on_batch(x_batch)
if attention:
embeddings[ii_emb, :] = out[0, :]
else:
embeddings[ii_emb, :] = out
# dist_mat = distance_matrix_embedding_classifier(embeddings)
dist_mat = (2.0 - squareform(pdist(embeddings, 'cosine'))) / 2.0
gt_mat = ground_truth_matrix(label_integer_val)
# we only compare teacher to student embeddings
dist_mat = dist_mat[:min(index_student), min(index_student):]
gt_mat = gt_mat[:min(index_student), min(index_student):]
np.save(file=os.path.join(path_eval,
'dist_mat_' + 'teacher_student_' + str(ii)),
arr=dist_mat)
ap = eval_embeddings_no_trim(dist_mat=dist_mat, gt_mat=gt_mat)
list_ap.append(ap)
post_fix = prefix + '_27_class' if val_test == 'val' else prefix + '_27_class_extra_student'
filename_eval = os.path.join(
path_eval, model_name + post_fix + attention_dense_str + '.csv')
with open(filename_eval, 'w') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
)
csvwriter.writerow([np.mean(list_ap), np.std(list_ap)])