def get_embeddings(self):
X_train, y_train, speaker_train_names = load(
get_speaker_pickle(self.get_validation_data_name() +
'_train_mfcc'))
X_test, y_test, speaker_test_names = load(
get_speaker_pickle(self.get_validation_data_name() + '_test_mfcc'))
model = load(get_experiment_nets(self.name))
set_of_embeddings = []
set_of_speakers = []
set_of_num_embeddings = []
train_outputs = self.generate_outputs(X_train, model)
test_outputs = self.generate_outputs(X_test, model)
set_of_times = [np.zeros((len(y_test) + len(y_train)), dtype=int)]
outputs, y_list, s_list = create_data_lists(False, train_outputs,
test_outputs, y_train,
y_test)
embeddings, speakers, number_embeddings = generate_embeddings(
outputs, y_list, len(model))
set_of_embeddings.append(embeddings)
set_of_speakers.append(speakers)
set_of_num_embeddings.append(number_embeddings)
checkpoints = [self.network_file]
return checkpoints, set_of_embeddings, set_of_speakers, set_of_num_embeddings, set_of_times
def plot_files(plot_file_name, files):
"""
Plots the results stored in the files given and stores them in a file with the given name
:param plot_file_name: the file name stored in common/data/results
:param files: a set of full file paths that hold result data
"""
logger = get_logger('analysis', logging.INFO)
curve_names, set_of_mrs, set_of_homogeneity_scores, \
set_of_completeness_scores, set_of_number_of_embeddings = read_result_pickle(files)
for i in range(len(set_of_mrs)):
print (len(set_of_mrs[i]))
print("Plot File Name")
print(plot_file_name)
loaded_dict = load(get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" + str(40) + "1"))
loaded_dict2 = load(get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" + str(40) + "2"))
loaded_dict3 = load(get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" + str(40) + "3"))
plot_curves(plot_file_name, curve_names, set_of_mrs, set_of_homogeneity_scores, set_of_completeness_scores,
set_of_number_of_embeddings,loaded_dict,loaded_dict2,loaded_dict3)
def _read_result_pickle(files):
"""
Reads the results of a network from these files.
:param files: can be 1-n files that contain a result.
:return: curve names, thresholds, metric scores as a list and number of embeddings
"""
logger = get_logger('analysis', logging.INFO)
logger.info('Read result pickle')
curve_names = []
# Initialize result sets
metric_sets_all_files = [[] for _ in metric_names]
set_of_number_of_embeddings = []
# Fill result sets
for file in files:
curve_name, metric_sets, number_of_embeddings = load(file)
for index, curve_name in enumerate(curve_name):
for m, metric_set in enumerate(metric_sets):
metric_sets_all_files[m].append(metric_set[index])
set_of_number_of_embeddings.append(number_of_embeddings[index])
curve_names.append(curve_name)
return curve_names, metric_sets_all_files, set_of_number_of_embeddings
def _read_result_pickle(files):
"""
Reads the results of a network from these files.
:param files: can be 1-n files that contain a result.
:return: curve names, thresholds, metric scores as a list and number of embeddings
"""
logger = get_logger('analysis', logging.INFO)
logger.info("Read result pickle")
# Initialize result sets
curve_names_all_files = []
number_of_embeddings_all_files = []
metric_sets_all_files = [[] for _ in metric_names]
for file in files:
#Load results from file
curve_names, metric_sets, number_of_embeddings = load(file)
#Add results from file to result sets
curve_names_all_files.extend(curve_names)
number_of_embeddings_all_files.extend(number_of_embeddings)
for m, metric_set in enumerate(metric_sets):
metric_sets_all_files[m].extend(metric_set)
return curve_names_all_files, metric_sets_all_files, number_of_embeddings_all_files
def create_and_train(num_epochs=1000, batch_size=100, epoch_batches=10, network_params_file_in=None,
network_params_file_out=None,
train_file=None,
network_fun=nf.create_network_10_speakers, with_validation=True):
# load training data
with open(train_file, 'rb') as f:
(X, y, speaker_names) = pickle.load(f)
# create symbolic theano variables
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
margin = T.scalar('margin')
# create network
network = network_fun(input_var)
if network_params_file_in is not None:
all_param_values = pickler.load(network_params_file_in)
lasagne.layers.set_all_param_values(network, all_param_values)
train_fn, val_fn = create_loss_functions_kl_div(input_var, network, target_var, margin)
# start training
if not with_validation:
val_fn = None
# Train network
train(X, y, num_epochs, train_fn, val_fn, SpectTrainBatchIterator(batch_size, epoch_batches, config),
SpectValidBatchIterator(batch_size, epoch_batches, config))
# Save if
if network_params_file_out is not None:
pickler.save(lasagne.layers.get_all_param_values(network), network_params_file_out)
def load_and_prepare_data(data_path, segment_size=50):
# Load and generate test data
x, y, s_list = load(data_path)
x, speakers = generate_test_data(x, y, segment_size)
# Reshape test data because it is an lstm
return x.reshape(x.shape[0], x.shape[3], x.shape[2]), speakers
def create_embeddings(self, X_train, y_train, X_test, y_test):
short_utterance = self.config.getboolean('validation',
'short_utterances')
x_list, y_list, _ = create_data_lists(short_utterance, X_train, X_test,
y_train, y_test)
x_cluster_list = []
y_cluster_list = []
for x_data, y_data in zip(x_list, y_list):
x_cluster, y_cluster = self._generate_cluster_data(x_data, y_data)
x_cluster_list.append(x_cluster)
y_cluster_list.append(y_cluster)
# Load the network and add Batchiterator
net = load(self.net_path)
net.batch_iterator_test = BatchIterator(batch_size=128)
# Predict the output
# predict = prepare_predict(net)
# output_train = predict(x_train_cluster)
# output_test = predict(x_test_cluster)
outputs = [None] * len(x_cluster_list)
for i, x_cluster in enumerate(x_cluster_list):
outputs[i] = net.predict_proba(x_cluster)
embeddings, speakers, number_embeddings =\
generate_embeddings(outputs, y_cluster_list, outputs[0].shape[1])
#Calculate the time per utterance
time = TimeCalculator.calc_time_all_utterances(
y_cluster_list, self.config.getint('luvo', 'seg_size'))
return embeddings, speakers, number_embeddings, time
def load_and_prepare_data(data_path, segment_size=50, cluster_count=None):
# Load and generate test data
x, y, s_list = load(data_path)
print(
"stored all speaker utter count ------------------------>>>>>>>>>>>>>>>>>>>>>>"
)
print(len(x))
print(
"stored all speaker index count ------------------------>>>>>>>>>>>>>>>>>>>>>>"
)
print(len(y))
if cluster_count is not None:
x = x[0:cluster_count]
y = y[0:cluster_count]
s_list = s_list[0:cluster_count]
# print(" =============> ", str(len(x)))
# print(" =============> ", len(y))
#print(" =============> ", s_list)
x, speakers = generate_test_data(x, y, segment_size)
# print("------------------>>>>>>>>>>> final shape before reshape\n")
# print(x.shape)
# Reshape test data because it is an lstm
return x.reshape(x.shape[0], x.shape[3], x.shape[2]), speakers
def analyse_results(network_name, checkpoint_names, set_of_predicted_clusters,
set_of_true_clusters, embedding_numbers, set_of_times,
set_of_utterance_embeddings):
"""
Analyses each checkpoint with the values of set_of_predicted_clusters and set_of_true_clusters.
After the analysis the result are stored in the Pickle network_name.pickle and the best Result
according to min MR is stored in network_name_best.pickle.
:param network_name: The name for the result pickle.
:param checkpoint_names: A list of names from the checkpoints. Later used as curvenames,
:param set_of_predicted_clusters: A 2D array of the predicted Clusters from the Network. [checkpoint, clusters]
:param set_of_true_clusters: A 2d array of the validation clusters. [checkpoint, validation-clusters]
:param embedding_numbers: A list which represent the number of embeddings in each checkpoint.
:param set_of_times: A 2d array of the time per utterance [checkpoint, times]
"""
logger = get_logger('analysis', logging.INFO)
logger.info('Run analysis')
metric_sets = [[None] * len(set_of_predicted_clusters)
for _ in range(len(metric_names))]
for index, predicted_clusters in enumerate(set_of_predicted_clusters):
checkpoint = checkpoint_names[index]
logger.info('Analysing checkpoint:' + checkpoint)
# Check if checkpoint is already stored
analysis_pickle = get_results_intermediate_analysis(checkpoint)
if os.path.isfile(analysis_pickle):
(metric_results, eer_result) = load(analysis_pickle)
else:
metric_results = _calculate_analysis_values(
predicted_clusters, set_of_true_clusters[index],
set_of_times[index])
eer_result = _calculate_eer_result(
set_of_utterance_embeddings[index])
save((metric_results, eer_result), analysis_pickle)
logger.info("\tEER: {}".format(round(eer_result, 5)))
for m, metric_result in enumerate(metric_results):
metric_sets[m][index] = metric_result
_write_result_pickle(network_name, checkpoint_names, metric_sets,
embedding_numbers)
_save_best_results(network_name, checkpoint_names, metric_sets,
embedding_numbers)
logger.info('Clearing intermediate result checkpoints')
for checkpoint in checkpoint_names:
analysis_pickle = get_results_intermediate_analysis(checkpoint)
test_pickle = get_results_intermediate_test(checkpoint)
if os.path.exists(analysis_pickle):
os.remove(analysis_pickle)
if os.path.exists(test_pickle):
os.remove(test_pickle)
logger.info('Analysis done')
def plot_files2(plot_file_name, files):
"""
Plots the results stored in the files given and stores them in a file with the given name
:param plot_file_name: the file name stored in common/data/results
:param files: a set of full file paths that hold result data
"""
print("Plot File Name")
print(plot_file_name)
loaded_dict = load(
get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" +
str(630) + "1"))
loaded_dict2 = load(
get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" +
str(630) + "2"))
loaded_dict3 = load(
get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" +
str(630) + "3"))
plot_curves(plot_file_name, loaded_dict, loaded_dict2, loaded_dict3)
def train_network(self):
mixture_count = self.config.getint('gmm', 'mixturecount')
X, y, speaker_names = load(
get_speaker_pickle(self.config.get('train', 'pickle') + '_mfcc'))
model = []
for i in range(len(X)):
features = X[i]
gmm = mixture.GaussianMixture(n_components=mixture_count,
covariance_type='diag',
n_init=1)
gmm.fit(features.transpose())
speaker = {'mfccs': features, 'gmm': gmm}
model.append(speaker)
save(model, get_experiment_nets(self.name))
def create_embeddings(self, train_data, test_data):
x_train_cluster, y_train_cluster = load_and_prepare_data(train_data)
x_test_cluster, y_test_cluster = load_and_prepare_data(test_data)
# Load the network and add Batchiterator
net = load(self.net_path)
net.batch_iterator_test = BatchIterator(batch_size=128)
# Predict the output
# predict = prepare_predict(net)
# output_train = predict(x_train_cluster)
# output_test = predict(x_test_cluster)
output_train = net.predict_proba(x_train_cluster)
output_test = net.predict_proba(x_test_cluster)
return generate_embeddings(output_train, output_test, y_train_cluster, y_test_cluster, output_train.shape[1])
def generate_output(X,
y,
speaker_names,
network_params_file_in=None,
output_file_out=None,
network_fun=None,
get_conv_output=None,
output_layer=None,
overlapping=False):
X_cluster, y_cluster = generate_cluster_data(X, y, overlapping=overlapping)
input_var = T.tensor4('inputs')
network = network_fun(input_var)
if network_params_file_in is not None:
all_param_values = pickler.load(network_params_file_in)
lasagne.layers.set_all_param_values(network, all_param_values)
if output_layer is not None:
network = lasagne.layers.get_all_layers(network)[output_layer]
prediction = lasagne.layers.get_output(network, deterministic=True)
predict = theano.function([input_var], prediction)
data_len = len(X_cluster)
if get_conv_output is not None:
X_conv = np.zeros((data_len, 1, 1, 1000), dtype=np.float32)
output = np.zeros((data_len, network.num_units), dtype=np.float32)
step = 100
for i in range(0, data_len, step):
next_step = data_len - i
if next_step > step:
next_step = step
if get_conv_output is None:
new_output = predict(X_cluster[i:i + next_step])
else:
X_conv[i:i + next_step, 0,
0] = get_conv_output(X_cluster[i:i + next_step])
new_output = predict(X_conv[i:i + next_step])
output[i:i + step] = new_output
if output_file_out is None:
return output, y_cluster, speaker_names
else:
with open(output_file_out, 'wb') as f:
pickle.dump((output, y_cluster, speaker_names), f, -1)
def load_dev_test_data(data_path, number_speakers, sentences):
X, y, s_list = load(data_path)
if number_speakers < 80:
X_sampled = np.zeros(X.shape)
y_sampled = np.zeros(y.shape, dtype=np.int8)
for i in range(number_speakers):
index = random.randrange(80 - i)
X_extraced, y_extracted, X, y = get_sentences_for_speaker_index(
X, y, index, i, sentences)
X_sampled[i * sentences:i * sentences + sentences] = X_extraced
y_sampled[i * sentences:i * sentences + sentences] = y_extracted
X, speakers = X_sampled[:(number_speakers *
sentences)], y_sampled[:(number_speakers *
sentences)]
return X, speakers, s_list
def create_and_train(self, training_data):
# Load training data
x, y, speaker_names = load(training_data)
# Create network
net = create_net(self.create_paper(x.shape[1]))
# Set new batch iterator
net.batch_iterator_train = SegmentBatchIterator(batch_size=128)
net.batch_iterator_test = SegmentBatchIterator(batch_size=128)
net.train_split = TrainSplit(eval_size=0)
# Train the network
self.logger.info("Fitting...")
net.fit(x, y)
# Comments from old spectrogram_cnn_100 implementation, don't delete yet if eventually needed later
# net.load_params_from('../data/experiments/paper/networks/net_100_81_not_reynolds.pickle');
# net.save_params_to('../../data/experiments/paper/networks/net_100_81_not_reynolds.pickle');
# network_helper.save(net, '../../data/experiments/paper/networks/net_100_81_not_reynolds.pickle')
save(net, self.net_path)
import pickle
from networks.pairwise_lstm.lstm_controller import LSTMController
from common.utils.paths import *
from common.utils.pickler import load
import controller
lstmcontroller = LSTMController
lstmcontroller.val_data = "speakers_all"
cluster_range = 22
mr_dict = {}
for i in range(20 , cluster_range + 1):
print("................Running CLuster loop for Speaker Count = " + str(i)+ "..................")
lstmcontroller.test_network(cluster_count=i, mr_list=mr_dict)
with open(get_experiment_results("VCTK_100_speakers_100_00999" + str(cluster_range)), 'wb') as f:
pickle.dump(mr_dict, f, -1)
loaded_dict = load(get_experiment_results("VCTK_100_speakers_100_00999" + str(cluster_range)))
#p#rint("----------->>>>>>>>> from disk")
#print("-------->>>>> mrdict")
print(mr_dict)
def load_test_data(data_path):
x, y, s_list = load(data_path)
return x, y, s_list
def read_result_pickle(files):
"""
Reads the results of a network from these files.
:param files: can be 1-n files that contain a result.
:return: curve names, thresholds, mrs, homogeneity scores, completeness scores and number of embeddings
"""
logger = get_logger('analysis', logging.INFO)
logger.info('Read result pickle')
curve_names = []
# Initialize result sets
set_of_thresholds = []
set_of_mrs = []
set_of_homogeneity_scores = []
set_of_completeness_scores = []
set_of_number_of_embeddings = []
logger = get_logger('analysis', logging.INFO)
# Fill result sets
for file in files:
print("File name : " + file)
curve_name, mrs, homogeneity_scores, completeness_scores, number_of_embeddings = load(
file)
for index, curve_name in enumerate(curve_name):
set_of_mrs.append(mrs[index])
set_of_homogeneity_scores.append(homogeneity_scores[index])
set_of_completeness_scores.append(completeness_scores[index])
set_of_number_of_embeddings.append(number_of_embeddings[index])
curve_names.append(curve_name + "_" + file[89:])
return curve_names, set_of_mrs, set_of_homogeneity_scores, set_of_completeness_scores, set_of_number_of_embeddings
def prepare(network_file, input_var):
net = pickler.load(network_file)
conv_network = net.get_all_layers()[5]
y = layers.get_output(conv_network, input_var)
return theano.function([input_var], y)
storeDict2("lstm_overlap_cluster_timit_40_corpus_", ii)
storeDict3("lstm_overlap_cluster_timit_40_corpus_", ii)
storeDict("TIMIT_OVERLAP_ON_LSTM_ALL_CHECKPOINT", "ALL")
storeDict2("TIMIT_OVERLAP_ON_LSTM_ALL_CHECKPOINT", "ALL")
storeDict3("TIMIT_OVERLAP_ON_LSTM_ALL_CHECKPOINT", "ALL")
print("----------->>>>>>>>> From memory")
print(mr_dict)
print("\n")
print(mr_dict2)
print("\n")
print(mr_dict3)
loaded_dict = load(
get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" +
str(cluster_range) + "1"))
loaded_dict2 = load(
get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" +
str(cluster_range) + "2"))
loaded_dict3 = load(
get_experiment_results("lstm_overlap_cluster_timit_40_corpus_" +
str(cluster_range) + "3"))
print("\n")
print("----------->>>>>>>>> from disk")
print("Hierechial")
print(loaded_dict)
print("\n")
print("K_Means")
def load_and_prepare_data(data_path):
# Load and generate test data
x, y, s_list = load(data_path)
return generate_cluster_data(x, y)
