def load_model(self, file_name):
# Handle gzipped files.
if file_name.endswith('gz'):
import gzip
open_meth = gzip.open
else:
open_meth = open
with open_meth(file_name, 'rb') as model_file:
(self.classifiers_features_list, self.classifiers_features_mapping,
self.trained_classifiers_params, self.parsed_classifiers,
self.features_size) = pickle.load(model_file)
self.trained_classifiers = {}
for clser in self.trained_classifiers_params:
self.trained_classifiers[clser] = tffnn.TheanoFFNN()
self.trained_classifiers[clser].set_params(
self.trained_classifiers_params[clser])
def train_nn(speech_data, speech_alignment):
print
print datetime.datetime.now()
print
random.seed(0)
try:
f = open(fetures_file_name, "rb")
crossvalid_x = np.load(f)
crossvalid_y = np.load(f)
train_x = np.load(f)
train_y = np.load(f)
tx_m = np.load(f)
tx_std = np.load(f)
f.close()
except IOError:
crossvalid_x, crossvalid_y, train_x, train_y, tx_m, tx_std = gen_features(
speech_data, speech_alignment)
input_size = train_x.shape[1] * (prev_frames + 1 + next_frames)
output_size = np.amax(train_y) + 1
print "The shape of non-multiplied training data: ", train_x.shape, train_y.shape
print "The shape of non-multiplied test data: ", crossvalid_x.shape, crossvalid_y.shape
# add prev, and last frames
tx_m = np.tile(tx_m, prev_frames + 1 + next_frames)
tx_std = np.tile(tx_std, prev_frames + 1 + next_frames)
print
print datetime.datetime.now()
print
print "prev_frames +1+ last_frames:", prev_frames, 1, next_frames
print "The shape of training data: ", train_x.shape, train_y.shape
print "The shape of test data: ", crossvalid_x.shape, crossvalid_y.shape
print "The shape of tx_m, tx_std: ", tx_m.shape, tx_std.shape
print "The output size: ", output_size
print
e = tffnn.TheanoFFNN(input_size,
hidden_units,
hidden_layers,
output_size,
hidden_activation=hact,
weight_l2=weight_l2,
training_set_x=train_x,
training_set_y=train_y,
prev_frames=prev_frames,
next_frames=next_frames,
amplify_center_frame=amplify_center_frame,
batch_size=batch_size)
e.set_input_norm(tx_m, tx_std)
dc_acc = []
dt_acc = []
epoch = 0
i_hidden_layers = hidden_layers
while True:
print
print '-' * 80
print 'Predictions'
print '-' * 80
predictions_y, gold_y = e.predict(crossvalid_x, batch_size,
prev_frames, next_frames,
crossvalid_y)
c_acc, c_sil = get_accuracy(gold_y, predictions_y)
predictions_y, gold_y = e.predict(train_x, batch_size, prev_frames,
next_frames, train_y)
t_acc, t_sil = get_accuracy(gold_y, predictions_y)
dc_acc.append(c_acc)
dt_acc.append(t_acc)
print
print "method, hact, max_frames, max_files, max_frames_per_segment, trim_segments, batch_size, max_epoch, " \
"hidden_units, hidden_layers, hidden_layers_add, prev_frames, next_frames, crossvalid_frames, usec0, usedelta, useacc, mel_banks_only "
print method, hact, max_frames, max_files, max_frames_per_segment, trim_segments, batch_size, max_epoch, \
hidden_units, hidden_layers, hidden_layers_add, prev_frames, next_frames, crossvalid_frames, usec0, usedelta, useacc, mel_banks_only
print "Epoch: %d" % (epoch, )
print
print "Cross-validation stats"
print "------------------------"
print "Epoch predictive accuracy: %0.4f" % c_acc
print "Last epoch accs:", ["%.4f" % x for x in dc_acc[-20:]]
print "Epoch sil bias: %0.2f" % c_sil
print
print "Training stats"
print "------------------------"
print "Epoch predictive accuracy: %0.4f" % t_acc
print "Last epoch accs:", ["%.4f" % x for x in dt_acc[-20:]]
print "Epoch sil bias: %0.2f" % t_sil
print
print "Best results"
print "------------------------"
print "Best iteration:", np.argmax(dc_acc)
print "Best iteration - cross-validation acc: %.4f" % dc_acc[np.argmax(
dc_acc)]
print "Best iteration - training acc: %.4f" % dt_acc[np.argmax(
dc_acc)]
print
print datetime.datetime.now()
print
if epoch == np.argmax(dc_acc):
print
print "Saving the FFNN and TFFN models"
print
file_name = "model_voip/lid_nnt_%d_hu%d_hl%d_hla%d_pf%d_nf%d_acf_%.1f_mfr%d_mfl%d_mfps%d_ts%d_usec0%d_usedelta%d_useacc%d_mbo%d_bs%d" % \
(input_size, hidden_units, hidden_layers, hidden_layers_add,
prev_frames, next_frames, amplify_center_frame, max_frames, max_files, max_frames_per_segment,
trim_segments,
usec0, usedelta, useacc, mel_banks_only, batch_size)
nn = ffnn.FFNN()
for w, b in [
e.params[n:n + 2] for n in range(0, len(e.params), 2)
]:
nn.add_layer(w.get_value(), b.get_value())
nn.set_input_norm(tx_m, tx_std)
nn.save(file_name + ".ffnn")
e.save(file_name + ".tffnn")
if epoch == max_epoch:
break
epoch += 1
if epoch > 1 and epoch % 1 == 0:
if i_hidden_layers < hidden_layers + hidden_layers_add:
print
print '-' * 80
print 'Adding a hidden layer: ', i_hidden_layers + 1
print '-' * 80
e.add_hidden_layer(hidden_units)
i_hidden_layers += 1
print
print '-' * 80
print 'Training'
print '-' * 80
e.train(method=method,
learning_rate=learning_rate * learning_rate_decay /
(learning_rate_decay + epoch))
def train(self, inverse_regularisation=1.0, verbose=True):
self.trained_classifiers = {}
if verbose:
print '=' * 120
print 'Training'
for n, clser in enumerate(sorted(self.classifiers)):
if verbose:
print '-' * 120
print "Training classifier: ", clser, ' #', n + 1, '/', len(
self.classifiers)
print " Matrix: ", (
len(self.classifiers_outputs[clser]),
len(self.classifiers_features_list[clser]))
classifier_input = np.zeros(
(len(self.classifiers_outputs[clser]),
len(self.classifiers_features_list[clser])),
dtype=np.float32)
for i, feat in enumerate(self.classifiers_features[clser]):
classifier_input[i] = feat.get_feature_vector(
self.classifiers_features_mapping[clser])
# standardise the data
m = np.mean(classifier_input, axis=0)
std = np.std(classifier_input, axis=0)
# replace 0.0 std by 1.0
inds = np.where(np.isclose(std, 0.0))
std[inds] = 1.0
m = np.zeros_like(m)
std = np.ones_like(std)
classifier_input -= m
classifier_input /= std
indices = np.random.permutation(classifier_input.shape[0])
training_data_size = int(0.9 * len(indices))
training_idx, test_idx = indices[:training_data_size], indices[
training_data_size:]
training_input, crossvalid_input = classifier_input[
training_idx, :], classifier_input[test_idx, :]
training_output, crossvalid_output = self.classifiers_outputs[
clser][training_idx, ], self.classifiers_outputs[clser][
test_idx, ]
if not np.isclose(training_output, 1.0).any():
print "Warning! At least one example should be positive"
training_output[0] = 1
# lr = LogisticRegression('l2', C=inverse_regularisation, tol=1e-3)
#
# lr.fit(training_input, training_output)
method = 'sg-fixedlr'
hact = 'tanh'
learning_rate = 20e-3 # 5e-2
learning_rate_decay = 1000.0
batch_size = 10
nn = tffnn.TheanoFFNN(training_input.shape[1],
16,
0,
2,
hidden_activation=hact,
weight_l2=1e-16,
training_set_x=training_input,
training_set_y=training_output,
batch_size=batch_size)
nn.set_input_norm(m, std)
max_crossvalid_mean_accuracy = 0.0
for epoch in range(200):
print "Epoch", epoch
predictions_y = nn.predict(training_input,
batch_size=batch_size)
training_mean_accuracy = np.mean(
np.equal(np.argmax(predictions_y, axis=1),
training_output)) * 100.0
print " Prediction accuracy on the training data: %6.2f" % (
training_mean_accuracy, )
print " the training data size: ", training_input.shape
predictions_y = nn.predict(crossvalid_input,
batch_size=batch_size)
crossvalid_mean_accuracy = np.mean(
np.equal(np.argmax(predictions_y, axis=1),
crossvalid_output)) * 100.0
print " Prediction accuracy on the crossvalid data: %6.2f" % (
crossvalid_mean_accuracy, )
print " the crossvalid data size: ", crossvalid_input.shape
if max_crossvalid_mean_accuracy < crossvalid_mean_accuracy:
print " Storing the best classifiers so far"
self.trained_classifiers[clser] = nn
max_crossvalid_mean_accuracy = crossvalid_mean_accuracy
if training_mean_accuracy >= 99.99 or max_crossvalid_mean_accuracy >= 99.99:
print " Stop: It does not have to be better"
break
print " Training "
nn.train(method=method,
learning_rate=learning_rate * learning_rate_decay /
(learning_rate_decay + epoch))