from util import dnn_load_data
import numpy as np
for tv in ['train', 'valid']:
X1 = dnn_load_data('../feature/%s1M.fbank'%tv)
X2 = dnn_load_data('../feature/%s1M.mfcc'%tv)
X = np.concatenate((X1, X2), axis=1)
print "Save ../feature/%s1M.fm" %tv
np.savetxt('../feature/%s1M.fm'%tv, X, fmt='%7f')
import numpy as np
from util import dnn_load_data, dnn_save_data
from sklearn.preprocessing import StandardScaler
import os
if __name__ == "__main__":
data_dir = '../feature'
feature = 'fbank7'
filename = os.path.join(data_dir, 'train.%s'%feature)
X = dnn_load_data(filename)
print "Standard Normalization..."
scaler = StandardScaler().fit(X)
X = scaler.transform(X)
filename = os.path.join(data_dir, 'train.%s.norm'%feature)
dnn_save_data(filename, X)
for t in ["test", 'test.old']:
filename = os.path.join(data_dir, '%s.%s'%(t, feature) )
X = dnn_load_data(filename)
X = scaler.transform(X)
filename = os.path.join(data_dir, '%s.%s.norm'%(t, feature) )
dnn_save_data(filename, X)
frame = FRAME()
fs = frame_str.split("_")
frame.input = frame_str
frame.ss = fs[0] + "_" + fs[1]
frame.id = int(fs[2])
frame.index = index
return frame
fm = 5 # number of frame to merge
feature_name = 'fbank'
for t in ['train', 'test', 'test.old']:
feature_filename = '../feature/%s.%s' % (t, feature_name)
X = dnn_load_data(feature_filename)
(N, D) = X.shape
x_zero = np.zeros(X[0].shape)
y_zero = 'zero'
frame_filename = '../frame/%s.frame' % t
print "Load %s" % frame_filename
input_list = np.loadtxt(frame_filename, dtype='str')
N = len(input_list)
frame_list = []
for i in range(N):
frame = make_frame(input_list[i], i)
frame_list.append(frame)
str(opts.rmsprop_alpha) )
opts.model_dir = '../model/%s' % parameters
# load model
model_filename = os.path.join(opts.model_dir, 'epoch%d.model' % opts.epoch)
dnn = dnn_load_model(model_filename)
# output
layer = 3
fv_out = '%s.%s_nn%s_%s_drop%s.L%d' \
%(opts.feature, \
opts.data_size, \
"_".join( str(h) for h in opts.hidden), \
opts.label_type, \
opts.dropout_prob, \
layer)
output_dir = '../../hw2/hw1_feature'
for t in ["train", "test", "test.old"]:
filename = '../feature/%s.%s' % (t, opts.feature)
X = dnn_load_data(filename)
print "Extract dnn feature..."
feature = dnn.get_hidden_feature(X, layer)
output_filename = os.path.join(output_dir, '%s.%s' % (t, fv_out))
dnn_save_data(output_filename, feature)
opts.update_grad, \
str(opts.rmsprop_alpha) )
if( opts.pretrain ):
parameters += '_RBMpretrain'
if( opts.data_size == 'all' ):
train_filename = '../feature/train.%s' %(opts.feature)
train_labelname = '../label/train.%s.index' %(opts.label_type)
else:
train_filename = '../feature/train%s.%s' %(opts.data_size, opts.feature)
train_labelname = '../label/train%s.%s.index' %(opts.data_size, opts.label_type)
X_train, Y_train = dnn_load_data(train_filename, train_labelname, opts.N_class)
if( opts.data_size == 'all' ):
valid_filename = '../feature/valid1M.%s' %(opts.feature)
valid_labelname = '../label/valid1M.%s.index' %(opts.label_type)
else:
valid_filename = '../feature/valid%s.%s' %(opts.data_size, opts.feature)
valid_labelname = '../label/valid%s.%s.index' %(opts.data_size, opts.label_type)
X_valid, Y_valid = dnn_load_data(valid_filename, valid_labelname, opts.N_class)
(N_data, N_dim) = X_train.shape
opts.structure = [N_dim] + opts.hidden + [opts.N_class]
str(opts.momentum), \
str(opts.weight_decay), \
str(opts.dropout_prob), \
opts.update_grad, \
str(opts.rmsprop_alpha) )
opts.model_dir = '../model/%s' %parameters
# load model
model_filename = os.path.join(opts.model_dir, 'epoch%d.model'%epoch)
dnn = dnn_load_model(model_filename)
# testing (old data)
test_filename = '../feature/test.old.%s' %opts.feature
X_test = dnn_load_data(test_filename)
output_filename = '../pred/%s_epoch%d.old.csv' %(parameters, epoch)
Y_pred = dnn.predict(X_test)
dnn_save_label('../frame/test.old.frame', output_filename, Y_pred, opts.label_type)
# testing (final data)
test_filename = '../feature/test.%s' %opts.feature
X_test = dnn_load_data(test_filename)
output_filename = '../pred/%s_epoch%d.csv' %(parameters, epoch)
Y_pred = dnn.predict(X_test)