def load_river_network(nnet_param = 'neural_network/river_network_params', nnet_cfg = 'neural_network/river_network_cfg'):
cfg = cPickle.load(smart_open(nnet_cfg,'r'))
cfg.init_activation()
model = DNN(numpy_rng=numpy_rng, cfg = cfg)
_file2nnet(model.layers, filename = nnet_param)
get_river_probs = model.build_extract_feat_function(-1)
return get_river_probs
def load_dnn(dnn_fname, flat_start=False):
""" Load a DNN from disk, automatically determining NetworkConfig.
If `flat_start` is True, returned DNN will have the same architecture
as the base DNN, but with randomly initialized weights instead.
"""
cfg = get_dnn_cfg(dnn_fname)
dnn = init_dnn(cfg)
_file2nnet(dnn.layers, filename=dnn_fname)
return dnn
def _set_MLPs(self):
'''
load the MLP learned model as MLP network
'''
nnet_param = os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir, 'models_makam',
'dampB.mdl')
nnet_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir, 'models_makam',
'dampB.cfg')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg = cPickle.load(smart_open(nnet_cfg, 'r'))
cfg.init_activation()
self.cfg = cfg
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load model parameters
_file2nnet(model.layers, filename=nnet_param) # this is very slow
self.model = model
batch_size = float(arguments['batch_size'])
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
cfg = cPickle.load(smart_open(nnet_cfg,'r'))
cfg.init_activation()
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
# load model parameters
_file2nnet(model.layers, filename = nnet_param)
# initialize data reading
cfg.init_data_reading_test(data_spec)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
output_mats = [] # store the features for all the data in memory. TODO: output the features in a streaming mode
log('> ... generating features from the specified layer')
while (not cfg.test_sets.is_finish()): # loop over the data
cfg.test_sets.load_next_partition(cfg.test_xy)
batch_num = int(math.ceil(1.0 * cfg.test_sets.cur_frame_num / batch_size))
for batch_index in xrange(batch_num): # loop over mini-batches
log('> ... getting the pre-training functions')
pretraining_fns = srbm.pretraining_functions(train_set_x=cfg.train_x,
batch_size=cfg.batch_size,
k=1,
weight_cost=0.0002)
start_layer_index = 0
start_epoch_index = 0
if os.path.exists(wdir +
'/nnet.tmp') and os.path.exists(wdir +
'/training_state.tmp'):
start_layer_index, start_epoch_index = read_two_integers(
wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from layer #'
+ str(start_layer_index) + ' epoch #' + str(start_epoch_index))
_file2nnet(dnn.layers, filename=wdir + '/nnet.tmp')
log('> ... training the model')
log('> r_cost = reconstruction cost, fe_cost = (approximate) value of free energy function'
)
# pre-train layer-wise
for i in xrange(start_layer_index, cfg.ptr_layer_number):
if (srbm.rbm_layers[i].is_gbrbm()):
pretrain_lr = cfg.gbrbm_learning_rate
else:
pretrain_lr = cfg.learning_rate
# go through pretraining epochs
momentum = cfg.initial_momentum
for epoch in range(start_epoch_index, cfg.epochs):
# go through the training set
if (epoch == cfg.initial_momentum_epoch):
log("Extracting all features per partition at once")
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
cfg = pickle.load(smart_open(nnet_cfg,'rb'))
cfg.init_activation()
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
# load model parameters
_file2nnet(model.layers, path = nnet_param)
# initialize data reading
cfg.init_data_reading_test(data_spec)
model.dumpLayerSize()
# get the function for feature extraction
log('> ... getting the feat-extraction function for layer='+str(layer_index))
extract_func = model.build_extract_feat_function(layer_index)
output_mats = [] # store the features for all the data in memory. TODO: output the features in a streaming mode
log('> ... generating features from the specified layer')
while (not cfg.test_sets.is_finish()): # loop over the data
cfg.test_sets.load_next_partition(cfg.test_xy)
print shared_layers
dnn = DNN_MTL(task_id=n,numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg,
dnn_shared = dnn_shared, shared_layers = shared_layers)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions for task %d' % (n))
train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size)
# add dnn and the functions to the list
dnn_array.append(dnn)
train_fn_array.append(train_fn); valid_fn_array.append(valid_fn)
# check the working dir to decide whether it's resuming training; if yes, load the tmp network files for initialization
if os.path.exists(wdir + '/nnet.tmp.task' + str(n)) and os.path.exists(wdir + '/training_state.tmp.task' + str(n)):
resume_training = True; resume_tasks.append(n)
cfg.lrate = _file2lrate(wdir + '/training_state.tmp.task' + str(n))
log('> ... found nnet.tmp.task%d and training_state.tmp.task%d, now resume task%d training from epoch %d' % (n, n, n, cfg.lrate.epoch))
_file2nnet(dnn.layers, filename = wdir + '/nnet.tmp.task' + str(n))
# pre-training works only if we are NOT resuming training
# we assume that we only pre-train the shared layers; thus we only use dnn_array[0] to load the parameters
# because the parameters are shared across the tasks
ptr_layer_number = 0; ptr_file = ''
if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
ptr_file = arguments['ptr_file']; ptr_layer_number = int(arguments['ptr_layer_number'])
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn_array[0].layers, set_layer_num = ptr_layer_number, filename = ptr_file)
log('> ... finetuning the model')
train_error_array = [[] for n in xrange(task_number)]
active_tasks = [n for n in xrange(task_number)] # the tasks with 0 learning rate are not considered
if resume_training:
active_tasks = resume_tasks
if arguments.has_key('use_fast'):
use_fast = string_2_bool(arguments['use_fast'])
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3])
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, conv_layer_configs = conv_configs, use_fast = use_fast)
_file2nnet(cnn.conv_layers, set_layer_num = len(conv_configs), filename=cnn_param_file)
out_function = cnn.build_out_function()
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
in_matrix = numpy.reshape(in_matrix, (in_matrix.shape[0],) + input_shape_1)
out_matrix = out_function(in_matrix)
kaldiwrite.write_kaldi_mat(uttid, out_matrix)
kaldiwrite.close()
log('> ... the saved features are %s' % (out_ark_file))
kaldiwrite = KaldiWriteOut(out_ark_file)
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
log('> ... setting up the CNN layers')
conv_configs = cfg.conv_layer_configs
conv_layer_number = len(conv_configs)
use_fast = cfg.use_fast
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3])
for i in xrange(conv_layer_number):
conv_configs[i]['activation'] = cfg.conv_activation
cfg.init_activation()
cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, conv_layer_configs = conv_configs, use_fast = use_fast)
_file2nnet(layers = cnn.conv_layers, set_layer_num = len(conv_configs), filename = cldnn_param_file, start_layer = 0)
out_function1 = cnn.build_out_function()
conv_output_dim = cfg.conv_layer_configs[-1]['output_shape'][1] * cfg.conv_layer_configs[-1]['output_shape'][2] * cfg.conv_layer_configs[-1]['output_shape'][3]
print conv_output_dim
log('> ... setting up the LSTM layers')
cfg.init_activation()
lstm = LSTM_Forward(lstm_layer_configs = cfg, n_ins = conv_output_dim)
_file2nnet(layers = lstm.lstm_layers, set_layer_num = lstm.lstm_layer_num+len(conv_configs), filename = cldnn_param_file, start_layer = len(conv_configs))
out_function2 = lstm.build_out_function()
cfg.n_ins = cfg.lstm_layers_sizes[-1]
log('> ... setting up the DNN layers')
dnn = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=lstm.lstm_layers[-1].output)
_file2nnet(layers = dnn.layers, set_layer_num = len(dnn.layers)+lstm.lstm_layer_num+len(conv_configs), filename = cldnn_param_file, start_layer = lstm.lstm_layer_num+len(conv_configs))
out_function3 = dnn.build_extract_feat_function(layer_index)
path = "/home/piero/Documents/Experiments/Real_Test/Spectral Coef/Noise vs BG_voice+Conversation vs Shout+Scream/fft coef/"
os.chdir(path)
filename = "rbm.cfg"
train_data = "train.pickle.gz"
test_data = "test.pickle.gz"
batch_size = 128
log('> ... setting up the model and loading parameters')
numpy_rng = np.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg_dnn = cPickle.load(open(filename, 'r'))
cfg_dnn.init_activation()
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn)
# load model parameters
_file2nnet(model.layers, filename=wdir + '/rbm.param')
# initialize data reading
cfg_dnn.init_data_reading_test(train_data_spec)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(-1)
output_mat = None # store the features for all the data in memory
log('> ... generating features from the specified layer')
while (not cfg_dnn.test_sets.is_finish()): # loop over the data
cfg_dnn.test_sets.load_next_partition(cfg_dnn.test_xy)
batch_num = int(math.ceil(cfg_dnn.test_sets.cur_frame_num / batch_size))
for batch_index in xrange(batch_num): # loop over mini-batches
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# construct the cnn architecture
cnn = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng,
batch_size = batch_size, n_outs=n_outs,
conv_layer_configs = conv_layer_configs,
hidden_layers_sizes = hidden_layers_sizes,
conv_activation = conv_activation,
full_activation = full_activation,
use_fast = use_fast, update_layers = update_layers)
total_layer_number = len(cnn.layers)
if full_ptr_layer_number > 0:
_file2nnet(cnn.layers[len(conv_layer_configs):total_layer_number], set_layer_num = full_ptr_layer_number, filename = full_ptr_file, withfinal=False)
if conv_ptr_layer_number > 0:
_file2cnn(cnn.layers[0:len(conv_layer_configs)], filename=conv_ptr_file)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = cnn.build_finetune_functions(
(train_x, train_y), (valid_x, valid_y),
batch_size=batch_size)
log('> ... finetunning the model')
start_time = time.clock()
while (lrate.get_rate() != 0):
train_error = []
while (not train_sets.is_finish()):
train_sets.load_next_partition(train_xy)
cfg.lrate = _file2lrate(wdir + '/training_state.tmp_DNN')
log('> ... found nnet.tmp_DNN and training_state.tmp_DNN, now resume training from epoch ' + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# initialize model parameters
if resume_training:
_file2nnet(dnn, filename = wdir + '/nnet.tmp_DNN')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
count = 0
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_acc, train_err = train_sgd(train_fn, cfg)
log('> epoch %d, lrate %f, training accuracy %f' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100*numpy.mean(train_acc)) + '(%)')
# validation
valid_acc, valid_err = validate_by_minibatch(valid_fn, cfg)
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir + "/nnet.tmp") and os.path.exists(wdir + "/training_state.tmp"):
resume_training = True
cfg.lrate = _file2lrate(wdir + "/training_state.tmp")
log("> ... found nnet.tmp and training_state.tmp, now resume training from epoch " + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log("> ... initializing the model")
# construct the cnn architecture
cnn = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load the pre-training networks, if any, for parameter initialization
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(cnn.layers, set_layer_num=ptr_layer_number, filename=ptr_file)
if resume_training:
_file2nnet(cnn.layers, filename=wdir + "/nnet.tmp")
# get the training, validation and testing function for the model
log("> ... getting the finetuning functions")
train_fn, valid_fn = cnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size
)
log("> ... finetunning the model")
while cfg.lrate.get_rate() != 0:
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log("> epoch %d, training error %f " % (cfg.lrate.epoch, 100 * numpy.mean(train_error)) + "(%)")
# validation
path = "/home/piero/Documents/Experiments/Real_Test/Spectral Coef/Noise vs BG_voice+Conversation vs Shout+Scream/fft coef/"
os.chdir(path)
filename = "rbm.cfg"
train_data = "train.pickle.gz"
test_data = "test.pickle.gz"
batch_size = 128
log('> ... setting up the model and loading parameters')
numpy_rng = np.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
cfg_dnn = cPickle.load(open(filename,'r'))
cfg_dnn.init_activation()
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_dnn)
# load model parameters
_file2nnet(model.layers, filename = wdir + '/rbm.param')
# initialize data reading
cfg_dnn.init_data_reading_test(train_data_spec)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(-1)
output_mat = None # store the features for all the data in memory
log('> ... generating features from the specified layer')
while (not cfg_dnn.test_sets.is_finish()): # loop over the data
cfg_dnn.test_sets.load_next_partition(cfg_dnn.test_xy)
batch_num = int(math.ceil(cfg_dnn.test_sets.cur_frame_num / batch_size))
for batch_index in xrange(batch_num): # loop over mini-batches
def main(arg_elements):
# check the arguments
arguments = parse_arguments(arg_elements)
required_arguments = [
'data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index',
'batch_size'
]
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
# mandatory arguments
data_spec = arguments['data']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
output_file = arguments['output_file']
layer_index = int(arguments['layer_index'])
batch_size = int(arguments['batch_size'])
argmax = arguments.has_key('argmax') and string2bool(arguments['argmax'])
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg = cPickle.load(smart_open(nnet_cfg, 'r'))
cfg.init_activation()
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg=cfg,
testing=True)
# load model parameters
_file2nnet(model.layers, filename=nnet_param)
# initialize data reading
cfg.init_data_reading_test(data_spec)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
output_mats = [
] # store the features for all the data in memory. TODO: output the features in a streaming mode
log('> ... generating features from the specified layer')
while (not cfg.test_sets.is_finish()): # loop over the data
cfg.test_sets.load_next_partition(cfg.test_xy)
batch_num = int(
math.ceil(1.0 * cfg.test_sets.cur_frame_num / batch_size))
for batch_index in xrange(batch_num): # loop over mini-batches
start_index = batch_index * batch_size
end_index = min((batch_index + 1) * batch_size,
cfg.test_sets.cur_frame_num
) # the residue may be smaller than a mini-batch
output = extract_func(
cfg.test_x.get_value()[start_index:end_index])
output_mats.append(output)
output_mat = numpy.concatenate(output_mats)
if argmax:
output_mat = output_mat.argmax(axis=1)
# output the feature representations using pickle
f = smart_open(output_file, 'wb')
cPickle.dump(output_mat, f, cPickle.HIGHEST_PROTOCOL)
log('> ... the features are stored in ' + output_file)
def dnn_run(arguments):
required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg); exit(1)
train_data_spec = arguments['train_data']
valid_data_spec = arguments['valid_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
cfg = NetworkConfig()
cfg.parse_config_dnn(arguments, nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
# parse pre-training options
# pre-training files and layer number (how many layers are set to the pre-training parameters)
ptr_layer_number = 0; ptr_file = ''
if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
ptr_file = arguments['ptr_file']
ptr_layer_number = int(arguments['ptr_layer_number'])
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file)
if resume_training:
_file2nnet(dnn.layers, filename = wdir + '/nnet.tmp')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) + '(%)')
# validation
valid_error = validate_by_minibatch(valid_fn, cfg)
log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100*numpy.mean(valid_error)) + '(%)')
cfg.lrate.get_next_rate(current_error = 100*numpy.mean(valid_error))
# output nnet parameters and lrate, for training resume
if cfg.lrate.epoch % cfg.model_save_step == 0:
_nnet2file(dnn.layers, filename=wdir + '/nnet.tmp')
_lrate2file(cfg.lrate, wdir + '/training_state.tmp')
# save the model and network configuration
if cfg.param_output_file != '':
_nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor)
log('> ... the final PDNN model parameter is ' + cfg.param_output_file)
if cfg.cfg_output_file != '':
_cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2],
input_shape_train[3])
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
cnn = CNN_Forward(numpy_rng=rng,
theano_rng=theano_rng,
conv_layer_configs=conv_configs,
use_fast=use_fast)
_file2nnet(cnn.conv_layers,
set_layer_num=len(conv_configs),
filename=cnn_param_file)
out_function = cnn.build_out_function()
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
in_matrix = numpy.reshape(in_matrix,
(in_matrix.shape[0], ) + input_shape_1)
out_matrix = out_function(in_matrix)
kaldiwrite.write_kaldi_mat(uttid, out_matrix)
kaldiwrite.close()
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2],
input_shape_train[3])
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2**30))
cfg.init_activation()
cnn = CNN_Forward(numpy_rng=rng,
theano_rng=theano_rng,
conv_layer_configs=conv_configs,
use_fast=use_fast)
#cnn = CNNV(numpy_rng = rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(cnn.conv_layers,
set_layer_num=len(conv_configs),
filename=cnn_param_file)
out_function = cnn.build_out_function()
#out_function = cnn.build_extract_feat_function(-1)
#print cnn.conv_layers[1].filter_shape
model = DNNV(numpy_rng=rng,
theano_rng=theano_rng,
cfg=cfg,
input=cnn.conv_layers[1].output)
_file2nnet(model.layers,
set_layer_num=len(model.layers) + len(conv_configs),
start_layer=len(conv_configs),
filename=cnn_param_file)
log('> ... processing the data')
# parse the structure of the i-vector network
cfg_adapt = NetworkConfig()
net_split = adapt_nnet_spec.split(":")
adapt_nnet_spec = ""
for n in xrange(len(net_split) - 1):
adapt_nnet_spec += net_split[n] + ":"
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + "0")
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log("> ... initializing the model")
# setup up the model
dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg_si=cfg_si, cfg_adapt=cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
# _file2nnet(dnn.cnn_si.layers, filename = init_model_file)
_file2nnet(dnn.cnn_si.layers, filename="BKUP/nnet.param.si")
_file2nnet(dnn.dnn_adapt.layers, filename="BKUP/nnet.param.adapt")
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log("> ... getting the finetuning functions for iVecNN")
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y), batch_size=cfg_adapt.batch_size
)
log("> ... learning the adaptation network")
cfg = cfg_adapt
while cfg.lrate.get_rate() != 0:
# one epoch of sgd training
# train_error = train_sgd_verbose(train_fn, cfg_si.train_sets, cfg_si.train_xy,
def dnn_run(arguments):
required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
train_data_spec = arguments['train_data']
valid_data_spec = arguments['valid_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
cfg = NetworkConfig()
cfg.parse_config_dnn(arguments, nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
# parse pre-training options
# pre-training files and layer number (how many layers are set to the pre-training parameters)
ptr_layer_number = 0
ptr_file = ''
if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
ptr_file = arguments['ptr_file']
ptr_layer_number = int(arguments['ptr_layer_number'])
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir +
'/nnet.tmp') and os.path.exists(wdir +
'/training_state.tmp'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch '
+ str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers,
set_layer_num=ptr_layer_number,
filename=ptr_file)
if resume_training:
_file2nnet(dnn.layers, filename=wdir + '/nnet.tmp')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log('> epoch %d, training error %f ' %
(cfg.lrate.epoch, 100 * numpy.mean(train_error)) + '(%)')
# validation
valid_error = validate_by_minibatch(valid_fn, cfg)
log('> epoch %d, lrate %f, validation error %f ' %
(cfg.lrate.epoch, cfg.lrate.get_rate(),
100 * numpy.mean(valid_error)) + '(%)')
cfg.lrate.get_next_rate(current_error=100 * numpy.mean(valid_error))
# output nnet parameters and lrate, for training resume
if cfg.lrate.epoch % cfg.model_save_step == 0:
_nnet2file(dnn.layers, filename=wdir + '/nnet.tmp')
_lrate2file(cfg.lrate, wdir + '/training_state.tmp')
# save the model and network configuration
if cfg.param_output_file != '':
_nnet2file(dnn.layers,
filename=cfg.param_output_file,
input_factor=cfg.input_dropout_factor,
factor=cfg.dropout_factor)
log('> ... the final PDNN model parameter is ' + cfg.param_output_file)
if cfg.cfg_output_file != '':
_cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
# parse the structure of the i-vector network
cfg_adapt = NetworkConfig()
net_split = adapt_nnet_spec.split(':')
adapt_nnet_spec = ''
for n in range(len(net_split) - 1):
adapt_nnet_spec += net_split[n] + ':'
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... initializing the model')
# setup up the model
dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
# _file2nnet(dnn.cnn_si.layers, filename = init_model_file)
_file2nnet(dnn.cnn_si.layers, filename = 'BKUP/nnet.param.si')
_file2nnet(dnn.dnn_adapt.layers, filename = 'BKUP/nnet.param.adapt')
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y),
batch_size = cfg_adapt.batch_size)
log('> ... learning the adaptation network')
cfg = cfg_adapt
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
# train_error = train_sgd_verbose(train_fn, cfg_si.train_sets, cfg_si.train_xy,
log('> ... setting up the CNN layers')
conv_configs = cfg.conv_layer_configs
conv_layer_number = len(conv_configs)
use_fast = cfg.use_fast
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2],
input_shape_train[3])
for i in xrange(conv_layer_number):
conv_configs[i]['activation'] = cfg.conv_activation
cfg.init_activation()
cnn = CNN_Forward(numpy_rng=rng,
theano_rng=theano_rng,
conv_layer_configs=conv_configs,
use_fast=use_fast)
_file2nnet(layers=cnn.conv_layers,
set_layer_num=len(conv_configs),
filename=cldnn_param_file,
start_layer=0)
out_function1 = cnn.build_out_function()
conv_output_dim = cfg.conv_layer_configs[-1]['output_shape'][
1] * cfg.conv_layer_configs[-1]['output_shape'][
2] * cfg.conv_layer_configs[-1]['output_shape'][3]
print conv_output_dim
log('> ... setting up the LSTM layers')
cfg.init_activation()
lstm = LSTM_Forward(lstm_layer_configs=cfg, n_ins=conv_output_dim)
_file2nnet(layers=lstm.lstm_layers,
set_layer_num=lstm.lstm_layer_num + len(conv_configs),
filename=cldnn_param_file,
start_layer=len(conv_configs))
out_function2 = lstm.build_out_function()
lstm_cfg_file = arguments['lstm_cfg_file']
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file,'r'))
cfg.init_activation()
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
lstm = ATTEND_LSTM(numpy_rng=rng, theano_rng=theano_rng, cfg = cfg)
_file2nnet(layers = lstm.layers, set_layer_num = len(lstm.layers), filename=lstm_param_file)
out_function = lstm.build_extract_feat_function()
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
print 'in_matrix:'+str(in_matrix.shape)
final_matrix = numpy.zeros((in_matrix.shape[0],cfg.n_outs), dtype=theano.config.floatX)
remainder = in_matrix.shape[0]%cfg.batch_size
for index in xrange(in_matrix.shape[0]/cfg.batch_size):
final_matrix[index*cfg.batch_size:(index+1)*cfg.batch_size] = out_function(in_matrix[index*cfg.batch_size:(index+1)*cfg.batch_size])
if remainder > 0:
print '\tremainder:'+str(remainder)
final_matrix[-remainder:] = out_function(in_matrix[-remainder:])
print 'final_matrix:'+str(final_matrix.shape)
kaldiread = KaldiReadIn(in_scp_file)
extra_kaldiread = KaldiReadIn(extra_in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3])
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cfg.init_activation()
log('> ... setting up the CNN layers')
cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, cfg = cfg)
_file2nnet(cnn.layers, cnn.extra_layers, set_layer_num = len(cnn.layers), filename=cnn_param_file)
out_function = cnn.build_out_function()
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
extra_uttid, extra_in_matrix = extra_kaldiread.read_next_utt()
if uttid == '':
break
print uttid
print extra_uttid
in_matrix = numpy.reshape(in_matrix, (in_matrix.shape[0],) + input_shape_1)
out_matrix = out_function(feat=in_matrix, extra_feat=extra_in_matrix)
print "out_matrix:" + str(out_matrix)
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg=cfg,
testing=True)
# load model parameters
_file2nnet(layers=model.layers,
set_layer_num=len(model.layers),
filename=nnet_param)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
log('> ... processing the data')
utt_number = 0
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
# in_matrix = numpy.reshape(in_matrix, (in_matrix.shape[0],) + input_shape_1)
out_matrix = extract_func(in_matrix)
print out_matrix
# network structure
cfg = cPickle.load(open(lstm_cfg_file, 'r'))
kaldiread = KaldiReadIn(in_scp_file)
extra_kaldiread = KaldiReadIn(extra_in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2**30))
cfg.init_activation()
lstm = PhaseATTENDLSTM_Forward(numpy_rng=rng,
lstm_layer_configs=cfg,
n_ins=cfg.n_ins)
_file2nnet(layers=lstm.lstm_layers,
set_layer_num=lstm.lstm_layer_num,
filename=lstm_param_file)
out_function = lstm.build_out_function()
log('> ... setting up the DNN layers')
dnn = DNNV(numpy_rng=rng,
theano_rng=theano_rng,
cfg=cfg,
input=lstm.lstm_layers[-1].output)
_file2nnet(layers=dnn.layers,
set_layer_num=len(dnn.layers) + lstm.lstm_layer_num,
filename=lstm_param_file,
start_layer=lstm.lstm_layer_num)
out_function2 = dnn.build_extract_feat_function(layer_index)
log('> ... processing the data')
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
# load model parameters
_file2nnet(layers = model.layers, set_layer_num = len(model.layers), filename = nnet_param)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
log('> ... processing the data')
utt_number = 0
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
# in_matrix = numpy.reshape(in_matrix, (in_matrix.shape[0],) + input_shape_1)
out_matrix = extract_func(in_matrix)
print out_matrix
kaldiwrite.write_kaldi_mat(uttid, out_matrix)
# parse the structure of the i-vector network
cfg_adapt = NetworkConfig()
# net_split = adapt_nnet_spec.split(':')
# adapt_nnet_spec = ''
# for n in xrange(len(net_split) - 1):
# adapt_nnet_spec += net_split[n] + ':'
# cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... initializing the model')
# setup up the model
dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
_file2nnet(dnn.dnn_si.layers, filename = init_model_file)
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y),
batch_size = cfg_adapt.batch_size)
log('> ... learning the adaptation network')
cfg = cfg_adapt
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd_verbose(train_fn, cfg_si.train_sets, cfg_si.train_xy,
cfg.batch_size, cfg.lrate.get_rate(), cfg.momentum)
log('> ... building the model')
if do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs,
activation = activation, dropout_factor = dropout_factor, input_dropout_factor = input_dropout_factor,
do_maxout = do_maxout, pool_size = pool_size,
max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs,
activation = activation, do_maxout = do_maxout, pool_size = pool_size,
do_pnorm = do_pnorm, pnorm_order = pnorm_order,
max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg)
if ptr_layer_number > 0:
_file2nnet(dnn.sigmoid_layers, set_layer_num = ptr_layer_number, filename = ptr_file, withfinal=False)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(train_x, train_y), (valid_x, valid_y),
batch_size=batch_size)
log('> ... finetunning the model')
start_time = time.clock()
while (lrate.get_rate() != 0):
train_error = []
while (not train_sets.is_finish()):
train_sets.load_next_partition(train_xy)
for batch_index in xrange(train_sets.cur_frame_num / batch_size): # loop over mini-batches
train_error.append(train_fn(index=batch_index, learning_rate = lrate.get_rate(), momentum = momentum))
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# construct the cnn architecture
cnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng,
batch_size = batch_size, n_outs=n_outs,
conv_layer_configs = conv_layer_configs,
hidden_layers_sizes = hidden_layers_sizes,
ivec_layers_sizes = ivec_layers_sizes,
conv_activation = conv_activation,
full_activation = full_activation,
use_fast = use_fast, update_part = update_part, ivec_dim = ivec_dim)
if arguments.has_key('conv_input_file'):
_file2cnn(cnn.conv_layers, filename=arguments['conv_input_file'])
if arguments.has_key('full_input_file'):
_file2nnet(cnn.full_layers, filename = arguments['full_input_file'])
if arguments.has_key('ivec_input_file'):
_file2nnet(cnn.ivec_layers, set_layer_num = len(ivec_layers_sizes) + 1, filename = arguments['ivec_input_file'], withfinal=False)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = cnn.build_finetune_functions(
(train_x, train_y), (valid_x, valid_y),
batch_size=batch_size)
log('> ... finetunning the model')
start_time = time.clock()
while (lrate.get_rate() != 0):
train_error = []
while (not train_sets.is_finish()):
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file, 'r'))
cfg.init_activation()
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2**30))
lstm = ATTEND_LSTM(numpy_rng=rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(layers=lstm.layers,
set_layer_num=len(lstm.layers),
filename=lstm_param_file)
out_function = lstm.build_extract_feat_function()
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
print 'in_matrix:' + str(in_matrix.shape)
final_matrix = numpy.zeros((in_matrix.shape[0], cfg.n_outs),
dtype=theano.config.floatX)
remainder = in_matrix.shape[0] % cfg.batch_size
for index in xrange(in_matrix.shape[0] / cfg.batch_size):
final_matrix[index * cfg.batch_size:(index + 1) *
cfg.batch_size] = out_function(
in_matrix[index * cfg.batch_size:(index + 1) *
valid_dataset, valid_dataset_args = read_data_args(valid_data_spec)
train_sets, train_xy, train_x, train_y = read_dataset(train_dataset, train_dataset_args)
valid_sets, valid_xy, valid_x, valid_y = read_dataset(valid_dataset, valid_dataset_args)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# doesn't deal with dropout
dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs,
activation = activation, do_maxout = do_maxout, pool_size = pool_size,
do_pnorm = do_pnorm, pnorm_order = pnorm_order,
max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg,
ivec_dim = ivec_dim, ivec_layers_sizes = ivec_layers_sizes)
# read the initial DNN
_file2nnet(dnn.sigmoid_layers, filename = si_model_file)
# get the training, validation and testing function for iVecNN
dnn.params = dnn.ivec_params
dnn.delta_params = dnn.ivec_delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(train_x, train_y), (valid_x, valid_y),
batch_size=batch_size)
log('> ... learning the iVecNN network')
while (lrate.get_rate() != 0):
train_error = []
while (not train_sets.is_finish()):
train_sets.load_next_partition(train_xy)
for batch_index in xrange(train_sets.cur_frame_num / batch_size): # loop over mini-batches
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3])
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cfg.init_activation()
cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, conv_layer_configs = conv_configs, use_fast = use_fast)
#cnn = CNNV(numpy_rng = rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(cnn.conv_layers, set_layer_num = len(conv_configs), filename=cnn_param_file)
out_function = cnn.build_out_function()
#out_function = cnn.build_extract_feat_function(-1)
#print cnn.conv_layers[1].filter_shape
model = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=cnn.conv_layers[1].output)
_file2nnet(model.layers, set_layer_num = len(model.layers)+len(conv_configs), start_layer=len(conv_configs), filename=cnn_param_file)
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
in_matrix = numpy.reshape(in_matrix, (in_matrix.shape[0],) + input_shape_1)
extract_function = model.build_extract_feat_function(layer_index)
resume_training = False
if os.path.exists(wdir + '/nnet.tmp_CNN') and os.path.exists(
wdir + '/training_state.tmp_CNN'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp_CNN')
log('> ... found nnet.tmp_CNN and training_state.tmp_CNN, now resume training from epoch '
+ str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# construct the cnn architecture
cnn = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load the pre-training networks, if any, for parameter initialization
if resume_training:
_file2nnet(cnn, filename=wdir + '/nnet.tmp_CNN')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = cnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetunning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_acc, train_error = train_sgd(train_fn, cfg)
log('> epoch %d, lrate %f, training accuracy %f' %
(cfg.lrate.epoch, cfg.lrate.get_rate(),
100 * numpy.mean(train_acc)) + '(%)')
# validation
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg = cPickle.load(open(nnet_cfg, 'r'))
model = None
log('> ... model type: %s' % cfg.model_type)
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'DNNV':
model = DNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNNV':
model = CNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load model parameters
_file2nnet(model.layers, filename=nnet_param)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... processing the data')
utt_number = 0
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
# in_matrix = numpy.reshape(in_matrix, (in_matrix.shape[0],) + input_shape_1)
out_matrix = extract_func(in_matrix)
current_nnet = wdir + 'nnet.ptr.current'
train_sets, train_xy, train_x, train_y = read_dataset(dataset, dataset_args)
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# construct the stacked denoising autoencoder class
srbm = SRBM(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes,
n_outs=n_outs, first_layer_gb = first_layer_gb)
if keep_layer_num > 0:
log('> ... initializing model from ' + current_nnet)
_file2nnet(srbm.sigmoid_layers, set_layer_num = keep_layer_num, filename = current_nnet, withfinal=False)
# PRETRAINING THE MODEL #
log('> ... getting the pretraining functions')
pretraining_fns = srbm.pretraining_functions(train_set_x=train_x,
batch_size=batch_size,
k = 1, weight_cost = 0.0002)
log('> ... pre-training the model')
start_time = time.clock()
## Pre-train layer-wise
for i in range(keep_layer_num, ptr_layer_number):
if (srbm.rbm_layers[i].is_gbrbm()):
pretrain_lr = gbrbm_learning_rate
else:
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file)
if resume_training:
_file2nnet(dnn.layers, filename = wdir + '/nnet.tmp')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) + '(%)')
# validation
adapt_nnet_spec = ''
for n in xrange(len(net_split) - 1):
adapt_nnet_spec += net_split[n] + ':'
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# setup up the model
dnn = CNN_SAT(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg_si=cfg_si,
cfg_adapt=cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
# _file2nnet(dnn.cnn_si.layers, filename = init_model_file)
_file2nnet(dnn.cnn_si.layers, filename='BKUP/nnet.param.si')
_file2nnet(dnn.dnn_adapt.layers, filename='BKUP/nnet.param.adapt')
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y),
batch_size=cfg_adapt.batch_size)
log('> ... learning the adaptation network')
cfg = cfg_adapt
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
# train_error = train_sgd_verbose(train_fn, cfg_si.train_sets, cfg_si.train_xy,
# adapt_nnet_spec = ''
# for n in xrange(len(net_split) - 1):
# adapt_nnet_spec += net_split[n] + ':'
# cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# setup up the model
dnn = DNN_SAT(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg_si=cfg_si,
cfg_adapt=cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
_file2nnet(dnn.dnn_si.layers, filename=init_model_file)
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y),
batch_size=cfg_adapt.batch_size)
log('> ... learning the adaptation network')
cfg = cfg_adapt
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd_verbose(train_fn, cfg_si.train_sets,
cfg_si.train_xy, cfg.batch_size,
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_file != '' and ptr_layer_number != 0) and (resume_training is False):
log('>... Loading pretrained data from '+str(ptr_file)+" Layer "+str(ptr_layer_number))
_file2nnet(dnn.layers, set_layer_num = ptr_layer_number, path = ptr_file)
if resume_training:
_file2nnet(dnn.layers, path = wdir + '/dnn.tmp')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
lowest_validation_error = None
fail_count = 0
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
stop_if_stop_is_requested()
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState()
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file)
if resume_training:
_file2nnet(dnn.layers, filename = wdir + '/nnet.tmp')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error, pred, labels = train_sgd(train_fn, cfg)
n_data_train = len(pred)
log("-->> Epoch %d, \n " % cfg.lrate.epoch)
lstm_cfg_file = arguments['lstm_cfg_file']
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file,'r'))
kaldiread = KaldiReadIn(in_scp_file)
extra_kaldiread = KaldiReadIn(extra_in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cfg.init_activation()
lstm = PhaseATTENDLSTM_Forward(numpy_rng=rng, lstm_layer_configs = cfg, n_ins = cfg.n_ins)
_file2nnet(layers = lstm.lstm_layers, set_layer_num = lstm.lstm_layer_num, filename=lstm_param_file)
out_function = lstm.build_out_function()
log('> ... setting up the DNN layers')
dnn = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=lstm.lstm_layers[-1].output)
_file2nnet(layers = dnn.layers, set_layer_num = len(dnn.layers)+lstm.lstm_layer_num, filename = lstm_param_file, start_layer = lstm.lstm_layer_num)
out_function2 = dnn.build_extract_feat_function(layer_index)
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
extra_uttid, extra_in_matrix = extra_kaldiread.read_next_utt()
if uttid == '':
break
cfg_dnn.n_ins = cfg.n_ins; cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes; cfg_dnn.n_outs = cfg.n_outs
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_dnn)
# now set up the RBM model with dnn as an argument
srbm = SRBM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn = dnn)
# get the pre-training function
log('> ... getting the pre-training functions')
pretraining_fns = srbm.pretraining_functions(train_set_x=cfg.train_x, batch_size=cfg.batch_size,
k = 1, weight_cost = 0.0002)
start_layer_index = 0
start_epoch_index = 0
if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
start_layer_index, start_epoch_index = read_two_integers(wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from layer #' + str(start_layer_index) + ' epoch #' + str(start_epoch_index))
_file2nnet(dnn.layers, filename = wdir + '/nnet.tmp')
log('> ... training the model')
log('> r_cost = reconstruction cost, fe_cost = (approximate) value of free energy function')
# pre-train layer-wise
for i in xrange(start_layer_index, cfg.ptr_layer_number):
if (srbm.rbm_layers[i].is_gbrbm()):
pretrain_lr = cfg.gbrbm_learning_rate
else:
pretrain_lr = cfg.learning_rate
# go through pretraining epochs
momentum = cfg.initial_momentum
for epoch in range(start_epoch_index, cfg.epochs):
# go through the training set
if (epoch == cfg.initial_momentum_epoch):
momentum = cfg.final_momentum
extra_kaldiread = KaldiReadIn(extra_in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2],
input_shape_train[3])
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2**30))
cfg.init_activation()
log('> ... setting up the CNN layers')
cnn = CNN_Forward(numpy_rng=rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(cnn.layers,
cnn.extra_layers,
set_layer_num=len(cnn.layers),
filename=cnn_param_file)
out_function = cnn.build_out_function()
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
extra_uttid, extra_in_matrix = extra_kaldiread.read_next_utt()
if uttid == '':
break
print uttid
print extra_uttid
in_matrix = numpy.reshape(in_matrix,
(in_matrix.shape[0], ) + input_shape_1)
out_matrix = out_function(feat=in_matrix, extra_feat=extra_in_matrix)
