def stop_if_stop_is_requested():
global init
global stop_requested
if init == False:
init_signal_handler()
if stop_requested:
log("STOP REQUESTED")
sys.exit(1)
return
def init_signal_handler():
def handler(x,y):
log("SIGUSR1 SIGNAL RECEIVED")
global stop_requested
stop_requested = True
global init
log("SETUP SIGNAL HANDLER SIGUSR1")
signal.signal(signal.SIGUSR1, handler)
init = True
def get_next_rate(self, current_error):
log("> get_next_rate:");
log("PRE: current_error="+str(current_error)+" prev="+str( self.prev_error)+" lowest_error="+str( self.lowest_error)+ " epoch="+str(self.epoch)+" max epoch="+str(self.max_epoch)+" fails="+str(self.fails)+" max fail="+str(self.max_fail))
if self.epoch >= self.max_epoch:
self.rate = 0.0
elif self.prev_error is not None:
if current_error < self.prev_error * self.thres_inc:
self.rate *= self.factor_inc
elif current_error >= self.prev_error * self.thres_dec:
self.rate *= self.factor_dec
if self.lowest_error is None:
self.lowest_error = current_error;
self.fails = 0
else:
if current_error >= self.lowest_error * self.thres_fail:
self.fails += 1
if self.fails >= self.max_fail:
self.rate = 0.0
else:
if current_error < self.lowest_error * 0.995:
self.lowest_error = current_error
self.fails = 0
else:
self.fails += 1
if self.fails >= self.max_fail:
self.rate = 0.0
self.epoch += 1
self.prev_error = current_error
log("POST: rate="+str(self.rate)+" current_error="+str(current_error)+" prev="+str( self.prev_error)+" lowest_error="+str( self.lowest_error)+ " epoch="+str(self.epoch)+" max epoch="+str(self.max_epoch)+" fails="+str(self.fails)+" max fail="+str(self.max_fail))
log("< get_next_rate")
self.tRate.set_value(self.rate)
return self.rate
def saveModel(dnn,cfg):
log("> Start saveModel")
# save the model and network configuration
if cfg.param_output_file != '':
_nnet2file(dnn.layers, path=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor)
log('> ... the best PDNN model param so far is ' + cfg.param_output_file)
if cfg.cfg_output_file != '':
_cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
log('> ... the best PDNN model config so far is ' + cfg.cfg_output_file)
# output the model into Kaldi-compatible format
if cfg.kaldi_output_file != '':
dnn.write_model_to_kaldi(cfg.kaldi_output_file)
log('> ... the best Kaldi model so far is ' + cfg.kaldi_output_file)
log("< End SaveModel")
def __init__(self,thres_fail = 1.00,max_fail=6,max_epoch=100,learning_rate=0.001, beta1=0.9,beta2=0.999, epsilon=1e-8,gamma=1-1e-8):
log("Init Adam with thres_fail="+str(thres_fail)+" max_fail="+str(max_fail)+" max_epoch="+str(max_epoch)+" learning_rate="+str(learning_rate)+" beta1="+str(beta1)+" beta2="+str(beta2)+" epsilon="+str(epsilon)+" gamma="+str(gamma))
self.learning_rate = learning_rate
self.beta1 = beta1
self.beta2 = beta2
self.epsilon = epsilon
self.gamma = gamma
self.max_fail = max_fail
self.max_epoch = max_epoch
self.thres_fail = thres_fail
self.lowest_error = None
self.epoch = 1
self.rate = 1
self.prev_error = None
self.fails = 0
self.do_resume = False
def __init__(self, momentum = 0.5,lr_init = 0.08,
thres_inc = 1.00, factor_inc = 1.05,
thres_dec = 1.04, factor_dec = 0.7,
thres_fail = 1.00, max_fail = 6,
max_epoch = 100):
log("Init LearningRate Adaptive with momentum="+str(momentum)+" lr_init="+str(lr_init)+" thres_inc="+str( thres_inc)+" factor_inc="+str(factor_inc)+" thres_dec="+str(thres_dec)+" factor_dec="+str(factor_dec)+" thres_fail="+str(thres_fail)+" max_fail="+str(max_fail)+" max_epoch="+str(max_epoch));
self.rate = lr_init
self.thres_inc = thres_inc
self.factor_inc = factor_inc
self.thres_dec = thres_dec
self.factor_dec = factor_dec
self.thres_fail = thres_fail
self.max_fail = max_fail
self.max_epoch = max_epoch
self.lowest_error = None
self.epoch = 1
self.prev_error = None
self.fails = 0
self.momentum=theano.shared(np.asarray(momentum, dtype=theano.config.floatX))
self.tRate = theano.shared(np.asarray(self.rate, dtype=theano.config.floatX))
valid_data_spec = arguments['valid_data']
conv_nnet_spec = arguments['conv_nnet_spec']
lstm_nnet_spec = arguments['lstm_nnet_spec']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
# parse network configuration from arguments, and initialize data reading
cfg = NetworkConfig()
cfg.model_type = 'CLDNNV'
cfg.parse_config_cldnn(arguments, nnet_spec, conv_nnet_spec,
lstm_nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... building the model')
# setup model
dnn = CLDNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log('> epoch %d, training error %0.4f ' %
(cfg.lrate.epoch, numpy.mean(train_error)))
print "Error: the argument %s has to be specified" % (arg)
exit(1)
# mandatory arguments
in_scp_file = arguments['in_scp_file']
out_ark_file = arguments['out_ark_file']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
layer_index = int(arguments['layer_index'])
# load network configuration
cfg = cPickle.load(open(nnet_cfg, 'r'))
cfg.init_activation()
# set up the model with model config
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg = cPickle.load(open(nnet_cfg, 'r'))
model = None
log('> ... model type: %s' % cfg.model_type)
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'DNNV':
model = DNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNNV':
model = CNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load model parameters
conv_configs = cfg.conv_layer_configs
conv_layer_number = len(conv_configs)
for i in xrange(conv_layer_number):
conv_configs[i]['activation'] = cfg.conv_activation
# whether to use the fast mode
use_fast = cfg.use_fast
if arguments.has_key('use_fast'):
use_fast = string_2_bool(arguments['use_fast'])
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3])
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cfg.init_activation()
cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, conv_layer_configs = conv_configs, use_fast = use_fast)
#cnn = CNNV(numpy_rng = rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(cnn.conv_layers, set_layer_num = len(conv_configs), filename=cnn_param_file)
out_function = cnn.build_out_function()
#out_function = cnn.build_extract_feat_function(-1)
#print cnn.conv_layers[1].filter_shape
model = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=cnn.conv_layers[1].output)
# check the arguments
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['data', 'nnet_param', 'nnet_cfg', 'output_path', 'batch_size']
for arg in required_arguments:
if (arg in arguments) == False:
print("Error: the argument %s has to be specified" % (arg)); exit(1)
# mandatory arguments
data_spec = arguments['data']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
output_path = arguments['output_path']
batch_size = float(arguments['batch_size'])
log("Dump original data")
countItems,initialDim = dumpInput(output_path,perplexity,data_spec)
log("Original data dumped. Items="+str(countItems)+" initialDim="+str(initialDim))
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
cfg = pickle.load(smart_open(nnet_cfg,'rb'))
layerNr = cfg.totalNumerOfLayers()
log('Total number of layers '+str(layerNr))
for i in range(0,layerNr):
count = 0
log("Going to output layer="+str(i))
files = []
layer_index = i
# mandatory arguments
in_scp_file = arguments['in_scp_file']
out_ark_file = arguments['out_ark_file']
lstm_param_file = arguments['lstm_param_file']
lstm_cfg_file = arguments['lstm_cfg_file']
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file,'r'))
cfg.init_activation()
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
lstm = ATTEND_LSTM(numpy_rng=rng, theano_rng=theano_rng, cfg = cfg)
_file2nnet(layers = lstm.layers, set_layer_num = len(lstm.layers), filename=lstm_param_file)
out_function = lstm.build_extract_feat_function()
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
print 'in_matrix:'+str(in_matrix.shape)
final_matrix = numpy.zeros((in_matrix.shape[0],cfg.n_outs), dtype=theano.config.floatX)
remainder = in_matrix.shape[0]%cfg.batch_size
for index in xrange(in_matrix.shape[0]/cfg.batch_size):
def dnn_run(arguments):
required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
train_data_spec = arguments['train_data']
valid_data_spec = arguments['valid_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
cfg = NetworkConfig()
cfg.parse_config_dnn(arguments, nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
# parse pre-training options
# pre-training files and layer number (how many layers are set to the pre-training parameters)
ptr_layer_number = 0
ptr_file = ''
if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
ptr_file = arguments['ptr_file']
ptr_layer_number = int(arguments['ptr_layer_number'])
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir +
'/nnet.tmp') and os.path.exists(wdir +
'/training_state.tmp'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch '
+ str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers,
set_layer_num=ptr_layer_number,
filename=ptr_file)
if resume_training:
_file2nnet(dnn.layers, filename=wdir + '/nnet.tmp')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
log('> epoch %d, training error %f ' %
(cfg.lrate.epoch, 100 * numpy.mean(train_error)) + '(%)')
# validation
valid_error = validate_by_minibatch(valid_fn, cfg)
log('> epoch %d, lrate %f, validation error %f ' %
(cfg.lrate.epoch, cfg.lrate.get_rate(),
100 * numpy.mean(valid_error)) + '(%)')
cfg.lrate.get_next_rate(current_error=100 * numpy.mean(valid_error))
# output nnet parameters and lrate, for training resume
if cfg.lrate.epoch % cfg.model_save_step == 0:
_nnet2file(dnn.layers, filename=wdir + '/nnet.tmp')
_lrate2file(cfg.lrate, wdir + '/training_state.tmp')
# save the model and network configuration
if cfg.param_output_file != '':
_nnet2file(dnn.layers,
filename=cfg.param_output_file,
input_factor=cfg.input_dropout_factor,
factor=cfg.dropout_factor)
log('> ... the final PDNN model parameter is ' + cfg.param_output_file)
if cfg.cfg_output_file != '':
_cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3])
num_utt = len(feat_mats_np)
feat_mats = []
for i in xrange(num_utt):
feat_mats.append(numpy.reshape(feat_mats_np[i], (feat_rows[i],) + input_shape_1))
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng,
conv_layer_configs = conv_configs, use_fast = use_fast)
_file2cnn(cnn.conv_layers, filename=conv_net_file)
out_function = cnn.build_out_function(feat_mats)
log('> ... processing the data')
kaldiOut = KaldiWriteOut(output_scp,output_ark)
kaldiOut.open()
for i in xrange(num_utt):
feat_out = out_function(feat_mats[i])
kaldiOut.write(uttIDs[i], feat_out)
kaldiOut.close()
end_time = time.clock()
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
import bloscpack as bp
import pandas as pd
from io_func.model_io import log
from utils.stop_handler import stop_if_stop_is_requested;
pred_file = sys.argv[1]
if '.gz' in pred_file:
pred_mat = pickle.load(gzip.open(pred_file, 'rb'))
else:
pred_mat = pickle.load(open(pred_file, 'rb'))
l = sorted(glob.glob(sys.argv[2]))
if len(l) == 0:
log("ERROR in show_results. Test partitions is empty. Argument "+sys.argv[2])
subclassificationMapping = pd.read_csv("/ssd/subclassificationMapping",sep="=",names=["NUMBER","NAME"],index_col="NUMBER")
print(subclassificationMapping.loc[121][0])
test_labels = bp.unpack_ndarray_file(l[0]+".labels")
test_labels = test_labels.astype(numpy.int32)
# Read the subclassifications
assert l[0][-4:] == '.blp' , "Invalid extension "+l[0][-4:]
fn = l[0][:-4]+".ignored.csv.gz"
df = pd.read_csv(fn,sep=';',usecols=['SUBCLASSIFICATION'],dtype=numpy.int32)
assert df.shape[0] == test_labels.shape[0],"Shapes not equal"
# End read the subclassification
cfg.init_data_reading(train_data_spec, valid_data_spec)
# parse pre-training options
# pre-training files and layer number (how many layers are set to the pre-training parameters)
ptr_layer_number = 0
ptr_file = ""
if arguments.has_key("ptr_file") and arguments.has_key("ptr_layer_number"):
ptr_file = arguments["ptr_file"]
ptr_layer_number = int(arguments["ptr_layer_number"])
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir + "/nnet.tmp") and os.path.exists(wdir + "/training_state.tmp"):
resume_training = True
cfg.lrate = _file2lrate(wdir + "/training_state.tmp")
log("> ... found nnet.tmp and training_state.tmp, now resume training from epoch " + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log("> ... initializing the model")
# construct the cnn architecture
cnn = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load the pre-training networks, if any, for parameter initialization
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(cnn.layers, set_layer_num=ptr_layer_number, filename=ptr_file)
if resume_training:
_file2nnet(cnn.layers, filename=wdir + "/nnet.tmp")
# get the training, validation and testing function for the model
log("> ... getting the finetuning functions")
train_fn, valid_fn = cnn.build_finetune_functions(
def parse_config_cnn(self, arguments, nnet_spec, conv_nnet_spec):
self.parse_config_dnn(arguments, nnet_spec)
# parse convolutional layer structure
self.conv_layer_configs = parse_conv_spec(conv_nnet_spec, self.batch_size)
# parse convolutional layer activation
# parse activation function, including maxout
if 'conv_activation' in arguments:
self.conv_activation_text = arguments['conv_activation']
self.conv_activation = parse_activation(arguments['conv_activation'])
# maxout not supported yet
# whether we use the fast version of convolution
if 'use_fast' in arguments:
self.use_fast = string2bool(arguments['use_fast'])
log(">> CNN Config parsed. Start Dump")
log("batch_size="+str(self.batch_size))
log("momentum="+str(self.momentum ))
log("lrate="+str(self.lrate ))
log("activation="+str(self.activation))
log("activation_text="+str(self.activation_text))
log("do_maxout="+str(self.do_maxout))
log("pool_size="+str(self.pool_size))
log("do_dropout="+str(self.do_dropout))
log("dropout_factor="+str(self.dropout_factor))
log("input_dropout_factor="+str(self.input_dropout_factor))
log("max_col_norm"+str(self.max_col_norm))
log("l1_reg="+str(self.l1_reg))
log("l2_reg="+str(self.l2_reg))
log("n_ins="+str(self.n_ins))
log("hidden_layers_sizes="+str(self.hidden_layers_sizes))
log("n_outs="+str(self.n_outs))
log("non_updated_layers="+str(self.non_updated_layers))
log("ivec_n_ins="+str(self.ivec_n_ins))
log("ivec_hidden_layers_sizes="+str(self.ivec_hidden_layers_sizes))
log("ivec_n_outt="+str(self.ivec_n_outs))
log("conv_layer_configs="+str(self.conv_layer_configs))
log("conv_activation="+str(self.conv_activation))
log("conv_activation_tex="+str(self.conv_activation_text))
log("use_fast="+str(self.use_fast))
log("model_save_step="+str(self.model_save_step))
log("cfg_output_file="+str(self.cfg_output_file))
log("param_output_file="+str(self.param_output_file))
log("kaldi_output_file="+str(self.kaldi_output_file))
log("<< CNN Config parsed. End Dump")
def parse_config_dnn(self, arguments, nnet_spec):
self.parse_config_common(arguments)
# parse DNN network structure
if 'interrupt_after_epoch' in arguments:
self.interrupt_epoch = int(arguments['interrupt_after_epoch'])
nnet_layers = nnet_spec.split(':')
self.n_ins = int(nnet_layers[0])
self.hidden_layers_sizes = [int(nnet_layers[i]) for i in range(1, len(nnet_layers)-1)]
self.n_outs = int(nnet_layers[-1])
log(">> DNN Config parsed. Start Dump")
log("interrupt_epoch="+str(self.interrupt_epoch))
log("batch_size="+str(self.batch_size))
log("momentum="+str(self.momentum ))
log("lrate="+str(self.lrate ))
log("activation="+str(self.activation))
log("activation_text="+str(self.activation_text))
log("do_maxout="+str(self.do_maxout))
log("pool_size="+str(self.pool_size))
log("do_dropout="+str(self.do_dropout))
log("dropout_factor="+str(self.dropout_factor))
log("input_dropout_factor="+str(self.input_dropout_factor))
log("max_col_norm="+str(self.max_col_norm))
log("l1_reg="+str(self.l1_reg))
log("l2_reg="+str(self.l2_reg))
log("n_ins="+str(self.n_ins))
log("hidden_layers_sizes="+str(self.hidden_layers_sizes))
log("n_outs="+str(self.n_outs))
log("non_updated_layers="+str(self.non_updated_layers))
log("use_fast="+str(self.use_fast))
log("model_save_step="+str(self.model_save_step))
log("cfg_output_file="+str(self.cfg_output_file))
log("param_output_file="+str(self.param_output_file))
log("kaldi_output_file="+str(self.kaldi_output_file))
log("<< DNN Config parsed. End Dump")
# check the arguments
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['train_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg); exit(1)
train_data_spec = arguments['train_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... initializing the model')
# parse network configuration from arguments, and initialize data reading
cfg = SdAConfig()
cfg.parse_config_common(arguments)
cfg.init_data_reading(train_data_spec)
# we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons:
# first, we can use DNN's model reading and writing functions, instead of designing these functions for SdA specifically
# second, DNN generates the inputs for the l+1-th autoencoder, with the first l layers have been pre-trained
cfg_dnn = NetworkConfig()
cfg_dnn.n_ins = cfg.n_ins; cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes; cfg_dnn.n_outs = cfg.n_outs
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_dnn)
# now set up the SdA model with dnn as an argument
sda = SdA(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn = dnn)
# parse network configuration from arguments, and initialize data reading
cfg_si = NetworkConfig()
cfg_si.parse_config_dnn(arguments, si_nnet_spec)
cfg_si.init_data_reading(train_data_spec, valid_data_spec)
# parse the structure of the i-vector network
cfg_adapt = NetworkConfig()
# net_split = adapt_nnet_spec.split(':')
# adapt_nnet_spec = ''
# for n in xrange(len(net_split) - 1):
# adapt_nnet_spec += net_split[n] + ':'
# cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# setup up the model
dnn = DNN_SAT(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg_si=cfg_si,
cfg_adapt=cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
_file2nnet(dnn.dnn_si.layers, filename=init_model_file)
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y),
batch_size=cfg_adapt.batch_size)
# parse network configuration from arguments, and initialize data reading
cfg_si = NetworkConfig(); cfg_si.model_type = 'CNN'
cfg_si.parse_config_cnn(arguments, '10:' + si_nnet_spec, si_conv_nnet_spec)
cfg_si.init_data_reading(train_data_spec, valid_data_spec)
# parse the structure of the i-vector network
cfg_adapt = NetworkConfig()
net_split = adapt_nnet_spec.split(':')
adapt_nnet_spec = ''
for n in range(len(net_split) - 1):
adapt_nnet_spec += net_split[n] + ':'
cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... initializing the model')
# setup up the model
dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt)
# read the initial DNN (the SI DNN which has been well trained)
# _file2nnet(dnn.cnn_si.layers, filename = init_model_file)
_file2nnet(dnn.cnn_si.layers, filename = 'BKUP/nnet.param.si')
_file2nnet(dnn.dnn_adapt.layers, filename = 'BKUP/nnet.param.adapt')
# get the training and validation functions for adaptation network training
dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet
dnn.delta_params = dnn.dnn_adapt.delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y),
batch_size = cfg_adapt.batch_size)
def main(arg_elements):
# check the arguments
arguments = parse_arguments(arg_elements)
required_arguments = [
'data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index',
'batch_size'
]
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
# mandatory arguments
data_spec = arguments['data']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
output_file = arguments['output_file']
layer_index = int(arguments['layer_index'])
batch_size = int(arguments['batch_size'])
argmax = arguments.has_key('argmax') and string2bool(arguments['argmax'])
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg = cPickle.load(smart_open(nnet_cfg, 'r'))
cfg.init_activation()
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg=cfg,
testing=True)
# load model parameters
_file2nnet(model.layers, filename=nnet_param)
# initialize data reading
cfg.init_data_reading_test(data_spec)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
output_mats = [
] # store the features for all the data in memory. TODO: output the features in a streaming mode
log('> ... generating features from the specified layer')
while (not cfg.test_sets.is_finish()): # loop over the data
cfg.test_sets.load_next_partition(cfg.test_xy)
batch_num = int(
math.ceil(1.0 * cfg.test_sets.cur_frame_num / batch_size))
for batch_index in xrange(batch_num): # loop over mini-batches
start_index = batch_index * batch_size
end_index = min((batch_index + 1) * batch_size,
cfg.test_sets.cur_frame_num
) # the residue may be smaller than a mini-batch
output = extract_func(
cfg.test_x.get_value()[start_index:end_index])
output_mats.append(output)
output_mat = numpy.concatenate(output_mats)
if argmax:
output_mat = output_mat.argmax(axis=1)
# output the feature representations using pickle
f = smart_open(output_file, 'wb')
cPickle.dump(output_mat, f, cPickle.HIGHEST_PROTOCOL)
log('> ... the features are stored in ' + output_file)
# mandatory arguments
in_scp_file = arguments['in_scp_file']
out_ark_file = arguments['out_ark_file']
lstm_param_file = arguments['lstm_param_file']
lstm_cfg_file = arguments['lstm_cfg_file']
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file, 'r'))
cfg.init_activation()
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2**30))
lstm = LSTMV(numpy_rng=rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(layers=lstm.layers,
set_layer_num=len(lstm.layers),
filename=lstm_param_file)
out_function = lstm.build_extract_feat_function()
log('> ... processing the data')
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
# parse pre-training options
# pre-training files and layer number (how many layers are set to the pre-training parameters)
ptr_layer_number = 0
ptr_file = ''
if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
ptr_file = arguments['ptr_file']
ptr_layer_number = int(arguments['ptr_layer_number'])
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir +
'/nnet.tmp') and os.path.exists(wdir +
'/training_state.tmp'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch '
+ str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers,
# mandatory arguments
train_data_spec = arguments['train_data']
valid_data_spec = arguments['valid_data']
nnet_spec = arguments['nnet_spec']
lstm_nnet_spec = arguments['lstm_nnet_spec']
wdir = arguments['wdir']
# parse network configuration from arguments, and initialize data reading
cfg = NetworkConfig();cfg.model_type = 'ATTEND_LSTM'
cfg.parse_config_attend(arguments, nnet_spec, lstm_nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
dnn = ATTEND_LSTM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
while (cfg.lrate.get_rate() != 0):
# one epoch of sgd training
train_error = train_sgd(train_fn, cfg)
if numpy.isnan(numpy.mean(train_error)) == True:
log('> NaN!!! %0.4f ' % (numpy.mean(train_error)))
do_dropout = True
if arguments.has_key('input_dropout_factor'):
input_dropout_factor = float(arguments['input_dropout_factor'])
dropout_factor = []
factors = arguments['dropout_factor'].split(',')
for n in xrange(len(factors)):
dropout_factor.append(float(factors[n]))
train_dataset, train_dataset_args = read_data_args(train_data_spec)
valid_dataset, valid_dataset_args = read_data_args(valid_data_spec)
train_sets, train_xy, train_x, train_y = read_dataset(train_dataset, train_dataset_args)
valid_sets, valid_xy, valid_x, valid_y = read_dataset(valid_dataset, valid_dataset_args)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
if do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs,
activation = activation, dropout_factor = dropout_factor, input_dropout_factor = input_dropout_factor,
do_maxout = do_maxout, pool_size = pool_size,
max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs,
activation = activation, do_maxout = do_maxout, pool_size = pool_size,
do_pnorm = do_pnorm, pnorm_order = pnorm_order,
max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg)
if ptr_layer_number > 0:
_file2nnet(dnn.sigmoid_layers, set_layer_num = ptr_layer_number, filename = ptr_file, withfinal=False)
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg); exit(1)
# mandatory arguments
in_scp_file = arguments['in_scp_file']
out_ark_file = arguments['out_ark_file']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
layer_index = int(arguments['layer_index'])
# load network configuration
cfg = cPickle.load(open(nnet_cfg,'r'))
cfg.init_activation()
# set up the model with model config
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
cfg = cPickle.load(open(nnet_cfg,'r'))
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
# load model parameters
_file2nnet(model.layers, filename = nnet_param)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(layer_index)
# mandatory arguments
in_scp_file = arguments['in_scp_file']
out_ark_file = arguments['out_ark_file']
lstm_param_file = arguments['lstm_param_file']
lstm_cfg_file = arguments['lstm_cfg_file']
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file, 'r'))
cfg.init_activation()
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2**30))
lstm = ATTEND_LSTM(numpy_rng=rng, theano_rng=theano_rng, cfg=cfg)
_file2nnet(layers=lstm.layers,
set_layer_num=len(lstm.layers),
filename=lstm_param_file)
out_function = lstm.build_extract_feat_function()
while True:
uttid, in_matrix = kaldiread.read_next_utt()
if uttid == '':
break
print 'in_matrix:' + str(in_matrix.shape)
final_matrix = numpy.zeros((in_matrix.shape[0], cfg.n_outs),
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index', 'batch_size']
for arg in required_arguments:
if (arg in arguments) == False:
print("Error: the argument %s has to be specified" % (arg)); exit(1)
# mandatory arguments
data_spec = arguments['data']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
output_file = arguments['output_file']
layer_index = int(arguments['layer_index'])
batch_size = float(arguments['batch_size'])
argmax = 'argmax' in arguments and string2bool(arguments['argmax'])
log("Extracting in batches with size="+str(batch_size))
if batch_size == -1:
log("Extracting all features per partition at once")
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
cfg = pickle.load(smart_open(nnet_cfg,'rb'))
cfg.init_activation()
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
def handler(x,y):
log("SIGUSR1 SIGNAL RECEIVED")
global stop_requested
stop_requested = True
# split shared_spec ans indiv_spec into individual task's networks
nnet_spec_array, shared_layers_num = parse_nnet_spec_mtl(shared_spec, indiv_spec)
if len(nnet_spec_array) != task_number:
print "Error: #networks specified by --indiv-spec doesn't match #tasks"; exit(1)
# parse network configuration from arguments, and initialize data reading
for n in xrange(task_number):
network_config = NetworkConfig()
network_config.parse_config_dnn(arguments, nnet_spec_array[n])
network_config.init_data_reading(train_data_spec_array[n], valid_data_spec_array[n])
config_array.append(network_config)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
resume_training = False; resume_tasks = [] # if we are resuming training, then MLT only operates on the terminated tasks
for n in xrange(task_number):
log('> ... building the model for task %d' % (n))
cfg = config_array[n]
# set up the model
dnn_shared = None; shared_layers = []
if n > 0:
dnn_shared = dnn_array[0]; shared_layers = [m for m in xrange(shared_layers_num)]
print shared_layers
dnn = DNN_MTL(task_id=n,numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg,
dnn_shared = dnn_shared, shared_layers = shared_layers)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions for task %d' % (n))
train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size)
# add dnn and the functions to the list
dnn_array.append(dnn)
train_fn_array.append(train_fn); valid_fn_array.append(valid_fn)
activation = parse_activation(arguments['activation'])
if arguments['activation'].startswith('maxout'):
do_maxout = True
pool_size = int(arguments['activation'].replace('maxout:',''))
elif arguments['activation'].startswith('pnorm'):
do_pnorm = True
pool_size, pnorm_order = parse_two_integers(arguments['activation'])
train_dataset, train_dataset_args = read_data_args(train_data_spec)
valid_dataset, valid_dataset_args = read_data_args(valid_data_spec)
train_sets, train_xy, train_x, train_y = read_dataset(train_dataset, train_dataset_args)
valid_sets, valid_xy, valid_x, valid_y = read_dataset(valid_dataset, valid_dataset_args)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# doesn't deal with dropout
dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs,
activation = activation, do_maxout = do_maxout, pool_size = pool_size,
do_pnorm = do_pnorm, pnorm_order = pnorm_order,
max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg,
ivec_dim = ivec_dim, ivec_layers_sizes = ivec_layers_sizes)
# read the initial DNN
_file2nnet(dnn.sigmoid_layers, filename = si_model_file)
# get the training, validation and testing function for iVecNN
dnn.params = dnn.ivec_params
dnn.delta_params = dnn.ivec_delta_params
log('> ... getting the finetuning functions for iVecNN')
train_fn, valid_fn = dnn.build_finetune_functions(
arguments = parse_arguments(arg_elements)
required_arguments = ['data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index', 'batch_size']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg); exit(1)
# mandatory arguments
data_spec = arguments['data']
nnet_param = arguments['nnet_param']
nnet_cfg = arguments['nnet_cfg']
output_file = arguments['output_file']
layer_index = int(arguments['layer_index'])
batch_size = float(arguments['batch_size'])
# load network configuration and set up the model
log('> ... setting up the model and loading parameters')
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
cfg = cPickle.load(smart_open(nnet_cfg,'r'))
cfg.init_activation()
model = None
if cfg.model_type == 'DNN':
model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
elif cfg.model_type == 'CNN':
model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
# load model parameters
_file2nnet(model.layers, filename = nnet_param)
# initialize data reading
cfg.init_data_reading_test(data_spec)
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
# parse network configuration from arguments, and initialize data reading
cfg = NetworkConfig()
cfg.model_type = 'CNN'
cfg.parse_config_cnn(arguments, '10:' + nnet_spec, conv_nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir + '/nnet.tmp_CNN') and os.path.exists(
wdir + '/training_state.tmp_CNN'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp_CNN')
log('> ... found nnet.tmp_CNN and training_state.tmp_CNN, now resume training from epoch '
+ str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# construct the cnn architecture
cnn = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg)
# load the pre-training networks, if any, for parameter initialization
if resume_training:
_file2nnet(cnn, filename=wdir + '/nnet.tmp_CNN')
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = cnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['train_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
train_data_spec = arguments['train_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# parse network configuration from arguments, and initialize data reading
cfg = RBMConfig()
cfg.parse_config_common(arguments)
cfg.init_data_reading(train_data_spec)
# we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons:
# first, we can use DNN's model reading and writing functions, instead of designing these functions for RBM specifically
# second, DNN generates
cfg_dnn = NetworkConfig()
cfg_dnn.n_ins = cfg.n_ins
cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes
cfg_dnn.n_outs = cfg.n_outs
dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn)
first_layer_gb = True
if arguments.has_key('first_layer_type'):
if arguments['first_layer_type'] == 'bb':
first_layer_gb = False
# if initialized with current
keep_layer_num=0
current_nnet = wdir + 'nnet.ptr.current'
train_sets, train_xy, train_x, train_y = read_dataset(dataset, dataset_args)
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# construct the stacked denoising autoencoder class
srbm = SRBM(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins,
hidden_layers_sizes=hidden_layers_sizes,
n_outs=n_outs, first_layer_gb = first_layer_gb)
if keep_layer_num > 0:
log('> ... initializing model from ' + current_nnet)
_file2nnet(srbm.sigmoid_layers, set_layer_num = keep_layer_num, filename = current_nnet, withfinal=False)
# PRETRAINING THE MODEL #
log('> ... getting the pretraining functions')
pretraining_fns = srbm.pretraining_functions(train_set_x=train_x,
batch_size=batch_size,
k = 1, weight_cost = 0.0002)
cfg = cPickle.load(smart_open(cnn_cfg_file, 'r'))
conv_configs = cfg.conv_layer_configs
conv_layer_number = len(conv_configs)
for i in xrange(conv_layer_number):
conv_configs[i]['activation'] = cfg.conv_activation
# whether to use the fast mode
use_fast = cfg.use_fast
if arguments.has_key('use_fast'):
use_fast = string2bool(arguments['use_fast'])
kaldiread = KaldiReadIn(in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the CNN convolution layers')
input_shape_train = conv_configs[0]['input_shape']
input_shape_1 = (input_shape_train[1], input_shape_train[2],
input_shape_train[3])
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
cnn = CNN_Forward(numpy_rng=rng,
theano_rng=theano_rng,
conv_layer_configs=conv_configs,
use_fast=use_fast)
_file2nnet(cnn.conv_layers,
set_layer_num=len(conv_configs),
filename=cnn_param_file)
out_function = cnn.build_out_function()
# mandatory arguments
train_data_spec = arguments['train_data']
valid_data_spec = arguments['valid_data']
nnet_spec = arguments['nnet_spec']
lstm_nnet_spec = arguments['lstm_nnet_spec']
wdir = arguments['wdir']
# parse network configuration from arguments, and initialize data reading
cfg = NetworkConfig();cfg.model_type = 'ATTEND_LSTM'
cfg.parse_config_attend(arguments, nnet_spec, lstm_nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
dnn = ATTEND_LSTM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# get the training, validation and testing function for the model
log('> ... getting the finetuning functions')
train_fn, valid_fn = dnn.build_finetune_functions(
(cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y),
batch_size=cfg.batch_size)
log('> ... finetuning the model')
min_verr = 100
while (cfg.lrate.get_rate() != 0):
train_error = train_sgd(train_fn, cfg)
log('> epoch %d, training error %0.4f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) )
valid_error = validate_by_minibatch(valid_fn, cfg)
cfg.parse_config_dnn(arguments, nnet_spec)
cfg.init_data_reading(train_data_spec, valid_data_spec)
# parse pre-training options
# pre-training files and layer number (how many layers are set to the pre-training parameters)
ptr_layer_number = 0; ptr_file = ''
if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'):
ptr_file = arguments['ptr_file']
ptr_layer_number = int(arguments['ptr_layer_number'])
# check working dir to see whether it's resuming training
resume_training = False
if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'):
resume_training = True
cfg.lrate = _file2lrate(wdir + '/training_state.tmp')
log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... building the model')
# setup model
if cfg.do_dropout:
dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
else:
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg)
# initialize model parameters
# if not resuming training, initialized from the specified pre-training file
# if resuming training, initialized from the tmp model file
if (ptr_layer_number > 0) and (resume_training is False):
_file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file)
if cfg.param_output_file != '':
_nnet2file(dnn.layers, path=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor)
log('> ... the best PDNN model param so far is ' + cfg.param_output_file)
if cfg.cfg_output_file != '':
_cfg2file(dnn.cfg, filename=cfg.cfg_output_file)
log('> ... the best PDNN model config so far is ' + cfg.cfg_output_file)
# output the model into Kaldi-compatible format
if cfg.kaldi_output_file != '':
dnn.write_model_to_kaldi(cfg.kaldi_output_file)
log('> ... the best Kaldi model so far is ' + cfg.kaldi_output_file)
log("< End SaveModel")
if __name__ == '__main__':
stop_if_stop_is_requested()
log('Run DNN')
log('Arguments: '+str(sys.argv[1:]))
log('Theano Config:')
log(theano.config)
# check the arguments
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if (arg in arguments) == False:
print("Error: the argument %s has to be specified" % (arg)); exit(1)
# mandatory arguments
train_data_spec = arguments['train_data']
valid_data_spec = arguments['valid_data']
nnet_spec = arguments['nnet_spec']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
train_data_spec = arguments['train_data']
wdir = arguments['wdir']
path = "/home/piero/Documents/Experiments/Real_Test/Spectral Coef/Noise vs BG_voice+Conversation vs Shout+Scream/fft coef/"
os.chdir(path)
filename = "rbm.cfg"
train_data = "train.pickle.gz"
test_data = "test.pickle.gz"
batch_size = 128
log('> ... setting up the model and loading parameters')
numpy_rng = np.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
cfg_dnn = cPickle.load(open(filename, 'r'))
cfg_dnn.init_activation()
model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn)
# load model parameters
_file2nnet(model.layers, filename=wdir + '/rbm.param')
# initialize data reading
cfg_dnn.init_data_reading_test(train_data_spec)
# get the function for feature extraction
log('> ... getting the feat-extraction function')
extract_func = model.build_extract_feat_function(-1)
# mandatory arguments
in_scp_file = arguments['in_scp_file']
out_ark_file = arguments['out_ark_file']
extra_in_scp_file = arguments['extra_in_scp_file']
lstm_param_file = arguments['lstm_param_file']
lstm_cfg_file = arguments['lstm_cfg_file']
layer_index = int(arguments['layer_index'])
# network structure
cfg = cPickle.load(open(lstm_cfg_file,'r'))
kaldiread = KaldiReadIn(in_scp_file)
extra_kaldiread = KaldiReadIn(extra_in_scp_file)
kaldiwrite = KaldiWriteOut(out_ark_file)
log('> ... setting up the ATTEND LSTM layers')
rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(rng.randint(2 ** 30))
cfg.init_activation()
lstm = PhaseATTENDLSTM_Forward(numpy_rng=rng, lstm_layer_configs = cfg, n_ins = cfg.n_ins)
_file2nnet(layers = lstm.lstm_layers, set_layer_num = lstm.lstm_layer_num, filename=lstm_param_file)
out_function = lstm.build_out_function()
log('> ... setting up the DNN layers')
dnn = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=lstm.lstm_layers[-1].output)
_file2nnet(layers = dnn.layers, set_layer_num = len(dnn.layers)+lstm.lstm_layer_num, filename = lstm_param_file, start_layer = lstm.lstm_layer_num)
out_function2 = dnn.build_extract_feat_function(layer_index)
log('> ... processing the data')
while True:
# check the arguments
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['train_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg); exit(1)
train_data_spec = arguments['train_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
log('> ... initializing the model')
# parse network configuration from arguments, and initialize data reading
cfg = RBMConfig()
cfg.parse_config_common(arguments)
cfg.init_data_reading(train_data_spec)
# we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons:
# first, we can use DNN's model reading and writing functions, instead of designing these functions for RBM specifically
# second, DNN generates
cfg_dnn = NetworkConfig()
cfg_dnn.n_ins = cfg.n_ins; cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes; cfg_dnn.n_outs = cfg.n_outs
dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_dnn)
# now set up the RBM model with dnn as an argument
srbm = SRBM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn = dnn)
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))]
arguments = parse_arguments(arg_elements)
required_arguments = ['train_data', 'nnet_spec', 'wdir']
for arg in required_arguments:
if arguments.has_key(arg) == False:
print "Error: the argument %s has to be specified" % (arg)
exit(1)
train_data_spec = arguments['train_data']
nnet_spec = arguments['nnet_spec']
wdir = arguments['wdir']
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
theano_rng = RandomStreams(numpy_rng.randint(2**30))
log('> ... initializing the model')
# parse network configuration from arguments, and initialize data reading
cfg = SdAConfig()
cfg.parse_config_common(arguments)
cfg.init_data_reading(train_data_spec)
# we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons:
# first, we can use DNN's model reading and writing functions, instead of designing these functions for SdA specifically
# second, DNN generates the inputs for the l+1-th autoencoder, with the first l layers have been pre-trained
cfg_dnn = NetworkConfig()
cfg_dnn.n_ins = cfg.n_ins
cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes
cfg_dnn.n_outs = cfg.n_outs
dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn)
