def stop_if_stop_is_requested(): global init global stop_requested if init == False: init_signal_handler() if stop_requested: log("STOP REQUESTED") sys.exit(1) return
def init_signal_handler(): def handler(x,y): log("SIGUSR1 SIGNAL RECEIVED") global stop_requested stop_requested = True global init log("SETUP SIGNAL HANDLER SIGUSR1") signal.signal(signal.SIGUSR1, handler) init = True
def get_next_rate(self, current_error): log("> get_next_rate:"); log("PRE: current_error="+str(current_error)+" prev="+str( self.prev_error)+" lowest_error="+str( self.lowest_error)+ " epoch="+str(self.epoch)+" max epoch="+str(self.max_epoch)+" fails="+str(self.fails)+" max fail="+str(self.max_fail)) if self.epoch >= self.max_epoch: self.rate = 0.0 elif self.prev_error is not None: if current_error < self.prev_error * self.thres_inc: self.rate *= self.factor_inc elif current_error >= self.prev_error * self.thres_dec: self.rate *= self.factor_dec if self.lowest_error is None: self.lowest_error = current_error; self.fails = 0 else: if current_error >= self.lowest_error * self.thres_fail: self.fails += 1 if self.fails >= self.max_fail: self.rate = 0.0 else: if current_error < self.lowest_error * 0.995: self.lowest_error = current_error self.fails = 0 else: self.fails += 1 if self.fails >= self.max_fail: self.rate = 0.0 self.epoch += 1 self.prev_error = current_error log("POST: rate="+str(self.rate)+" current_error="+str(current_error)+" prev="+str( self.prev_error)+" lowest_error="+str( self.lowest_error)+ " epoch="+str(self.epoch)+" max epoch="+str(self.max_epoch)+" fails="+str(self.fails)+" max fail="+str(self.max_fail)) log("< get_next_rate") self.tRate.set_value(self.rate) return self.rate
def saveModel(dnn,cfg): log("> Start saveModel") # save the model and network configuration if cfg.param_output_file != '': _nnet2file(dnn.layers, path=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor) log('> ... the best PDNN model param so far is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the best PDNN model config so far is ' + cfg.cfg_output_file) # output the model into Kaldi-compatible format if cfg.kaldi_output_file != '': dnn.write_model_to_kaldi(cfg.kaldi_output_file) log('> ... the best Kaldi model so far is ' + cfg.kaldi_output_file) log("< End SaveModel")
def __init__(self,thres_fail = 1.00,max_fail=6,max_epoch=100,learning_rate=0.001, beta1=0.9,beta2=0.999, epsilon=1e-8,gamma=1-1e-8): log("Init Adam with thres_fail="+str(thres_fail)+" max_fail="+str(max_fail)+" max_epoch="+str(max_epoch)+" learning_rate="+str(learning_rate)+" beta1="+str(beta1)+" beta2="+str(beta2)+" epsilon="+str(epsilon)+" gamma="+str(gamma)) self.learning_rate = learning_rate self.beta1 = beta1 self.beta2 = beta2 self.epsilon = epsilon self.gamma = gamma self.max_fail = max_fail self.max_epoch = max_epoch self.thres_fail = thres_fail self.lowest_error = None self.epoch = 1 self.rate = 1 self.prev_error = None self.fails = 0 self.do_resume = False
def __init__(self, momentum = 0.5,lr_init = 0.08, thres_inc = 1.00, factor_inc = 1.05, thres_dec = 1.04, factor_dec = 0.7, thres_fail = 1.00, max_fail = 6, max_epoch = 100): log("Init LearningRate Adaptive with momentum="+str(momentum)+" lr_init="+str(lr_init)+" thres_inc="+str( thres_inc)+" factor_inc="+str(factor_inc)+" thres_dec="+str(thres_dec)+" factor_dec="+str(factor_dec)+" thres_fail="+str(thres_fail)+" max_fail="+str(max_fail)+" max_epoch="+str(max_epoch)); self.rate = lr_init self.thres_inc = thres_inc self.factor_inc = factor_inc self.thres_dec = thres_dec self.factor_dec = factor_dec self.thres_fail = thres_fail self.max_fail = max_fail self.max_epoch = max_epoch self.lowest_error = None self.epoch = 1 self.prev_error = None self.fails = 0 self.momentum=theano.shared(np.asarray(momentum, dtype=theano.config.floatX)) self.tRate = theano.shared(np.asarray(self.rate, dtype=theano.config.floatX))
valid_data_spec = arguments['valid_data'] conv_nnet_spec = arguments['conv_nnet_spec'] lstm_nnet_spec = arguments['lstm_nnet_spec'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # parse network configuration from arguments, and initialize data reading cfg = NetworkConfig() cfg.model_type = 'CLDNNV' cfg.parse_config_cldnn(arguments, nnet_spec, conv_nnet_spec, lstm_nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... building the model') # setup model dnn = CLDNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %0.4f ' % (cfg.lrate.epoch, numpy.mean(train_error)))
print "Error: the argument %s has to be specified" % (arg) exit(1) # mandatory arguments in_scp_file = arguments['in_scp_file'] out_ark_file = arguments['out_ark_file'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] layer_index = int(arguments['layer_index']) # load network configuration cfg = cPickle.load(open(nnet_cfg, 'r')) cfg.init_activation() # set up the model with model config log('> ... setting up the model and loading parameters') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) cfg = cPickle.load(open(nnet_cfg, 'r')) model = None log('> ... model type: %s' % cfg.model_type) if cfg.model_type == 'DNN': model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) elif cfg.model_type == 'CNN': model = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) elif cfg.model_type == 'DNNV': model = DNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) elif cfg.model_type == 'CNNV': model = CNNV(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # load model parameters
conv_configs = cfg.conv_layer_configs conv_layer_number = len(conv_configs) for i in xrange(conv_layer_number): conv_configs[i]['activation'] = cfg.conv_activation # whether to use the fast mode use_fast = cfg.use_fast if arguments.has_key('use_fast'): use_fast = string_2_bool(arguments['use_fast']) kaldiread = KaldiReadIn(in_scp_file) kaldiwrite = KaldiWriteOut(out_ark_file) log('> ... setting up the CNN convolution layers') input_shape_train = conv_configs[0]['input_shape'] input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3]) rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(rng.randint(2 ** 30)) cfg.init_activation() cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, conv_layer_configs = conv_configs, use_fast = use_fast) #cnn = CNNV(numpy_rng = rng, theano_rng=theano_rng, cfg=cfg) _file2nnet(cnn.conv_layers, set_layer_num = len(conv_configs), filename=cnn_param_file) out_function = cnn.build_out_function() #out_function = cnn.build_extract_feat_function(-1) #print cnn.conv_layers[1].filter_shape model = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=cnn.conv_layers[1].output)
# check the arguments arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['data', 'nnet_param', 'nnet_cfg', 'output_path', 'batch_size'] for arg in required_arguments: if (arg in arguments) == False: print("Error: the argument %s has to be specified" % (arg)); exit(1) # mandatory arguments data_spec = arguments['data'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] output_path = arguments['output_path'] batch_size = float(arguments['batch_size']) log("Dump original data") countItems,initialDim = dumpInput(output_path,perplexity,data_spec) log("Original data dumped. Items="+str(countItems)+" initialDim="+str(initialDim)) # load network configuration and set up the model log('> ... setting up the model and loading parameters') cfg = pickle.load(smart_open(nnet_cfg,'rb')) layerNr = cfg.totalNumerOfLayers() log('Total number of layers '+str(layerNr)) for i in range(0,layerNr): count = 0 log("Going to output layer="+str(i)) files = [] layer_index = i
# mandatory arguments in_scp_file = arguments['in_scp_file'] out_ark_file = arguments['out_ark_file'] lstm_param_file = arguments['lstm_param_file'] lstm_cfg_file = arguments['lstm_cfg_file'] layer_index = int(arguments['layer_index']) # network structure cfg = cPickle.load(open(lstm_cfg_file,'r')) cfg.init_activation() kaldiread = KaldiReadIn(in_scp_file) kaldiwrite = KaldiWriteOut(out_ark_file) log('> ... setting up the ATTEND LSTM layers') rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(rng.randint(2 ** 30)) lstm = ATTEND_LSTM(numpy_rng=rng, theano_rng=theano_rng, cfg = cfg) _file2nnet(layers = lstm.layers, set_layer_num = len(lstm.layers), filename=lstm_param_file) out_function = lstm.build_extract_feat_function() while True: uttid, in_matrix = kaldiread.read_next_utt() if uttid == '': break print 'in_matrix:'+str(in_matrix.shape) final_matrix = numpy.zeros((in_matrix.shape[0],cfg.n_outs), dtype=theano.config.floatX) remainder = in_matrix.shape[0]%cfg.batch_size for index in xrange(in_matrix.shape[0]/cfg.batch_size):
def dnn_run(arguments): required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] cfg = NetworkConfig() cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0 ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers, set_layer_num=ptr_layer_number, filename=ptr_file) if resume_training: _file2nnet(dnn.layers, filename=wdir + '/nnet.tmp') # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100 * numpy.mean(train_error)) + '(%)') # validation valid_error = validate_by_minibatch(valid_fn, cfg) log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100 * numpy.mean(valid_error)) + '(%)') cfg.lrate.get_next_rate(current_error=100 * numpy.mean(valid_error)) # output nnet parameters and lrate, for training resume if cfg.lrate.epoch % cfg.model_save_step == 0: _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp') _lrate2file(cfg.lrate, wdir + '/training_state.tmp') # save the model and network configuration if cfg.param_output_file != '': _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor=cfg.input_dropout_factor, factor=cfg.dropout_factor) log('> ... the final PDNN model parameter is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
input_shape_train = conv_configs[0]['input_shape'] input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3]) num_utt = len(feat_mats_np) feat_mats = [] for i in xrange(num_utt): feat_mats.append(numpy.reshape(feat_mats_np[i], (feat_rows[i],) + input_shape_1)) rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) cnn = CNN_Forward(numpy_rng = rng, theano_rng=theano_rng, conv_layer_configs = conv_configs, use_fast = use_fast) _file2cnn(cnn.conv_layers, filename=conv_net_file) out_function = cnn.build_out_function(feat_mats) log('> ... processing the data') kaldiOut = KaldiWriteOut(output_scp,output_ark) kaldiOut.open() for i in xrange(num_utt): feat_out = out_function(feat_mats[i]) kaldiOut.write(uttIDs[i], feat_out) kaldiOut.close() end_time = time.clock() print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))
import bloscpack as bp import pandas as pd from io_func.model_io import log from utils.stop_handler import stop_if_stop_is_requested; pred_file = sys.argv[1] if '.gz' in pred_file: pred_mat = pickle.load(gzip.open(pred_file, 'rb')) else: pred_mat = pickle.load(open(pred_file, 'rb')) l = sorted(glob.glob(sys.argv[2])) if len(l) == 0: log("ERROR in show_results. Test partitions is empty. Argument "+sys.argv[2]) subclassificationMapping = pd.read_csv("/ssd/subclassificationMapping",sep="=",names=["NUMBER","NAME"],index_col="NUMBER") print(subclassificationMapping.loc[121][0]) test_labels = bp.unpack_ndarray_file(l[0]+".labels") test_labels = test_labels.astype(numpy.int32) # Read the subclassifications assert l[0][-4:] == '.blp' , "Invalid extension "+l[0][-4:] fn = l[0][:-4]+".ignored.csv.gz" df = pd.read_csv(fn,sep=';',usecols=['SUBCLASSIFICATION'],dtype=numpy.int32) assert df.shape[0] == test_labels.shape[0],"Shapes not equal" # End read the subclassification
cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0 ptr_file = "" if arguments.has_key("ptr_file") and arguments.has_key("ptr_layer_number"): ptr_file = arguments["ptr_file"] ptr_layer_number = int(arguments["ptr_layer_number"]) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + "/nnet.tmp") and os.path.exists(wdir + "/training_state.tmp"): resume_training = True cfg.lrate = _file2lrate(wdir + "/training_state.tmp") log("> ... found nnet.tmp and training_state.tmp, now resume training from epoch " + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log("> ... initializing the model") # construct the cnn architecture cnn = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # load the pre-training networks, if any, for parameter initialization if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(cnn.layers, set_layer_num=ptr_layer_number, filename=ptr_file) if resume_training: _file2nnet(cnn.layers, filename=wdir + "/nnet.tmp") # get the training, validation and testing function for the model log("> ... getting the finetuning functions") train_fn, valid_fn = cnn.build_finetune_functions(
def parse_config_cnn(self, arguments, nnet_spec, conv_nnet_spec): self.parse_config_dnn(arguments, nnet_spec) # parse convolutional layer structure self.conv_layer_configs = parse_conv_spec(conv_nnet_spec, self.batch_size) # parse convolutional layer activation # parse activation function, including maxout if 'conv_activation' in arguments: self.conv_activation_text = arguments['conv_activation'] self.conv_activation = parse_activation(arguments['conv_activation']) # maxout not supported yet # whether we use the fast version of convolution if 'use_fast' in arguments: self.use_fast = string2bool(arguments['use_fast']) log(">> CNN Config parsed. Start Dump") log("batch_size="+str(self.batch_size)) log("momentum="+str(self.momentum )) log("lrate="+str(self.lrate )) log("activation="+str(self.activation)) log("activation_text="+str(self.activation_text)) log("do_maxout="+str(self.do_maxout)) log("pool_size="+str(self.pool_size)) log("do_dropout="+str(self.do_dropout)) log("dropout_factor="+str(self.dropout_factor)) log("input_dropout_factor="+str(self.input_dropout_factor)) log("max_col_norm"+str(self.max_col_norm)) log("l1_reg="+str(self.l1_reg)) log("l2_reg="+str(self.l2_reg)) log("n_ins="+str(self.n_ins)) log("hidden_layers_sizes="+str(self.hidden_layers_sizes)) log("n_outs="+str(self.n_outs)) log("non_updated_layers="+str(self.non_updated_layers)) log("ivec_n_ins="+str(self.ivec_n_ins)) log("ivec_hidden_layers_sizes="+str(self.ivec_hidden_layers_sizes)) log("ivec_n_outt="+str(self.ivec_n_outs)) log("conv_layer_configs="+str(self.conv_layer_configs)) log("conv_activation="+str(self.conv_activation)) log("conv_activation_tex="+str(self.conv_activation_text)) log("use_fast="+str(self.use_fast)) log("model_save_step="+str(self.model_save_step)) log("cfg_output_file="+str(self.cfg_output_file)) log("param_output_file="+str(self.param_output_file)) log("kaldi_output_file="+str(self.kaldi_output_file)) log("<< CNN Config parsed. End Dump")
def parse_config_dnn(self, arguments, nnet_spec): self.parse_config_common(arguments) # parse DNN network structure if 'interrupt_after_epoch' in arguments: self.interrupt_epoch = int(arguments['interrupt_after_epoch']) nnet_layers = nnet_spec.split(':') self.n_ins = int(nnet_layers[0]) self.hidden_layers_sizes = [int(nnet_layers[i]) for i in range(1, len(nnet_layers)-1)] self.n_outs = int(nnet_layers[-1]) log(">> DNN Config parsed. Start Dump") log("interrupt_epoch="+str(self.interrupt_epoch)) log("batch_size="+str(self.batch_size)) log("momentum="+str(self.momentum )) log("lrate="+str(self.lrate )) log("activation="+str(self.activation)) log("activation_text="+str(self.activation_text)) log("do_maxout="+str(self.do_maxout)) log("pool_size="+str(self.pool_size)) log("do_dropout="+str(self.do_dropout)) log("dropout_factor="+str(self.dropout_factor)) log("input_dropout_factor="+str(self.input_dropout_factor)) log("max_col_norm="+str(self.max_col_norm)) log("l1_reg="+str(self.l1_reg)) log("l2_reg="+str(self.l2_reg)) log("n_ins="+str(self.n_ins)) log("hidden_layers_sizes="+str(self.hidden_layers_sizes)) log("n_outs="+str(self.n_outs)) log("non_updated_layers="+str(self.non_updated_layers)) log("use_fast="+str(self.use_fast)) log("model_save_step="+str(self.model_save_step)) log("cfg_output_file="+str(self.cfg_output_file)) log("param_output_file="+str(self.param_output_file)) log("kaldi_output_file="+str(self.kaldi_output_file)) log("<< DNN Config parsed. End Dump")
# check the arguments arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) train_data_spec = arguments['train_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # numpy random generator numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... initializing the model') # parse network configuration from arguments, and initialize data reading cfg = SdAConfig() cfg.parse_config_common(arguments) cfg.init_data_reading(train_data_spec) # we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons: # first, we can use DNN's model reading and writing functions, instead of designing these functions for SdA specifically # second, DNN generates the inputs for the l+1-th autoencoder, with the first l layers have been pre-trained cfg_dnn = NetworkConfig() cfg_dnn.n_ins = cfg.n_ins; cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes; cfg_dnn.n_outs = cfg.n_outs dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_dnn) # now set up the SdA model with dnn as an argument sda = SdA(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn = dnn)
# parse network configuration from arguments, and initialize data reading cfg_si = NetworkConfig() cfg_si.parse_config_dnn(arguments, si_nnet_spec) cfg_si.init_data_reading(train_data_spec, valid_data_spec) # parse the structure of the i-vector network cfg_adapt = NetworkConfig() # net_split = adapt_nnet_spec.split(':') # adapt_nnet_spec = '' # for n in xrange(len(net_split) - 1): # adapt_nnet_spec += net_split[n] + ':' # cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0') cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... initializing the model') # setup up the model dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg_si=cfg_si, cfg_adapt=cfg_adapt) # read the initial DNN (the SI DNN which has been well trained) _file2nnet(dnn.dnn_si.layers, filename=init_model_file) # get the training and validation functions for adaptation network training dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet dnn.delta_params = dnn.dnn_adapt.delta_params log('> ... getting the finetuning functions for iVecNN') train_fn, valid_fn = dnn.build_finetune_functions( (cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y), batch_size=cfg_adapt.batch_size)
# parse network configuration from arguments, and initialize data reading cfg_si = NetworkConfig(); cfg_si.model_type = 'CNN' cfg_si.parse_config_cnn(arguments, '10:' + si_nnet_spec, si_conv_nnet_spec) cfg_si.init_data_reading(train_data_spec, valid_data_spec) # parse the structure of the i-vector network cfg_adapt = NetworkConfig() net_split = adapt_nnet_spec.split(':') adapt_nnet_spec = '' for n in range(len(net_split) - 1): adapt_nnet_spec += net_split[n] + ':' cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... initializing the model') # setup up the model dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt) # read the initial DNN (the SI DNN which has been well trained) # _file2nnet(dnn.cnn_si.layers, filename = init_model_file) _file2nnet(dnn.cnn_si.layers, filename = 'BKUP/nnet.param.si') _file2nnet(dnn.dnn_adapt.layers, filename = 'BKUP/nnet.param.adapt') # get the training and validation functions for adaptation network training dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet dnn.delta_params = dnn.dnn_adapt.delta_params log('> ... getting the finetuning functions for iVecNN') train_fn, valid_fn = dnn.build_finetune_functions( (cfg_si.train_x, cfg_si.train_y), (cfg_si.valid_x, cfg_si.valid_y), batch_size = cfg_adapt.batch_size)
def main(arg_elements): # check the arguments arguments = parse_arguments(arg_elements) required_arguments = [ 'data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index', 'batch_size' ] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) # mandatory arguments data_spec = arguments['data'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] output_file = arguments['output_file'] layer_index = int(arguments['layer_index']) batch_size = int(arguments['batch_size']) argmax = arguments.has_key('argmax') and string2bool(arguments['argmax']) # load network configuration and set up the model log('> ... setting up the model and loading parameters') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) cfg = cPickle.load(smart_open(nnet_cfg, 'r')) cfg.init_activation() model = None if cfg.model_type == 'DNN': model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) elif cfg.model_type == 'CNN': model = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg, testing=True) # load model parameters _file2nnet(model.layers, filename=nnet_param) # initialize data reading cfg.init_data_reading_test(data_spec) # get the function for feature extraction log('> ... getting the feat-extraction function') extract_func = model.build_extract_feat_function(layer_index) output_mats = [ ] # store the features for all the data in memory. TODO: output the features in a streaming mode log('> ... generating features from the specified layer') while (not cfg.test_sets.is_finish()): # loop over the data cfg.test_sets.load_next_partition(cfg.test_xy) batch_num = int( math.ceil(1.0 * cfg.test_sets.cur_frame_num / batch_size)) for batch_index in xrange(batch_num): # loop over mini-batches start_index = batch_index * batch_size end_index = min((batch_index + 1) * batch_size, cfg.test_sets.cur_frame_num ) # the residue may be smaller than a mini-batch output = extract_func( cfg.test_x.get_value()[start_index:end_index]) output_mats.append(output) output_mat = numpy.concatenate(output_mats) if argmax: output_mat = output_mat.argmax(axis=1) # output the feature representations using pickle f = smart_open(output_file, 'wb') cPickle.dump(output_mat, f, cPickle.HIGHEST_PROTOCOL) log('> ... the features are stored in ' + output_file)
# mandatory arguments in_scp_file = arguments['in_scp_file'] out_ark_file = arguments['out_ark_file'] lstm_param_file = arguments['lstm_param_file'] lstm_cfg_file = arguments['lstm_cfg_file'] layer_index = int(arguments['layer_index']) # network structure cfg = cPickle.load(open(lstm_cfg_file, 'r')) cfg.init_activation() kaldiread = KaldiReadIn(in_scp_file) kaldiwrite = KaldiWriteOut(out_ark_file) log('> ... setting up the LSTM layers') rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(rng.randint(2**30)) lstm = LSTMV(numpy_rng=rng, theano_rng=theano_rng, cfg=cfg) _file2nnet(layers=lstm.layers, set_layer_num=len(lstm.layers), filename=lstm_param_file) out_function = lstm.build_extract_feat_function() log('> ... processing the data') while True: uttid, in_matrix = kaldiread.read_next_utt() if uttid == '': break
# parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0 ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers,
# mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] lstm_nnet_spec = arguments['lstm_nnet_spec'] wdir = arguments['wdir'] # parse network configuration from arguments, and initialize data reading cfg = NetworkConfig();cfg.model_type = 'ATTEND_LSTM' cfg.parse_config_attend(arguments, nnet_spec, lstm_nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # setup model dnn = ATTEND_LSTM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) if numpy.isnan(numpy.mean(train_error)) == True: log('> NaN!!! %0.4f ' % (numpy.mean(train_error)))
do_dropout = True if arguments.has_key('input_dropout_factor'): input_dropout_factor = float(arguments['input_dropout_factor']) dropout_factor = [] factors = arguments['dropout_factor'].split(',') for n in xrange(len(factors)): dropout_factor.append(float(factors[n])) train_dataset, train_dataset_args = read_data_args(train_data_spec) valid_dataset, valid_dataset_args = read_data_args(valid_data_spec) train_sets, train_xy, train_x, train_y = read_dataset(train_dataset, train_dataset_args) valid_sets, valid_xy, valid_x, valid_y = read_dataset(valid_dataset, valid_dataset_args) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') if do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins, hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs, activation = activation, dropout_factor = dropout_factor, input_dropout_factor = input_dropout_factor, do_maxout = do_maxout, pool_size = pool_size, max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins, hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs, activation = activation, do_maxout = do_maxout, pool_size = pool_size, do_pnorm = do_pnorm, pnorm_order = pnorm_order, max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg) if ptr_layer_number > 0: _file2nnet(dnn.sigmoid_layers, set_layer_num = ptr_layer_number, filename = ptr_file, withfinal=False)
if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) # mandatory arguments in_scp_file = arguments['in_scp_file'] out_ark_file = arguments['out_ark_file'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] layer_index = int(arguments['layer_index']) # load network configuration cfg = cPickle.load(open(nnet_cfg,'r')) cfg.init_activation() # set up the model with model config log('> ... setting up the model and loading parameters') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) cfg = cPickle.load(open(nnet_cfg,'r')) model = None if cfg.model_type == 'DNN': model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) elif cfg.model_type == 'CNN': model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True) # load model parameters _file2nnet(model.layers, filename = nnet_param) # get the function for feature extraction log('> ... getting the feat-extraction function') extract_func = model.build_extract_feat_function(layer_index)
# mandatory arguments in_scp_file = arguments['in_scp_file'] out_ark_file = arguments['out_ark_file'] lstm_param_file = arguments['lstm_param_file'] lstm_cfg_file = arguments['lstm_cfg_file'] layer_index = int(arguments['layer_index']) # network structure cfg = cPickle.load(open(lstm_cfg_file, 'r')) cfg.init_activation() kaldiread = KaldiReadIn(in_scp_file) kaldiwrite = KaldiWriteOut(out_ark_file) log('> ... setting up the ATTEND LSTM layers') rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(rng.randint(2**30)) lstm = ATTEND_LSTM(numpy_rng=rng, theano_rng=theano_rng, cfg=cfg) _file2nnet(layers=lstm.layers, set_layer_num=len(lstm.layers), filename=lstm_param_file) out_function = lstm.build_extract_feat_function() while True: uttid, in_matrix = kaldiread.read_next_utt() if uttid == '': break print 'in_matrix:' + str(in_matrix.shape) final_matrix = numpy.zeros((in_matrix.shape[0], cfg.n_outs),
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index', 'batch_size'] for arg in required_arguments: if (arg in arguments) == False: print("Error: the argument %s has to be specified" % (arg)); exit(1) # mandatory arguments data_spec = arguments['data'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] output_file = arguments['output_file'] layer_index = int(arguments['layer_index']) batch_size = float(arguments['batch_size']) argmax = 'argmax' in arguments and string2bool(arguments['argmax']) log("Extracting in batches with size="+str(batch_size)) if batch_size == -1: log("Extracting all features per partition at once") # load network configuration and set up the model log('> ... setting up the model and loading parameters') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) cfg = pickle.load(smart_open(nnet_cfg,'rb')) cfg.init_activation() model = None if cfg.model_type == 'DNN': model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) elif cfg.model_type == 'CNN': model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True)
def handler(x,y): log("SIGUSR1 SIGNAL RECEIVED") global stop_requested stop_requested = True
# split shared_spec ans indiv_spec into individual task's networks nnet_spec_array, shared_layers_num = parse_nnet_spec_mtl(shared_spec, indiv_spec) if len(nnet_spec_array) != task_number: print "Error: #networks specified by --indiv-spec doesn't match #tasks"; exit(1) # parse network configuration from arguments, and initialize data reading for n in xrange(task_number): network_config = NetworkConfig() network_config.parse_config_dnn(arguments, nnet_spec_array[n]) network_config.init_data_reading(train_data_spec_array[n], valid_data_spec_array[n]) config_array.append(network_config) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) resume_training = False; resume_tasks = [] # if we are resuming training, then MLT only operates on the terminated tasks for n in xrange(task_number): log('> ... building the model for task %d' % (n)) cfg = config_array[n] # set up the model dnn_shared = None; shared_layers = [] if n > 0: dnn_shared = dnn_array[0]; shared_layers = [m for m in xrange(shared_layers_num)] print shared_layers dnn = DNN_MTL(task_id=n,numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn_shared = dnn_shared, shared_layers = shared_layers) # get the training, validation and testing function for the model log('> ... getting the finetuning functions for task %d' % (n)) train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) # add dnn and the functions to the list dnn_array.append(dnn) train_fn_array.append(train_fn); valid_fn_array.append(valid_fn)
activation = parse_activation(arguments['activation']) if arguments['activation'].startswith('maxout'): do_maxout = True pool_size = int(arguments['activation'].replace('maxout:','')) elif arguments['activation'].startswith('pnorm'): do_pnorm = True pool_size, pnorm_order = parse_two_integers(arguments['activation']) train_dataset, train_dataset_args = read_data_args(train_data_spec) valid_dataset, valid_dataset_args = read_data_args(valid_data_spec) train_sets, train_xy, train_x, train_y = read_dataset(train_dataset, train_dataset_args) valid_sets, valid_xy, valid_x, valid_y = read_dataset(valid_dataset, valid_dataset_args) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # doesn't deal with dropout dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins, hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs, activation = activation, do_maxout = do_maxout, pool_size = pool_size, do_pnorm = do_pnorm, pnorm_order = pnorm_order, max_col_norm = max_col_norm, l1_reg = l1_reg, l2_reg = l2_reg, ivec_dim = ivec_dim, ivec_layers_sizes = ivec_layers_sizes) # read the initial DNN _file2nnet(dnn.sigmoid_layers, filename = si_model_file) # get the training, validation and testing function for iVecNN dnn.params = dnn.ivec_params dnn.delta_params = dnn.ivec_delta_params log('> ... getting the finetuning functions for iVecNN') train_fn, valid_fn = dnn.build_finetune_functions(
arguments = parse_arguments(arg_elements) required_arguments = ['data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index', 'batch_size'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) # mandatory arguments data_spec = arguments['data'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] output_file = arguments['output_file'] layer_index = int(arguments['layer_index']) batch_size = float(arguments['batch_size']) # load network configuration and set up the model log('> ... setting up the model and loading parameters') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) cfg = cPickle.load(smart_open(nnet_cfg,'r')) cfg.init_activation() model = None if cfg.model_type == 'DNN': model = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) elif cfg.model_type == 'CNN': model = CNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, testing = True) # load model parameters _file2nnet(model.layers, filename = nnet_param) # initialize data reading cfg.init_data_reading_test(data_spec)
nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # parse network configuration from arguments, and initialize data reading cfg = NetworkConfig() cfg.model_type = 'CNN' cfg.parse_config_cnn(arguments, '10:' + nnet_spec, conv_nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp_CNN') and os.path.exists( wdir + '/training_state.tmp_CNN'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp_CNN') log('> ... found nnet.tmp_CNN and training_state.tmp_CNN, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... initializing the model') # construct the cnn architecture cnn = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # load the pre-training networks, if any, for parameter initialization if resume_training: _file2nnet(cnn, filename=wdir + '/nnet.tmp_CNN') # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = cnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size)
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) train_data_spec = arguments['train_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # numpy random generator numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... initializing the model') # parse network configuration from arguments, and initialize data reading cfg = RBMConfig() cfg.parse_config_common(arguments) cfg.init_data_reading(train_data_spec) # we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons: # first, we can use DNN's model reading and writing functions, instead of designing these functions for RBM specifically # second, DNN generates cfg_dnn = NetworkConfig() cfg_dnn.n_ins = cfg.n_ins cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes cfg_dnn.n_outs = cfg.n_outs dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn)
first_layer_gb = True if arguments.has_key('first_layer_type'): if arguments['first_layer_type'] == 'bb': first_layer_gb = False # if initialized with current keep_layer_num=0 current_nnet = wdir + 'nnet.ptr.current' train_sets, train_xy, train_x, train_y = read_dataset(dataset, dataset_args) # numpy random generator numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # construct the stacked denoising autoencoder class srbm = SRBM(numpy_rng=numpy_rng, theano_rng = theano_rng, n_ins=n_ins, hidden_layers_sizes=hidden_layers_sizes, n_outs=n_outs, first_layer_gb = first_layer_gb) if keep_layer_num > 0: log('> ... initializing model from ' + current_nnet) _file2nnet(srbm.sigmoid_layers, set_layer_num = keep_layer_num, filename = current_nnet, withfinal=False) # PRETRAINING THE MODEL # log('> ... getting the pretraining functions') pretraining_fns = srbm.pretraining_functions(train_set_x=train_x, batch_size=batch_size, k = 1, weight_cost = 0.0002)
cfg = cPickle.load(smart_open(cnn_cfg_file, 'r')) conv_configs = cfg.conv_layer_configs conv_layer_number = len(conv_configs) for i in xrange(conv_layer_number): conv_configs[i]['activation'] = cfg.conv_activation # whether to use the fast mode use_fast = cfg.use_fast if arguments.has_key('use_fast'): use_fast = string2bool(arguments['use_fast']) kaldiread = KaldiReadIn(in_scp_file) kaldiwrite = KaldiWriteOut(out_ark_file) log('> ... setting up the CNN convolution layers') input_shape_train = conv_configs[0]['input_shape'] input_shape_1 = (input_shape_train[1], input_shape_train[2], input_shape_train[3]) rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2**30)) cnn = CNN_Forward(numpy_rng=rng, theano_rng=theano_rng, conv_layer_configs=conv_configs, use_fast=use_fast) _file2nnet(cnn.conv_layers, set_layer_num=len(conv_configs), filename=cnn_param_file) out_function = cnn.build_out_function()
# mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] lstm_nnet_spec = arguments['lstm_nnet_spec'] wdir = arguments['wdir'] # parse network configuration from arguments, and initialize data reading cfg = NetworkConfig();cfg.model_type = 'ATTEND_LSTM' cfg.parse_config_attend(arguments, nnet_spec, lstm_nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # setup model dnn = ATTEND_LSTM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') min_verr = 100 while (cfg.lrate.get_rate() != 0): train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %0.4f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) ) valid_error = validate_by_minibatch(valid_fn, cfg)
cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0; ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file)
if cfg.param_output_file != '': _nnet2file(dnn.layers, path=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor) log('> ... the best PDNN model param so far is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the best PDNN model config so far is ' + cfg.cfg_output_file) # output the model into Kaldi-compatible format if cfg.kaldi_output_file != '': dnn.write_model_to_kaldi(cfg.kaldi_output_file) log('> ... the best Kaldi model so far is ' + cfg.kaldi_output_file) log("< End SaveModel") if __name__ == '__main__': stop_if_stop_is_requested() log('Run DNN') log('Arguments: '+str(sys.argv[1:])) log('Theano Config:') log(theano.config) # check the arguments arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if (arg in arguments) == False: print("Error: the argument %s has to be specified" % (arg)); exit(1) # mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec']
for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) train_data_spec = arguments['train_data'] wdir = arguments['wdir'] path = "/home/piero/Documents/Experiments/Real_Test/Spectral Coef/Noise vs BG_voice+Conversation vs Shout+Scream/fft coef/" os.chdir(path) filename = "rbm.cfg" train_data = "train.pickle.gz" test_data = "test.pickle.gz" batch_size = 128 log('> ... setting up the model and loading parameters') numpy_rng = np.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) cfg_dnn = cPickle.load(open(filename, 'r')) cfg_dnn.init_activation() model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn) # load model parameters _file2nnet(model.layers, filename=wdir + '/rbm.param') # initialize data reading cfg_dnn.init_data_reading_test(train_data_spec) # get the function for feature extraction log('> ... getting the feat-extraction function') extract_func = model.build_extract_feat_function(-1)
# mandatory arguments in_scp_file = arguments['in_scp_file'] out_ark_file = arguments['out_ark_file'] extra_in_scp_file = arguments['extra_in_scp_file'] lstm_param_file = arguments['lstm_param_file'] lstm_cfg_file = arguments['lstm_cfg_file'] layer_index = int(arguments['layer_index']) # network structure cfg = cPickle.load(open(lstm_cfg_file,'r')) kaldiread = KaldiReadIn(in_scp_file) extra_kaldiread = KaldiReadIn(extra_in_scp_file) kaldiwrite = KaldiWriteOut(out_ark_file) log('> ... setting up the ATTEND LSTM layers') rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(rng.randint(2 ** 30)) cfg.init_activation() lstm = PhaseATTENDLSTM_Forward(numpy_rng=rng, lstm_layer_configs = cfg, n_ins = cfg.n_ins) _file2nnet(layers = lstm.lstm_layers, set_layer_num = lstm.lstm_layer_num, filename=lstm_param_file) out_function = lstm.build_out_function() log('> ... setting up the DNN layers') dnn = DNNV(numpy_rng = rng, theano_rng = theano_rng, cfg = cfg, input=lstm.lstm_layers[-1].output) _file2nnet(layers = dnn.layers, set_layer_num = len(dnn.layers)+lstm.lstm_layer_num, filename = lstm_param_file, start_layer = lstm.lstm_layer_num) out_function2 = dnn.build_extract_feat_function(layer_index) log('> ... processing the data') while True:
# check the arguments arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) train_data_spec = arguments['train_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # numpy random generator numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... initializing the model') # parse network configuration from arguments, and initialize data reading cfg = RBMConfig() cfg.parse_config_common(arguments) cfg.init_data_reading(train_data_spec) # we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons: # first, we can use DNN's model reading and writing functions, instead of designing these functions for RBM specifically # second, DNN generates cfg_dnn = NetworkConfig() cfg_dnn.n_ins = cfg.n_ins; cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes; cfg_dnn.n_outs = cfg.n_outs dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_dnn) # now set up the RBM model with dnn as an argument srbm = SRBM(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn = dnn)
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) train_data_spec = arguments['train_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # numpy random generator numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... initializing the model') # parse network configuration from arguments, and initialize data reading cfg = SdAConfig() cfg.parse_config_common(arguments) cfg.init_data_reading(train_data_spec) # we also need to set up a DNN model, whose parameters are shared with RBM, for 2 reasons: # first, we can use DNN's model reading and writing functions, instead of designing these functions for SdA specifically # second, DNN generates the inputs for the l+1-th autoencoder, with the first l layers have been pre-trained cfg_dnn = NetworkConfig() cfg_dnn.n_ins = cfg.n_ins cfg_dnn.hidden_layers_sizes = cfg.hidden_layers_sizes cfg_dnn.n_outs = cfg.n_outs dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn)