adapt_nnet_spec = arguments['adapt_nnet_spec']; wdir = arguments['wdir'] init_model_file = arguments['init_model'] # parse network configuration from arguments, and initialize data reading cfg_si = NetworkConfig(); cfg_si.model_type = 'CNN' cfg_si.parse_config_cnn(arguments, '10:' + si_nnet_spec, si_conv_nnet_spec) cfg_si.init_data_reading(train_data_spec, valid_data_spec) # parse the structure of the i-vector network cfg_adapt = NetworkConfig() net_split = adapt_nnet_spec.split(':') adapt_nnet_spec = '' for n in range(len(net_split) - 1): adapt_nnet_spec += net_split[n] + ':' cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... initializing the model') # setup up the model dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt) # read the initial DNN (the SI DNN which has been well trained) # _file2nnet(dnn.cnn_si.layers, filename = init_model_file) _file2nnet(dnn.cnn_si.layers, filename = 'BKUP/nnet.param.si') _file2nnet(dnn.dnn_adapt.layers, filename = 'BKUP/nnet.param.adapt') # get the training and validation functions for adaptation network training dnn.params = dnn.dnn_adapt.params # only update the parameters of the i-vector nnet dnn.delta_params = dnn.dnn_adapt.delta_params log('> ... getting the finetuning functions for iVecNN')
for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) # mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] si_nnet_spec = arguments['si_nnet_spec'] adapt_nnet_spec = arguments['adapt_nnet_spec'] wdir = arguments['wdir'] init_model_file = arguments['init_model'] # parse network configuration from arguments, and initialize data reading cfg_si = NetworkConfig() cfg_si.parse_config_dnn(arguments, si_nnet_spec) cfg_si.init_data_reading(train_data_spec, valid_data_spec) # parse the structure of the i-vector network cfg_adapt = NetworkConfig() # net_split = adapt_nnet_spec.split(':') # adapt_nnet_spec = '' # for n in xrange(len(net_split) - 1): # adapt_nnet_spec += net_split[n] + ':' # cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0') cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... initializing the model') # setup up the model dnn = DNN_SAT(numpy_rng=numpy_rng,
for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) # mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] multi_label = arguments['multi_label'] if multi_label=="true": multi_label = True else: multi_label = False # parse network configuration from arguments, and initialize data reading cfg = NetworkConfig(multi_label) cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0; ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
# parse data specification train_data_spec_array = parse_data_spec_mtl(train_data_spec) valid_data_spec_array = parse_data_spec_mtl(valid_data_spec) if len(train_data_spec_array) != task_number or len(valid_data_spec_array) != task_number: print len(train_data_spec_array) print task_number print "Error: #datasets in data specification doesn't match #tasks"; exit(1) # split shared_spec ans indiv_spec into individual task's networks nnet_spec_array, shared_layers_num = parse_nnet_spec_mtl(shared_spec, indiv_spec) if len(nnet_spec_array) != task_number: print "Error: #networks specified by --indiv-spec doesn't match #tasks"; exit(1) # parse network configuration from arguments, and initialize data reading for n in xrange(task_number): network_config = NetworkConfig() network_config.parse_config_dnn(arguments, nnet_spec_array[n]) network_config.init_data_reading(train_data_spec_array[n], valid_data_spec_array[n]) config_array.append(network_config) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) resume_training = False; resume_tasks = [] # if we are resuming training, then MLT only operates on the terminated tasks for n in xrange(task_number): log('> ... building the model for task %d' % (n)) cfg = config_array[n] # set up the model dnn_shared = None; shared_layers = [] if n > 0: dnn_shared = dnn_array[0]; shared_layers = [m for m in xrange(shared_layers_num)] print shared_layers dnn = DNN_MTL(task_id=n,numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg,
arg_elements = [sys.argv[i] for i in range(1, len(sys.argv))] arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) # mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] # parse network configuration from arguments, and initialize data reading cfg = NetworkConfig() cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0; ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch))
wdir = arguments['wdir'] init_model_file = arguments['init_model'] # parse network configuration from arguments, and initialize data reading cfg_si = NetworkConfig() cfg_si.model_type = 'CNN' cfg_si.parse_config_cnn(arguments, '10:' + si_nnet_spec, si_conv_nnet_spec) cfg_si.init_data_reading(train_data_spec, valid_data_spec) # parse the structure of the i-vector network cfg_adapt = NetworkConfig() net_split = adapt_nnet_spec.split(':') adapt_nnet_spec = '' for n in xrange(len(net_split) - 1): adapt_nnet_spec += net_split[n] + ':' cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... initializing the model') # setup up the model dnn = CNN_SAT(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg_si=cfg_si, cfg_adapt=cfg_adapt) # read the initial DNN (the SI DNN which has been well trained) # _file2nnet(dnn.cnn_si.layers, filename = init_model_file) _file2nnet(dnn.cnn_si.layers, filename='BKUP/nnet.param.si') _file2nnet(dnn.dnn_adapt.layers, filename='BKUP/nnet.param.adapt') # get the training and validation functions for adaptation network training
arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'nnet_spec_tower1', 'nnet_spec_tower2', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) # mandatory arguments train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] nnet_spec_tower1 = arguments['nnet_spec_tower1'] nnet_spec_tower2 = arguments['nnet_spec_tower2'] wdir = arguments['wdir'] # parse network configuration from arguments, and initialize data reading cfg_tower1 = NetworkConfig(); cfg_tower1.parse_config_dnn(arguments, nnet_spec_tower1 + ":0") cfg_tower2 = NetworkConfig(); cfg_tower2.parse_config_dnn(arguments, nnet_spec_tower2 + ":0") cfg = NetworkConfig(); cfg.parse_config_dnn(arguments, str(cfg_tower1.hidden_layers_sizes[-1] + cfg_tower2.hidden_layers_sizes[-1]) + ":" + nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0; ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True
arguments = parse_arguments(arg_elements) required_arguments = ['train_data', 'valid_data', 'si_nnet_spec', 'wdir', 'adapt_nnet_spec', 'init_model'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) # mandatory arguments train_data_spec = arguments['train_data']; valid_data_spec = arguments['valid_data'] si_nnet_spec = arguments['si_nnet_spec'] adapt_nnet_spec = arguments['adapt_nnet_spec']; wdir = arguments['wdir'] init_model_file = arguments['init_model'] # parse network configuration from arguments, and initialize data reading cfg_si = NetworkConfig() cfg_si.parse_config_dnn(arguments, si_nnet_spec) cfg_si.init_data_reading(train_data_spec, valid_data_spec) # parse the structure of the i-vector network cfg_adapt = NetworkConfig() # net_split = adapt_nnet_spec.split(':') # adapt_nnet_spec = '' # for n in xrange(len(net_split) - 1): # adapt_nnet_spec += net_split[n] + ':' # cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + '0') cfg_adapt.parse_config_dnn(arguments, adapt_nnet_spec + ':0') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... initializing the model') # setup up the model dnn = DNN_SAT(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg_si = cfg_si, cfg_adapt = cfg_adapt)
def dnn_run(arguments): required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg); exit(1) train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] cfg = NetworkConfig() cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0; ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file) if resume_training: _file2nnet(dnn.layers, filename = wdir + '/nnet.tmp') # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) + '(%)') # validation valid_error = validate_by_minibatch(valid_fn, cfg) log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100*numpy.mean(valid_error)) + '(%)') cfg.lrate.get_next_rate(current_error = 100*numpy.mean(valid_error)) # output nnet parameters and lrate, for training resume if cfg.lrate.epoch % cfg.model_save_step == 0: _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp') _lrate2file(cfg.lrate, wdir + '/training_state.tmp') # save the model and network configuration if cfg.param_output_file != '': _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor) log('> ... the final PDNN model parameter is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
def dnn_run(arguments): required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] cfg = NetworkConfig() cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0 ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers, set_layer_num=ptr_layer_number, filename=ptr_file) if resume_training: _file2nnet(dnn.layers, filename=wdir + '/nnet.tmp') # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100 * numpy.mean(train_error)) + '(%)') # validation valid_error = validate_by_minibatch(valid_fn, cfg) log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100 * numpy.mean(valid_error)) + '(%)') cfg.lrate.get_next_rate(current_error=100 * numpy.mean(valid_error)) # output nnet parameters and lrate, for training resume if cfg.lrate.epoch % cfg.model_save_step == 0: _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp') _lrate2file(cfg.lrate, wdir + '/training_state.tmp') # save the model and network configuration if cfg.param_output_file != '': _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor=cfg.input_dropout_factor, factor=cfg.dropout_factor) log('> ... the final PDNN model parameter is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the final PDNN model config is ' + cfg.cfg_output_file)