def init_dnn(dnn_cfg, numpy_rng=None, rho=0.0, base_dnn=None): """ Initialize a DNN given a DNN config """ if numpy_rng is None: numpy_rng = numpy.random.RandomState() if rho != 0: return KLDNN(numpy_rng, base_dnn, rho=rho, cfg=dnn_cfg) return DNN(numpy_rng, cfg=dnn_cfg)
def load_river_network(nnet_param = 'neural_network/river_network_params', nnet_cfg = 'neural_network/river_network_cfg'): cfg = cPickle.load(smart_open(nnet_cfg,'r')) cfg.init_activation() model = DNN(numpy_rng=numpy_rng, cfg = cfg) _file2nnet(model.layers, filename = nnet_param) get_river_probs = model.build_extract_feat_function(-1) return get_river_probs
def init_srbm(rbm_cfg, numpy_rng=None): """ Initialize a SRBM given a RBM config """ if numpy_rng is None: numpy_rng = numpy.random.RandomState() theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) # Following pdnn, initialize a parallel DNN and use it to initialize SRBM dnn_cfg = NetworkConfig() dnn_cfg.n_ins = rbm_cfg.n_ins dnn_cfg.hidden_layers_sizes = rbm_cfg.hidden_layers_sizes dnn_cfg.n_outs = rbm_cfg.n_outs dnn = DNN(numpy_rng, theano_rng=theano_rng, cfg=dnn_cfg) return SRBM(numpy_rng, theano_rng=theano_rng, cfg=rbm_cfg, dnn=dnn)
def train(): if cfg.train_model == 'dnn': model = DNN() inputs = model.input_data() avg_cost, auc_var = model.net(inputs) optimizer = fluid.optimizer.Adam(cfg.learning_rate) optimizer.minimize(avg_cost) place = fluid.CUDAPlace(0) if cfg.use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) dataset, file_list = get_dataset(inputs) logger.info("Training Begin") for epoch in range(cfg.epoches): random.shuffle(file_list) dataset.set_filelist(file_list) start_time = time.time() exe.train_from_dataset( program=fluid.default_main_program(), dataset=dataset, fetch_list=[avg_cost, auc_var], fetch_info=['Epoch {} cost: '.format(epoch + 1), ' - auc: '], print_period=cfg.log_interval, debug=False) end_time = time.time() logger.info("epoch %d finished, use time = %ds \n" % ((epoch + 1), end_time - start_time)) if (epoch + 1) % cfg.save_interval == 0: model_path = os.path.join(str(cfg.save_path), model.name, model.name + "_epoch_" + str(epoch + 1)) if not os.path.isdir(model_path): os.makedirs(model_path) logger.info("saving model to %s \n" % (model_path)) fluid.save(fluid.default_main_program(), os.path.join(model_path, "checkpoint")) logger.info("Done.")
def _set_MLPs(self): ''' load the MLP learned model as MLP network ''' nnet_param = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, 'models_makam', 'dampB.mdl') nnet_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, 'models_makam', 'dampB.cfg') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) cfg = cPickle.load(smart_open(nnet_cfg, 'r')) cfg.init_activation() self.cfg = cfg model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # load model parameters _file2nnet(model.layers, filename=nnet_param) # this is very slow self.model = model
# check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState() theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers, set_layer_num = ptr_layer_number, filename = ptr_file) if resume_training: _file2nnet(dnn.layers, filename = wdir + '/nnet.tmp') # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size)
train_data_spec = arguments['train_data'] wdir = arguments['wdir'] path = "/home/piero/Documents/Experiments/Real_Test/Spectral Coef/Noise vs BG_voice+Conversation vs Shout+Scream/fft coef/" os.chdir(path) filename = "rbm.cfg" train_data = "train.pickle.gz" test_data = "test.pickle.gz" batch_size = 128 log('> ... setting up the model and loading parameters') numpy_rng = np.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) cfg_dnn = cPickle.load(open(filename, 'r')) cfg_dnn.init_activation() model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg_dnn) # load model parameters _file2nnet(model.layers, filename=wdir + '/rbm.param') # initialize data reading cfg_dnn.init_data_reading_test(train_data_spec) # get the function for feature extraction log('> ... getting the feat-extraction function') extract_func = model.build_extract_feat_function(-1) output_mat = None # store the features for all the data in memory log('> ... generating features from the specified layer') while (not cfg_dnn.test_sets.is_finish()): # loop over the data cfg_dnn.test_sets.load_next_partition(cfg_dnn.test_xy)
def train_DNN(train_xy_file_list, valid_xy_file_list, \ nnets_file_name, n_ins, n_outs, ms_outs, hyper_params, buffer_size, plot=False): # get loggers for this function # this one writes to both console and file logger = logging.getLogger("main.train_DNN") logger.debug('Starting train_DNN') if plot: # this one takes care of plotting duties plotlogger = logging.getLogger("plotting") # create an (empty) plot of training convergence, ready to receive data points logger.create_plot('training convergence', MultipleSeriesPlot) try: assert numpy.sum(ms_outs) == n_outs except AssertionError: logger.critical('the summation of multi-stream outputs does not equal to %d' % (n_outs)) raise ####parameters##### finetune_lr = numpy.asarray(hyper_params['learning_rate'], dtype='float32') training_epochs = int(hyper_params['training_epochs']) batch_size = int(hyper_params['batch_size']) l1_reg = float(hyper_params['l1_reg']) l2_reg = float(hyper_params['l2_reg']) # private_l2_reg = float(hyper_params['private_l2_reg']) warmup_epoch = int(hyper_params['warmup_epoch']) momentum = float(hyper_params['momentum']) warmup_momentum = float(hyper_params['warmup_momentum']) use_rprop = int(hyper_params['use_rprop']) use_rprop = int(hyper_params['use_rprop']) hidden_layers_sizes = hyper_params['hidden_layer_size'] # stream_weights = hyper_params['stream_weights'] # private_hidden_sizes = hyper_params['private_hidden_sizes'] buffer_utt_size = buffer_size early_stop_epoch = int(hyper_params['early_stop_epochs']) hidden_activation = hyper_params['hidden_activation'] output_activation = hyper_params['output_activation'] # stream_lr_weights = hyper_params['stream_lr_weights'] # use_private_hidden = hyper_params['use_private_hidden'] model_type = hyper_params['model_type'] ## use a switch to turn on pretraining ## pretraining may not help too much, if this case, we turn it off to save time do_pretraining = hyper_params['do_pretraining'] pretraining_epochs = int(hyper_params['pretraining_epochs']) pretraining_lr = float(hyper_params['pretraining_lr']) buffer_size = int(buffer_size / batch_size) * batch_size ################### (train_x_file_list, train_y_file_list) = train_xy_file_list (valid_x_file_list, valid_y_file_list) = valid_xy_file_list logger.debug('Creating training data provider') train_data_reader = ListDataProvider(x_file_list=train_x_file_list, y_file_list=train_y_file_list, n_ins=n_ins, n_outs=n_outs, buffer_size=buffer_size, shuffle=True) logger.debug('Creating validation data provider') valid_data_reader = ListDataProvider(x_file_list=valid_x_file_list, y_file_list=valid_y_file_list, n_ins=n_ins, n_outs=n_outs, buffer_size=buffer_size, shuffle=False) shared_train_set_xy, temp_train_set_x, temp_train_set_y = train_data_reader.load_next_partition() train_set_x, train_set_y = shared_train_set_xy shared_valid_set_xy, temp_valid_set_x, temp_valid_set_y = valid_data_reader.load_next_partition() valid_set_x, valid_set_y = shared_valid_set_xy train_data_reader.reset() valid_data_reader.reset() ##temporally we use the training set as pretrain_set_x. ##we need to support any data for pretraining pretrain_set_x = train_set_x # numpy random generator numpy_rng = numpy.random.RandomState(123) logger.info('building the model') dnn_model = None pretrain_fn = None ## not all the model support pretraining right now train_fn = None valid_fn = None valid_model = None ## valid_fn and valid_model are the same. reserve to computer multi-stream distortion if model_type == 'DNN': dnn_model = DNN(numpy_rng=numpy_rng, n_ins=n_ins, n_outs=n_outs, l1_reg=l1_reg, l2_reg=l2_reg, hidden_layers_sizes=hidden_layers_sizes, hidden_activation=hidden_activation, output_activation=output_activation, use_rprop=use_rprop, rprop_init_update=finetune_lr) train_fn, valid_fn = dnn_model.build_finetune_functions( (train_set_x, train_set_y), (valid_set_x, valid_set_y), batch_size=batch_size) else: logger.critical('%s type NN model is not supported!' % (model_type)) raise logger.info('fine-tuning the %s model' % (model_type)) start_time = time.clock() best_dnn_model = dnn_model best_validation_loss = sys.float_info.max previous_loss = sys.float_info.max early_stop = 0 epoch = 0 previous_finetune_lr = finetune_lr while (epoch < training_epochs): epoch = epoch + 1 current_momentum = momentum current_finetune_lr = finetune_lr if epoch <= warmup_epoch: current_finetune_lr = finetune_lr current_momentum = warmup_momentum else: current_finetune_lr = previous_finetune_lr * 0.5 previous_finetune_lr = current_finetune_lr train_error = [] sub_start_time = time.clock() while (not train_data_reader.is_finish()): shared_train_set_xy, temp_train_set_x, temp_train_set_y = train_data_reader.load_next_partition() train_set_x.set_value(numpy.asarray(temp_train_set_x, dtype=theano.config.floatX), borrow=True) train_set_y.set_value(numpy.asarray(temp_train_set_y, dtype=theano.config.floatX), borrow=True) n_train_batches = train_set_x.get_value().shape[0] / batch_size logger.debug('this partition: %d frames (divided into %d batches of size %d)' % ( train_set_x.get_value(borrow=True).shape[0], n_train_batches, batch_size)) for minibatch_index in range(n_train_batches): this_train_error = train_fn(minibatch_index, current_finetune_lr, current_momentum) train_error.append(this_train_error) if numpy.isnan(this_train_error): logger.warning('training error over minibatch %d of %d was %s' % ( minibatch_index + 1, n_train_batches, this_train_error)) train_data_reader.reset() logger.debug('calculating validation loss') validation_losses = valid_fn() this_validation_loss = numpy.mean(validation_losses) # this has a possible bias if the minibatches were not all of identical size # but it should not be siginficant if minibatches are small this_train_valid_loss = numpy.mean(train_error) sub_end_time = time.clock() loss_difference = this_validation_loss - previous_loss logger.info('epoch %i, validation error %f, train error %f time spent %.2f' % ( epoch, this_validation_loss, this_train_valid_loss, (sub_end_time - sub_start_time))) if plot: plotlogger.add_plot_point('training convergence', 'validation set', (epoch, this_validation_loss)) plotlogger.add_plot_point('training convergence', 'training set', (epoch, this_train_valid_loss)) plotlogger.save_plot('training convergence', title='Progress of training and validation error', xlabel='epochs', ylabel='error') if this_validation_loss < best_validation_loss: best_dnn_model = dnn_model best_validation_loss = this_validation_loss logger.debug('validation loss decreased, so saving model') early_stop = 0 else: logger.debug('validation loss did not improve') dbn = best_dnn_model early_stop += 1 if early_stop >= early_stop_epoch: # too many consecutive epochs without surpassing the best model logger.debug('stopping early') break if math.isnan(this_validation_loss): break previous_loss = this_validation_loss end_time = time.clock() pickle.dump(best_dnn_model, open(nnets_file_name, 'wb')) logger.info( 'overall training time: %.2fm validation error %f' % ((end_time - start_time) / 60., best_validation_loss)) if plot: plotlogger.save_plot('training convergence', title='Final training and validation error', xlabel='epochs', ylabel='error') return best_validation_loss
# -*- coding: utf-8 -*- import os import sys import numpy as np import tensorflow as tf from models.dnn import DNN from data_generate import * from data_process import get_node2id os.environ['CUDA_VISIBLE_DEVICES'] = '0' config = tf.ConfigProto() config.gpu_options.allow_growth = True if __name__ == '__main__': if sys.argv[1] == 'deepwalk': embeddings_file = 'deepwalk.embeddings' elif sys.argv[1] == 'hin2vec': embeddings_file = 'node_vectors.txt' node2id = get_node2id() node_embeddings = get_embeddings(embeddings_file, node2id) train_dataset, test_dataset = train_test_split('./data/all_data.csv', train_size=0.7) model = DNN(config=config, batch_size=2048, node_embeddings=node_embeddings, optimizer='adam', learning_rate=0.001, epoch_num=5) model.train(train_dataset=train_dataset, test_dataset=test_dataset)
def evaluate(): place = fluid.CUDAPlace(0) if cfg.use_cuda else fluid.CPUPlace() inference_scope = fluid.Scope() test_files = [ os.path.join(cfg.evaluate_file_path, x) for x in os.listdir(cfg.evaluate_file_path) ] dataset = CriteoDataset() test_reader = paddle.batch(dataset.test(test_files), batch_size=cfg.batch_size) startup_program = fluid.framework.Program() test_program = fluid.framework.Program() model = DNN() model_path = os.path.join(cfg.save_path, model.name + "_epoch_" + str(cfg.test_epoch), "checkpoint") with fluid.framework.program_guard(test_program, startup_program): with fluid.unique_name.guard(): inputs = model.input_data() loss, auc_var = model.net(inputs) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=inputs, place=place) fluid.load(fluid.default_main_program(), model_path, exe) auc_states_names = [ '_generated_var_0', '_generated_var_1', '_generated_var_2', '_generated_var_3' ] for var in auc_states_names: set_zero(var, scope=inference_scope, place=place) run_index = 0 infer_auc = 0 L = [] for batch_id, data_test in enumerate(test_reader()): loss_val, auc_val = exe.run(test_program, feed=feeder.feed(data_test), fetch_list=[loss, auc_var]) run_index += 1 infer_auc = auc_val L.append(loss_val / cfg.batch_size) if batch_id % cfg.log_interval == 0: logger.info("TEST --> batch: {} loss: {} auc: {}".format( batch_id, loss_val / cfg.batch_size, auc_val)) infer_loss = np.mean(L) infer_result = {} infer_result['loss'] = infer_loss infer_result['auc'] = infer_auc if not os.path.isdir(cfg.log_dir): os.makedirs(cfg.log_dir) log_path = os.path.join(cfg.log_dir, model.name + '_infer_result.log') logger.info(str(infer_result)) with open(log_path, 'w+') as f: f.write(str(infer_result)) logger.info("Done.") return infer_result
) train_dnn(dnn_model_1, user_list, ae_list, user_id=0, batch_size=128, epochs=1000, display_step=10) dnn_model_2 = DNN( input_size=hidden_size, output_size=user_list[0].n_labels, architecture={'h1': 128, 'h2': 128}, learning_rate=0.001 ) train_dnn(dnn_model_2, user_list, ae_list, user_id=1, batch_size=128, epochs=1000, display_step=10) ''' dnn_model = DNN(input_size=hidden_size, output_size=user_list[0].n_labels, architecture={ 'h1': 128, 'h2': 128 }, learning_rate=0.001) train_dnn(dnn_model, user_list, ae_list, batch_size=128, epochs=10000, display_step=100) batch_xs, batch_ys = user_list[0].next_batch(user_list[0].n_samples_test, is_train=False) for i in range(n_users): acc_test = cal_acc(dnn_model.predict(ae_list[i].transform(batch_xs)), batch_ys)
def main(arg_elements): # check the arguments arguments = parse_arguments(arg_elements) required_arguments = [ 'data', 'nnet_param', 'nnet_cfg', 'output_file', 'layer_index', 'batch_size' ] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) # mandatory arguments data_spec = arguments['data'] nnet_param = arguments['nnet_param'] nnet_cfg = arguments['nnet_cfg'] output_file = arguments['output_file'] layer_index = int(arguments['layer_index']) batch_size = int(arguments['batch_size']) argmax = arguments.has_key('argmax') and string2bool(arguments['argmax']) # load network configuration and set up the model log('> ... setting up the model and loading parameters') numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) cfg = cPickle.load(smart_open(nnet_cfg, 'r')) cfg.init_activation() model = None if cfg.model_type == 'DNN': model = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) elif cfg.model_type == 'CNN': model = CNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg, testing=True) # load model parameters _file2nnet(model.layers, filename=nnet_param) # initialize data reading cfg.init_data_reading_test(data_spec) # get the function for feature extraction log('> ... getting the feat-extraction function') extract_func = model.build_extract_feat_function(layer_index) output_mats = [ ] # store the features for all the data in memory. TODO: output the features in a streaming mode log('> ... generating features from the specified layer') while (not cfg.test_sets.is_finish()): # loop over the data cfg.test_sets.load_next_partition(cfg.test_xy) batch_num = int( math.ceil(1.0 * cfg.test_sets.cur_frame_num / batch_size)) for batch_index in xrange(batch_num): # loop over mini-batches start_index = batch_index * batch_size end_index = min((batch_index + 1) * batch_size, cfg.test_sets.cur_frame_num ) # the residue may be smaller than a mini-batch output = extract_func( cfg.test_x.get_value()[start_index:end_index]) output_mats.append(output) output_mat = numpy.concatenate(output_mats) if argmax: output_mat = output_mat.argmax(axis=1) # output the feature representations using pickle f = smart_open(output_file, 'wb') cPickle.dump(output_mat, f, cPickle.HIGHEST_PROTOCOL) log('> ... the features are stored in ' + output_file)
def dnn_run(arguments): required_arguments = ['train_data', 'valid_data', 'nnet_spec', 'wdir'] for arg in required_arguments: if arguments.has_key(arg) == False: print "Error: the argument %s has to be specified" % (arg) exit(1) train_data_spec = arguments['train_data'] valid_data_spec = arguments['valid_data'] nnet_spec = arguments['nnet_spec'] wdir = arguments['wdir'] cfg = NetworkConfig() cfg.parse_config_dnn(arguments, nnet_spec) cfg.init_data_reading(train_data_spec, valid_data_spec) # parse pre-training options # pre-training files and layer number (how many layers are set to the pre-training parameters) ptr_layer_number = 0 ptr_file = '' if arguments.has_key('ptr_file') and arguments.has_key('ptr_layer_number'): ptr_file = arguments['ptr_file'] ptr_layer_number = int(arguments['ptr_layer_number']) # check working dir to see whether it's resuming training resume_training = False if os.path.exists(wdir + '/nnet.tmp') and os.path.exists(wdir + '/training_state.tmp'): resume_training = True cfg.lrate = _file2lrate(wdir + '/training_state.tmp') log('> ... found nnet.tmp and training_state.tmp, now resume training from epoch ' + str(cfg.lrate.epoch)) numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2**30)) log('> ... building the model') # setup model if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng=theano_rng, cfg=cfg) # initialize model parameters # if not resuming training, initialized from the specified pre-training file # if resuming training, initialized from the tmp model file if (ptr_layer_number > 0) and (resume_training is False): _file2nnet(dnn.layers, set_layer_num=ptr_layer_number, filename=ptr_file) if resume_training: _file2nnet(dnn.layers, filename=wdir + '/nnet.tmp') # get the training, validation and testing function for the model log('> ... getting the finetuning functions') train_fn, valid_fn = dnn.build_finetune_functions( (cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100 * numpy.mean(train_error)) + '(%)') # validation valid_error = validate_by_minibatch(valid_fn, cfg) log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100 * numpy.mean(valid_error)) + '(%)') cfg.lrate.get_next_rate(current_error=100 * numpy.mean(valid_error)) # output nnet parameters and lrate, for training resume if cfg.lrate.epoch % cfg.model_save_step == 0: _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp') _lrate2file(cfg.lrate, wdir + '/training_state.tmp') # save the model and network configuration if cfg.param_output_file != '': _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor=cfg.input_dropout_factor, factor=cfg.dropout_factor) log('> ... the final PDNN model parameter is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the final PDNN model config is ' + cfg.cfg_output_file)
from uci_adult import ADULT from models.dnn import DNN, train_dnn if __name__ == '__main__': adult = ADULT(path='../adult', n_users=10, user_id=0) dnn_model = DNN(input_size=adult.n_features, output_size=adult.n_labels, architecture={ 'h1': 128, 'h2': 128 }, learning_rate=0.001) train_dnn(dnn_model, [adult], batch_size=128, epochs=10000, display_step=10)
numpy_rng = numpy.random.RandomState(89677) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) resume_training = False; resume_tasks = [] # if we are resuming training, then MLT only operates on the terminated tasks for n in xrange(task_number): log('> ... building the model for task %d' % (n)) cfg = config_array[n] # set up the model dnn_shared = None; shared_layers = [] if n > 0: dnn_shared = dnn_array[0]; shared_layers = [m for m in xrange(shared_layers_num)] if cfg.do_dropout: dnn = DNN_Dropout(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn_shared = dnn_shared, shared_layers = shared_layers) else: dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg, dnn_shared = dnn_shared, shared_layers = shared_layers) # get the training, validation and testing function for the model log('> ... getting the finetuning functions for task %d' % (n)) train_fn, valid_fn = dnn.build_finetune_functions((cfg.train_x, cfg.train_y), (cfg.valid_x, cfg.valid_y), batch_size=cfg.batch_size) # add dnn and the functions to the list dnn_array.append(dnn) train_fn_array.append(train_fn); valid_fn_array.append(valid_fn) # check the working dir to decide whether it's resuming training; if yes, load the tmp network files for initialization if os.path.exists(wdir + '/nnet.tmp.task' + str(n)) and os.path.exists(wdir + '/training_state.tmp.task' + str(n)): resume_training = True; resume_tasks.append(n) cfg.lrate = _file2lrate(wdir + '/training_state.tmp.task' + str(n)) log('> ... found nnet.tmp.task%d and training_state.tmp.task%d, now resume task%d training from epoch %d' % (n, n, n, cfg.lrate.epoch)) _file2nnet(dnn.layers, filename = wdir + '/nnet.tmp.task' + str(n)) # pre-training works only if we are NOT resuming training