def learn(self, dataset, mask, actions=None, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=False, savefreq=None, savefile=None, dataset_eval=None, mask_eval=None, actions_eval=None, replicate_K=None, normalization='frame'): """ Train DKF """ assert not self.params[ 'validate_only'], 'cannot learn in validate only mode' assert len(dataset.shape) == 3, 'Expecting 3D tensor for data' assert dataset.shape[2] == self.params[ 'dim_observations'], 'Dim observations not valid' N = dataset.shape[0] idxlist = range(N) batchlist = np.split(idxlist, range(batch_size, N, batch_size)) bound_train_list, bound_valid_list, bound_tsbn_list = [], [], [] p_norm, g_norm, opt_norm = None, None, None #Lists used to track quantities mu_list_train, cov_list_train, mu_list_valid, cov_list_valid, nll_valid_list = [],[],[],[],[] current_lr = self.params['lr'] assert actions is None, 'No actions supported' #Start of training loop for epoch in range(epoch_start, epoch_end): #Shuffle if shuffle: np.random.shuffle(idxlist) batchlist = np.split(idxlist, range(batch_size, N, batch_size)) #Always shuffle order the batches are presented in np.random.shuffle(batchlist) start_time = time.time() bound = 0 #if epoch>100 and epoch%500==0: # self._p('ANNEALING LEARNING RATE NOW') # current_lr = self.decay_lr() for bnum, batch_idx in enumerate(batchlist): batch_idx = batchlist[bnum] X = dataset[batch_idx, :, :].astype(config.floatX) M = mask[batch_idx, :].astype(config.floatX) U = None #Tack on 0's if the matrix size (optimization->theano doesnt have to redefine matrices) if X.shape[0] < batch_size: Nremaining = int(batch_size - X.shape[0]) X = np.concatenate( [X, np.zeros((Nremaining, X.shape[1], X.shape[2]))], axis=0).astype(config.floatX) M = np.concatenate( [M, np.zeros((Nremaining, X.shape[1]))], axis=0).astype(config.floatX) #Reduce the dimensionality of the tensors based on the maximum size of the mask maxT = int(np.max(M.sum(1))) X = X[:, :maxT, :] M = M[:, :maxT] eps = np.random.randn(X.shape[0], maxT, self.params['dim_stochastic']).astype( config.floatX) batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal = self.train_debug( X=X, M=M, eps=eps) #Number of frames M_sum = M.sum() #Correction for replicating batch if replicate_K is not None: batch_bound, negCLL, KL = batch_bound / replicate_K, negCLL / replicate_K, KL / replicate_K, M_sum = M_sum / replicate_K #Update bound bound += batch_bound ### Display ### if bnum % 10 == 0: if normalization == 'frame': bval = batch_bound / float(M_sum) elif normalization == 'sequence': bval = batch_bound / float(X.shape[0]) else: assert False, 'Invalid normalization' if p_norm is not None: self._p(( 'Bnum:%d, Batch Bound: %.4f, |w|: %.4f, |dw|: %.4f,|w_opt|:%.4f, -veCLL:%.4f, KL:%.4f: , anneal:%.4f' ) % (bnum, bval, p_norm, g_norm, opt_norm, negCLL, KL, anneal)) else: self._p(('Bnum:%d, Batch Bound: %.4f') % (bnum, bval)) if normalization == 'frame': bound /= float(mask.sum()) elif normalization == 'sequence': bound /= float(N) else: assert False, 'Invalid normalization' bound_train_list.append((epoch, bound)) end_time = time.time() self._p(('(Ep %d) Bound: %.4f , LR: %.4e[Took %.4f seconds] ') % (epoch, bound, current_lr, end_time - start_time)) #Save at intermediate stages if savefreq is not None and epoch % savefreq == 0: assert savefile is not None, 'expecting savefile' self._p(('Saving at epoch %d' % epoch)) self._saveModel(fname=savefile + '-EP' + str(epoch)) intermediate = {} if dataset_eval is not None and mask_eval is not None: tmpMap = {} #Track the validation bound and the TSBN bound bound_valid_list.append( (epoch, self.evaluateBound(dataset_eval, mask_eval, actions=actions_eval, batch_size=batch_size, additional=tmpMap, normalization=normalization))) bound_tsbn_list.append((epoch, tmpMap['tsbn_bound'])) nll_valid_list.append( self.impSamplingNLL(dataset_eval, mask_eval, batch_size, actions=actions_eval, normalization=normalization)) intermediate['valid_bound'] = np.array(bound_valid_list) intermediate['train_bound'] = np.array(bound_train_list) intermediate['tsbn_bound'] = np.array(bound_tsbn_list) intermediate['valid_nll'] = np.array(nll_valid_list) saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate) #Final information to be collected retMap = {} retMap['train_bound'] = np.array(bound_train_list) retMap['valid_bound'] = np.array(bound_valid_list) retMap['tsbn_bound'] = np.array(bound_tsbn_list) retMap['valid_nll'] = np.array(nll_valid_list) if 'synthetic' in self.params['dataset']: retMap['mu_posterior_train'] = np.concatenate(mu_list_train, axis=2) retMap['cov_posterior_train'] = np.concatenate(cov_list_train, axis=2) retMap['mu_posterior_valid'] = np.concatenate(mu_list_valid, axis=2) retMap['cov_posterior_valid'] = np.concatenate(cov_list_valid, axis=2) return retMap
dkf = DKF(params, paramFile=pfile) displayTime('Building dkf', start_time, time.time()) """Set save prefix""" savef = os.path.join(params['savedir'], params['unique_id']) print 'Savefile: ', savef start_time = time.time() """Learn the model (see stinfmodel/learning.py)""" savedata = DKF_learn.learn(dkf, dataset['train'], dataset['mask_train'], epoch_start=0, epoch_end=params['epochs'], batch_size=params['batch_size'], savefreq=params['savefreq'], savefile=savef, dataset_eval=dataset['valid'], mask_eval=dataset['mask_valid'], replicate_K=params['replicate_K'], shuffle=False) displayTime('Running DKF', start_time, time.time()) """ Evaluate bound on test set (see stinfmodel/evaluate.py)""" savedata['bound_test'] = DKF_evaluate.evaluateBound( dkf, dataset['test'], dataset['mask_test'], batch_size=params['batch_size']) saveHDF5(savef + '-final.h5', savedata) print 'Test Bound: ', savedata['bound_test'] import ipdb ipdb.set_trace()
params = readPickle(pfile)[0] suffix = '-EP' + str(epochval[mname]) + '-params.npz' rfile = models[mname] + suffix assert os.path.exists(rfile), 'not found' params['EVALUATION'] = True if 'wikicorp' in mname: vae = VAE(params, paramFile=pfile, reloadFile=rfile, additional_attrs=additional_attrs_wiki) trainData = dataset_wiki['train'] validData = dataset_wiki['valid'] Ntrain = trainData.shape[0] np.random.seed(0) trainData = trainData[np.random.permutation(Ntrain)[:100000]] else: vae = VAE(params, paramFile=pfile, reloadFile=rfile, additional_attrs=additional_attrs_rcv2) savef = SAVEDIR + mname train_map_init_final = VAE_evaluate.getInitFinal(vae, trainData, 500) train_map_init_final['data'] = trainData.toarray() saveHDF5(savef + '-if_train.h5', train_map_init_final) eval_map_init_final = VAE_evaluate.getInitFinal(vae, validData, 500) eval_map_init_final['data'] = validData.toarray() saveHDF5(savef + '-if_eval.h5', eval_map_init_final) print 'Saved: ', mname print 'Done'
def learn(self, dataset, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=False, savefile=None, savefreq=None, dataset_eval=None, replicate_K=None): assert len(dataset.shape) == 2, 'Expecting 2D dataset matrix' assert dataset.shape[1] == self.params[ 'dim_observations'], 'dim observations incorrect' N = dataset.shape[0] idxlist = range(N) if shuffle: np.random.shuffle(idxlist) trainbound, validbound, validll, current_lr = [], [], [], self.params[ 'lr'] #Training epochs for epoch in range(epoch_start, epoch_end + 1): start_time = time.time() bound, grad_norm, param_norm = 0, [], [] #Update learning rate current_lr = self.decay_lr() #Go through dataset for bnum, st_idx in enumerate(range(0, N, batch_size)): end_idx = min(st_idx + batch_size, N) X = dataset[idxlist[st_idx:end_idx]].astype(config.floatX) Nbatch = X.shape[0] if replicate_K is not None: X = X.repeat(replicate_K, 0) eps = np.random.randn(X.shape[0], self.params['dim_stochastic']).astype( config.floatX) #Forward/Backward pass if self.params['inference_model'] == 'single': batch_bound, anneal = self.train(X=X, eps=eps) else: assert False, 'Should not be here' #Divide value of the bound by replicateK if replicate_K is not None: batch_bound /= float(replicate_K) bound += batch_bound if bnum % 50 == 0: self._p(( '--Batch: %d, Batch Bound: %.4f, Anneal : %.4f, Lr : %.6e--' ) % (bnum, batch_bound / Nbatch, anneal, current_lr)) bound /= float(N) trainbound.append((epoch, bound)) end_time = time.time() self._p(('Ep (%d) Upper Bound: %.4f [Took %.4f seconds]') % (epoch, bound, (end_time - start_time))) if savefreq is not None and epoch % savefreq == 0: self._p(('Saving at epoch %d' % epoch)) self._saveModel(fname=savefile + '-EP' + str(epoch)) if dataset_eval is not None: v_bound = self.evaluateBound(dataset_eval, batch_size=batch_size) validbound.append((epoch, v_bound)) v_ll = self.impSamplingNLL(dataset_eval, batch_size=batch_size) validll.append((epoch, v_ll)) self._p(('Ep (%d): Valid Bound: %.4f, Valid LL: %.4f') % (epoch, v_bound, v_ll)) intermediate = {} intermediate['train_bound'] = np.array(trainbound) intermediate['valid_bound'] = np.array(validbound) intermediate['valid_ll'] = np.array(validll) intermediate['samples'] = self.sample() saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate) ret_map = {} ret_map['train_bound'] = np.array(trainbound) ret_map['valid_bound'] = np.array(validbound) ret_map['valid_ll'] = np.array(validll) return ret_map
savef = os.path.join(params['savedir'], params['unique_id']) start_time = time.time() trainData = dataset['train'] validData = dataset['valid'] savedata = Learn.learn(model, dataset=trainData, epoch_start=0, epoch_end=params['epochs'], batch_size=params['batch_size'], savefreq=params['savefreq'], savefile=savef, dataset_eval=validData) displayTime('Running Model', start_time, time.time()) saveHDF5(savef + '-final.h5', savedata) if 'wiki' not in params['dataset']: evaluate = {} test_results = Evaluate.evaluateBound(model, dataset['test'], batch_size=params['batch_size']) evaluate['test_perp_0'] = test_results['perp_0'] evaluate['test_perp_f'] = test_results['perp_f'] print 'Test Bound: ', evaluate['test_perp_f'] kname = 'valid_perp_f' # Work w/ the best model thus far epochMin, valMin, idxMin = getLowestError(savedata[kname]) reloadFile = pfile.replace('-config.pkl', '') + '-EP' + str( int(epochMin)) + '-params.npz' print 'Loading model from epoch : ', epochMin #reloadFile
additional_attrs_wiki['idf'] = getTF(dataset_wiki) #additional_attrs_rcv2['idf'] = getTF(dataset_rcv2) for mname in MODELS_TO_USE: if 'wikicorp' not in mname: continue print 'Model: ', mname pfile = models[mname].split('uid')[0] + 'uid-config.pkl' params = readPickle(pfile)[0] suffix = '-EP' + str(epochval[mname]) + '-params.npz' rfile = models[mname] + suffix assert os.path.exists(rfile), 'not found' params['EVALUATION'] = True if 'wikicorp' in mname: vae = VAE(params, paramFile=pfile, reloadFile=rfile, additional_attrs=additional_attrs_wiki) else: vae = VAE(params, paramFile=pfile, reloadFile=rfile, additional_attrs=additional_attrs_rcv2) jacob = expectedJacobian(vae, nsamples=1) _, s, _ = np.linalg.svd(jacob) fname = SAVEDIR + mname + '-jacob.h5' saveHDF5(fname, {'jacob': jacob, 'svals': s}) print 'Saved: ', mname print 'Done'
def learn(self, dataset, mask, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=False, savefreq=None, savefile=None, dataset_eval=None, mask_eval=None): """ Loop through dataset for training """ assert not self.params['validate_only'], 'Cannot learn in validate mode' assert len(dataset.shape) == 3, 'Expecting 3D tensor' assert dataset.shape[2] == self.params[ 'dim_observations'], 'Dimension of observation not valid' N = dataset.shape[0] idxlist = range(N) if shuffle: np.random.shuffle(idxlist) nll_train_list, nll_valid_list = [], [] for epoch in range(epoch_start, epoch_end): start_time = time.time() nll = 0 for bnum, st_idx in enumerate(range(0, N, batch_size)): end_idx = min(st_idx + batch_size, N) X = dataset[idxlist[st_idx:end_idx], :, :].astype( config.floatX) M = mask[idxlist[st_idx:end_idx], :].astype(config.floatX) batch_nll = self.train(X=X, M=M) nll += batch_nll self._p(('Bnum:%d, Batch Bound: %.4f') % (bnum, batch_nll / float(M.sum()))) nll /= float(mask.sum()) nll_train_list.append((epoch, nll)) end_time = time.time() if bnum % 10 == 0: self._p(('(Ep %d) NLL: %.4f [Took %.4f seconds]') % (epoch, nll, end_time - start_time)) if savefreq is not None and epoch % savefreq == 0: assert savefile is not None, 'expecting savefile' self._p(('Saving at epoch %d' % epoch)) self._saveModel(fname=savefile + '-EP' + str(epoch)) if dataset_eval is not None and mask_eval is not None: nll_valid_list.append( (epoch, self.evaluateNLL(dataset_eval, mask_eval, batch_size=batch_size))) intermediate = {} intermediate['train_nll'] = np.array(nll_train_list) intermediate['valid_nll'] = np.array(nll_valid_list) saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate) retMap = {} retMap['train_nll'] = np.array(nll_train_list) retMap['valid_nll'] = np.array(nll_valid_list) return retMap
def learn(dmm, dataset, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=True, savefreq=None, savefile=None, dataset_eval=None): """ Train a DMM using data dmm: DMM object dataset, dataset_eval: <dict> object with data epoch_start, epoch_end, batch_size, savefreq: <int> savefile: <str> Savefile for intermediate results """ assert not dmm.params[ 'validate_only'], 'cannot run function in validate only mode' N = dataset['tensor'].shape[0] idxlist = range(N) batchlist = np.split(idxlist, range(batch_size, N, batch_size)) bound_train_list, bound_valid_list = [], [] epfreq = 1 dmm.resetDataset(dataset) for epoch in range(epoch_start, epoch_end): #Shuffle and reset batches if shuffle: np.random.shuffle(idxlist) batchlist = np.split(idxlist, range(batch_size, N, batch_size)) start_time = time.time() bound = 0 for bnum, batch_idx in enumerate(batchlist): batch_idx = batchlist[bnum] batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal = dmm.train_debug( idx=batch_idx) if np.any( np.isnan( np.array([ batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal ]))): print('Warning: you have encountered a NaN') import ipdb ipdb.set_trace() bound += batch_bound if epoch % epfreq == 0 and bnum % 10 == 0: bval = batch_bound / float(dataset['mask'][batch_idx].sum()) dmm._p(( 'Bnum: %d, Batch Bound: %.4f, |w|: %.4f, |dw|: %.4f, |w_opt|: %.4f' ) % (bnum, bval, p_norm, g_norm, opt_norm)) dmm._p(('-veCLL:%.4f, KL:%.4f, anneal:%.4f') % (negCLL, KL, anneal)) bound /= float(dataset['mask'].sum()) bound_train_list.append((epoch, bound)) end_time = time.time() if epoch % epfreq == 0: dmm._p(('(Ep %d) Bound: %.4f [Took %.4f seconds] ') % (epoch, bound, end_time - start_time)) if savefreq is not None and epoch % savefreq == 0: assert savefile is not None, 'expecting savefile' dmm._p(('Saving at epoch %d' % epoch)) dmm._saveModel(fname=savefile + '-EP' + str(epoch)) intermediate = {} if dataset_eval is not None: tmpMap = {} bound_valid_list.append( (epoch, DMM_evaluate.evaluateBound(dmm, dataset_eval, batch_size=batch_size))) intermediate['valid_bound'] = np.array(bound_valid_list) intermediate['train_bound'] = np.array(bound_train_list) saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate) dmm.resetDataset(dataset) retMap = {} retMap['train_bound'] = np.array(bound_train_list) retMap['valid_bound'] = np.array(bound_valid_list) return retMap
def learn(vae, dataset=None, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=False, savefile=None, savefreq=None, dataset_eval=None): assert dataset is not None, 'Expecting 2D dataset matrix' assert np.prod( dataset.shape[1:] ) == vae.params['dim_observations'], 'dim observations incorrect' N = dataset.shape[0] idxlist = range(N) trainbound_0, trainbound_f = [], [] trainperp_0, trainperp_f, validperp_0, validperp_f, svallist = [],[],[],[],[] gmulist, glcovlist, diff_elbolist, batchtimelist = [], [], [], [] gdifflists = {} for name in vae.p_names: gdifflists[name] = [] learnBatch = None print 'OPT TYPE: ', vae.params['opt_type'] if vae.params['opt_type'] in ['none']: learnBatch = _optNone elif vae.params['opt_type'] in ['finopt']: learnBatch = _optFinopt elif vae.params['opt_type'] in ['finopt_none', 'none_finopt']: learnBatch = _optMixed else: raise ValueError('Invalid optimization type: ' + str(vae.params['opt_type'])) for epoch in range(epoch_start, epoch_end + 1): np.random.shuffle(idxlist) start_time = time.time() bd_0, bd_f, gmu, glcov, diff_elbo, time_taken = 0, 0, 0, 0, 0, 0 """ Evaluate more frequently in the initial few epochs """ if epoch > 10: sfreq = savefreq tfreq = savefreq else: sfreq = 3 tfreq = 3 for bnum, st_idx in enumerate(range(0, N, batch_size)): end_idx = min(st_idx + batch_size, N) X = dataset[idxlist[st_idx:end_idx]] if X.__class__.__name__ == 'csr_matrix' or X.__class__.__name__ == 'csc_matrix': X = X.toarray() X = X.astype(config.floatX) Nbatch = X.shape[0] update_opt = None if vae.params['opt_type'] in ['finopt_none', 'none_finopt']: if vae.params[ 'opt_type'] == 'finopt_none': #start w/ optimizing var. params, then stop if epoch < 10: #(epoch_end/2.): update_opt = True else: update_opt = False else: #'none_finopt' - start w/out optimizing var. params, then optimize them if epoch < 10: #(epoch_end/2.): update_opt = False else: update_opt = True retVal = learnBatch(vae, bnum, Nbatch, X, calcELBOfinal=(epoch % tfreq == 0), update_opt=update_opt) bd_0 += retVal['elbo_0'] bd_f += retVal['elbo_f'] gmu += retVal['gmu'] glcov += retVal['glcov'] diff_elbo += retVal['diff_elbo'] time_taken += retVal['time_taken'] if bnum > 2: pass #break bd_0 /= float(N) bd_f /= float(N) gmu /= float(N) glcov /= float(N) diff_elbo /= float(N) time_taken /= float(bnum) batchtimelist.append((epoch, time_taken)) trainbound_0.append((epoch, bd_0)) trainbound_f.append((epoch, bd_f)) gmulist.append((epoch, gmu)) glcovlist.append((epoch, glcov)) diff_elbolist.append((epoch, diff_elbo)) end_time = time.time() print '\n' vae._p(( 'Ep(%d) ELBO[0]: %.4f, ELBO[f]: %.4f, gmu: %.4f, glcov: %.4f, [%.3f, %.3f] [%.4f seconds]' ) % (epoch, bd_0, bd_f, gmu, glcov, diff_elbo, 0., (end_time - start_time))) if savefreq is not None and epoch % sfreq == 0: vae._p(('Saving at epoch %d' % epoch)) vae._saveModel(fname=savefile + '-EP' + str(epoch)) if dataset_eval is not None: if vae.params['data_type'] == 'bow': k0 = 'perp_0' kf = 'perp_f' else: k0 = 'elbo_0' kf = 'elbo_f' eval_retVal = VAE_evaluate.evaluateBound(vae, dataset_eval, batch_size=batch_size) validperp_0.append((epoch, eval_retVal[k0])) validperp_f.append((epoch, eval_retVal[kf])) train_retVal = VAE_evaluate.evaluateBound( vae, dataset, batch_size=batch_size) trainperp_0.append((epoch, train_retVal[k0])) trainperp_f.append((epoch, train_retVal[kf])) vae._p(('\nEp (%d): Valid[0]: %.4f, Valid[f]: %.4f') % (epoch, eval_retVal[k0], eval_retVal[kf])) vae._p(('\nEp (%d): Train[0]: %.4f, Train[f]: %.4f') % (epoch, train_retVal[k0], train_retVal[kf])) intermediate = {} intermediate['train_bound_0'] = np.array(trainbound_0) intermediate['train_bound_f'] = np.array(trainbound_f) if vae.params['data_type'] == 'bow': k0 = 'valid_perp_0' kf = 'valid_perp_f' else: k0 = 'valid_bound_0' kf = 'valid_bound_f' intermediate[k0] = np.array(validperp_0) intermediate[kf] = np.array(validperp_f) intermediate[k0.replace('valid', 'train')] = np.array(trainperp_0) intermediate[kf.replace('valid', 'train')] = np.array(trainperp_f) intermediate['batch_time'] = np.array(batchtimelist) intermediate['gmu'] = np.array(gmulist) intermediate['glcov'] = np.array(glcovlist) intermediate['diff_elbo'] = np.array(diff_elbolist) jacob = vae.jacobian_logprobs( np.zeros((vae.params['dim_stochastic'], )).astype('float32')) _, svals, _ = np.linalg.svd(jacob) epres = np.array([epoch] + svals.ravel().tolist()) svallist.append(epres) intermediate['svals'] = np.array(svallist) saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate) ret_map = {} ret_map['train_bound_0'] = np.array(trainbound_0) ret_map['train_bound_f'] = np.array(trainbound_f) ret_map['batch_time'] = np.array(batchtimelist) if vae.params['data_type'] == 'bow': k0 = 'valid_perp_0' kf = 'valid_perp_f' else: k0 = 'valid_bound_0' kf = 'valid_bound_f' ret_map[k0] = np.array(validperp_0) ret_map[kf] = np.array(validperp_f) ret_map[k0.replace('valid', 'train')] = np.array(trainperp_0) ret_map[kf.replace('valid', 'train')] = np.array(trainperp_f) ret_map['gmu'] = np.array(gmulist) ret_map['glcov'] = np.array(glcovlist) ret_map['diff_elbo'] = np.array(diff_elbolist) ret_map['svals'] = np.array(svallist) return ret_map
def learn(dkf, dataset, mask, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=True, savefreq=None, savefile=None, dataset_eval=None, mask_eval=None, replicate_K=None, normalization='frame'): """ Train DKF """ assert not dkf.params['validate_only'], 'cannot learn in validate only mode' assert len(dataset.shape) == 3, 'Expecting 3D tensor for data' assert dataset.shape[2] == dkf.params[ 'dim_observations'], 'Dim observations not valid' N = dataset.shape[0] idxlist = range(N) batchlist = np.split(idxlist, range(batch_size, N, batch_size)) bound_train_list,bound_valid_list,bound_tsbn_list,nll_valid_list = [],[],[],[] p_norm, g_norm, opt_norm = None, None, None #Lists used to track quantities for synthetic experiments mu_list_train, cov_list_train, mu_list_valid, cov_list_valid = [],[],[],[] model_params = {} #Start of training loop if 'synthetic' in dkf.params['dataset']: epfreq = 10 else: epfreq = 1 #Set data dkf.resetDataset(dataset, mask) for epoch in range(epoch_start, epoch_end): #Shuffle if shuffle: np.random.shuffle(idxlist) batchlist = np.split(idxlist, range(batch_size, N, batch_size)) #Always shuffle order the batches are presented in np.random.shuffle(batchlist) start_time = time.time() bound = 0 for bnum, batch_idx in enumerate(batchlist): batch_idx = batchlist[bnum] batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal = dkf.train_debug( idx=batch_idx) #Number of frames M_sum = mask[batch_idx].sum() #Correction for replicating batch if replicate_K is not None: batch_bound, negCLL, KL = batch_bound / replicate_K, negCLL / replicate_K, KL / replicate_K, M_sum = M_sum / replicate_K #Update bound bound += batch_bound ### Display ### if epoch % epfreq == 0 and bnum % 10 == 0: if normalization == 'frame': bval = batch_bound / float(M_sum) elif normalization == 'sequence': bval = batch_bound / float(X.shape[0]) else: assert False, 'Invalid normalization' dkf._p(( 'Bnum: %d, Batch Bound: %.4f, |w|: %.4f, |dw|: %.4f, |w_opt|: %.4f' ) % (bnum, bval, p_norm, g_norm, opt_norm)) dkf._p(('-veCLL:%.4f, KL:%.4f, anneal:%.4f') % (negCLL, KL, anneal)) if normalization == 'frame': bound /= (float(mask.sum()) / replicate_K) elif normalization == 'sequence': bound /= float(N) else: assert False, 'Invalid normalization' bound_train_list.append((epoch, bound)) end_time = time.time() if epoch % epfreq == 0: dkf._p(('(Ep %d) Bound: %.4f [Took %.4f seconds] ') % (epoch, bound, end_time - start_time)) #Save at intermediate stages if savefreq is not None and epoch % savefreq == 0: assert savefile is not None, 'expecting savefile' dkf._p(('Saving at epoch %d' % epoch)) dkf._saveModel(fname=savefile + '-EP' + str(epoch)) intermediate = {} if dataset_eval is not None and mask_eval is not None: tmpMap = {} bound_valid_list.append( (epoch, DKF_evaluate.evaluateBound(dkf, dataset_eval, mask_eval, batch_size=batch_size, additional=tmpMap, normalization=normalization))) bound_tsbn_list.append((epoch, tmpMap['tsbn_bound'])) nll_valid_list.append( DKF_evaluate.impSamplingNLL(dkf, dataset_eval, mask_eval, batch_size, normalization=normalization)) intermediate['valid_bound'] = np.array(bound_valid_list) intermediate['train_bound'] = np.array(bound_train_list) intermediate['tsbn_bound'] = np.array(bound_tsbn_list) intermediate['valid_nll'] = np.array(nll_valid_list) if 'synthetic' in dkf.params['dataset']: mu_train, cov_train, mu_valid, cov_valid, learned_params = _syntheticProc( dkf, dataset, mask, dataset_eval, mask_eval) if dkf.params['dim_stochastic'] == 1: mu_list_train.append(mu_train) cov_list_train.append(cov_train) mu_list_valid.append(mu_valid) cov_list_valid.append(cov_valid) intermediate['mu_posterior_train'] = np.concatenate( mu_list_train, axis=2) intermediate['cov_posterior_train'] = np.concatenate( cov_list_train, axis=2) intermediate['mu_posterior_valid'] = np.concatenate( mu_list_valid, axis=2) intermediate['cov_posterior_valid'] = np.concatenate( cov_list_valid, axis=2) else: mu_list_train.append(mu_train[None, :]) cov_list_train.append(cov_train[None, :]) mu_list_valid.append(mu_valid[None, :]) cov_list_valid.append(cov_valid[None, :]) intermediate['mu_posterior_train'] = np.concatenate( mu_list_train, axis=0) intermediate['cov_posterior_train'] = np.concatenate( cov_list_train, axis=0) intermediate['mu_posterior_valid'] = np.concatenate( mu_list_valid, axis=0) intermediate['cov_posterior_valid'] = np.concatenate( cov_list_valid, axis=0) for k in dkf.params_synthetic[dkf.params['dataset']]['params']: if k in model_params: model_params[k].append(learned_params[k]) else: model_params[k] = [learned_params[k]] for k in dkf.params_synthetic[dkf.params['dataset']]['params']: intermediate[k + '_learned'] = np.array( model_params[k]).squeeze() saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate) ### Update X in the computational flow_graph to point to training data dkf.resetDataset(dataset, mask) #Final information to be collected retMap = {} retMap['train_bound'] = np.array(bound_train_list) retMap['valid_bound'] = np.array(bound_valid_list) retMap['tsbn_bound'] = np.array(bound_tsbn_list) retMap['valid_nll'] = np.array(nll_valid_list) if 'synthetic' in dkf.params['dataset']: if dkf.params['dim_stochastic'] == 1: retMap['mu_posterior_train'] = np.concatenate(mu_list_train, axis=2) retMap['cov_posterior_train'] = np.concatenate(cov_list_train, axis=2) retMap['mu_posterior_valid'] = np.concatenate(mu_list_valid, axis=2) retMap['cov_posterior_valid'] = np.concatenate(cov_list_valid, axis=2) else: retMap['mu_posterior_train'] = np.concatenate(mu_list_train, axis=0) retMap['cov_posterior_train'] = np.concatenate(cov_list_train, axis=0) retMap['mu_posterior_valid'] = np.concatenate(mu_list_valid, axis=0) retMap['cov_posterior_valid'] = np.concatenate(cov_list_valid, axis=0) for k in dkf.params_synthetic[dkf.params['dataset']]['params']: retMap[k + '_learned'] = np.array(model_params[k]) return retMap
def learn(self, dataset, epoch_start=0, epoch_end=1000, batch_size=200, shuffle=True, savefile=None, savefreq=None, evalfreq=1, dataset_eval=None): assert len(dataset['X'].shape) == 2, 'Expecting 2D dataset matrix' assert dataset['X'].shape[1] == self.params[ 'dim_observations'], 'dim observations incorrect' assert dataset['X'].shape[0] == dataset['Y'].shape[ 0], 'len X and Y do not match' assert shuffle == True, 'Shuffle should be true, especially when using batchnorm' N = dataset['X'].shape[0] idxlist = range(N) train_crossentropy,train_accuracy,valid_crossentropy,valid_accuracy,current_lr = [],[],[],[],self.params['lr'] #Training epochs for epoch in range(epoch_start, epoch_end + 1): start_time = time.time() crossentropy, ncorrect, grad_norm, param_norm = 0, 0, [], [] #Update learning rate #current_lr = self.decay_lr() if shuffle: np.random.shuffle(idxlist) #Go through dataset for bnum, st_idx in enumerate(range(0, N, batch_size)): end_idx = min(st_idx + batch_size, N) X = self.sampleDataset( dataset['X'][idxlist[st_idx:end_idx]].astype( config.floatX)) Y = dataset['Y'][idxlist[st_idx:end_idx]].astype('int32') Nbatch = X.shape[0] #Forward/Backward pass batch_crossentropy, batch_ncorrect = self.train(X=X, Y=Y) crossentropy += batch_crossentropy ncorrect += batch_ncorrect crossentropy /= float(N) accuracy = ncorrect / float(N) train_crossentropy.append((epoch, crossentropy)) train_accuracy.append((epoch, accuracy)) end_time = time.time() self._p(( 'Ep (%d) Train CE_NLL: %.4f, Train Accuracy: %0.4f [Took %.4f seconds]' ) % (epoch, crossentropy, accuracy, (end_time - start_time))) if evalfreq is not None and epoch % evalfreq == 0: if dataset_eval is not None: v_crossentropy, v_accuracy = self.evaluateClassifier( dataset_eval, batch_size=batch_size) valid_crossentropy.append((epoch, v_crossentropy)) valid_accuracy.append((epoch, v_accuracy)) self._p( ('Ep (%d): Valid CE_NLL: %.4f, Valid Accuracy: %.4f') % (epoch, v_crossentropy, v_accuracy)) if savefreq is not None and epoch % savefreq == 0: self._p(('Saving at epoch %d' % epoch)) self._saveModel(fname=savefile + 'EP' + str(epoch)) intermediate = {} intermediate['train_crossentropy'] = np.array( train_crossentropy) intermediate['train_accuracy'] = np.array(train_accuracy) intermediate['valid_crossentropy'] = np.array( valid_crossentropy) intermediate['valid_accuracy'] = np.array(valid_accuracy) saveHDF5(savefile + 'EP' + str(epoch) + '-stats.h5', intermediate) ret_map = {} ret_map['train_crossentropy'] = np.array(train_crossentropy) ret_map['train_accuracy'] = np.array(train_accuracy) ret_map['valid_crossentropy'] = np.array(valid_crossentropy) ret_map['valid_accuracy'] = np.array(valid_accuracy) return ret_map