def learn(self,
dataset,
mask,
actions=None,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=False,
savefreq=None,
savefile=None,
dataset_eval=None,
mask_eval=None,
actions_eval=None,
replicate_K=None,
normalization='frame'):
"""
Train DKF
"""
assert not self.params[
'validate_only'], 'cannot learn in validate only mode'
assert len(dataset.shape) == 3, 'Expecting 3D tensor for data'
assert dataset.shape[2] == self.params[
'dim_observations'], 'Dim observations not valid'
N = dataset.shape[0]
idxlist = range(N)
batchlist = np.split(idxlist, range(batch_size, N, batch_size))
bound_train_list, bound_valid_list, bound_tsbn_list = [], [], []
p_norm, g_norm, opt_norm = None, None, None
#Lists used to track quantities
mu_list_train, cov_list_train, mu_list_valid, cov_list_valid, nll_valid_list = [],[],[],[],[]
current_lr = self.params['lr']
assert actions is None, 'No actions supported'
#Start of training loop
for epoch in range(epoch_start, epoch_end):
#Shuffle
if shuffle:
np.random.shuffle(idxlist)
batchlist = np.split(idxlist, range(batch_size, N, batch_size))
#Always shuffle order the batches are presented in
np.random.shuffle(batchlist)
start_time = time.time()
bound = 0
#if epoch>100 and epoch%500==0:
# self._p('ANNEALING LEARNING RATE NOW')
# current_lr = self.decay_lr()
for bnum, batch_idx in enumerate(batchlist):
batch_idx = batchlist[bnum]
X = dataset[batch_idx, :, :].astype(config.floatX)
M = mask[batch_idx, :].astype(config.floatX)
U = None
#Tack on 0's if the matrix size (optimization->theano doesnt have to redefine matrices)
if X.shape[0] < batch_size:
Nremaining = int(batch_size - X.shape[0])
X = np.concatenate(
[X, np.zeros((Nremaining, X.shape[1], X.shape[2]))],
axis=0).astype(config.floatX)
M = np.concatenate(
[M, np.zeros((Nremaining, X.shape[1]))],
axis=0).astype(config.floatX)
#Reduce the dimensionality of the tensors based on the maximum size of the mask
maxT = int(np.max(M.sum(1)))
X = X[:, :maxT, :]
M = M[:, :maxT]
eps = np.random.randn(X.shape[0], maxT,
self.params['dim_stochastic']).astype(
config.floatX)
batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal = self.train_debug(
X=X, M=M, eps=eps)
#Number of frames
M_sum = M.sum()
#Correction for replicating batch
if replicate_K is not None:
batch_bound, negCLL, KL = batch_bound / replicate_K, negCLL / replicate_K, KL / replicate_K,
M_sum = M_sum / replicate_K
#Update bound
bound += batch_bound
### Display ###
if bnum % 10 == 0:
if normalization == 'frame':
bval = batch_bound / float(M_sum)
elif normalization == 'sequence':
bval = batch_bound / float(X.shape[0])
else:
assert False, 'Invalid normalization'
if p_norm is not None:
self._p((
'Bnum:%d, Batch Bound: %.4f, |w|: %.4f, |dw|: %.4f,|w_opt|:%.4f, -veCLL:%.4f, KL:%.4f: , anneal:%.4f'
) % (bnum, bval, p_norm, g_norm, opt_norm, negCLL, KL,
anneal))
else:
self._p(('Bnum:%d, Batch Bound: %.4f') % (bnum, bval))
if normalization == 'frame':
bound /= float(mask.sum())
elif normalization == 'sequence':
bound /= float(N)
else:
assert False, 'Invalid normalization'
bound_train_list.append((epoch, bound))
end_time = time.time()
self._p(('(Ep %d) Bound: %.4f , LR: %.4e[Took %.4f seconds] ') %
(epoch, bound, current_lr, end_time - start_time))
#Save at intermediate stages
if savefreq is not None and epoch % savefreq == 0:
assert savefile is not None, 'expecting savefile'
self._p(('Saving at epoch %d' % epoch))
self._saveModel(fname=savefile + '-EP' + str(epoch))
intermediate = {}
if dataset_eval is not None and mask_eval is not None:
tmpMap = {}
#Track the validation bound and the TSBN bound
bound_valid_list.append(
(epoch,
self.evaluateBound(dataset_eval,
mask_eval,
actions=actions_eval,
batch_size=batch_size,
additional=tmpMap,
normalization=normalization)))
bound_tsbn_list.append((epoch, tmpMap['tsbn_bound']))
nll_valid_list.append(
self.impSamplingNLL(dataset_eval,
mask_eval,
batch_size,
actions=actions_eval,
normalization=normalization))
intermediate['valid_bound'] = np.array(bound_valid_list)
intermediate['train_bound'] = np.array(bound_train_list)
intermediate['tsbn_bound'] = np.array(bound_tsbn_list)
intermediate['valid_nll'] = np.array(nll_valid_list)
saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5',
intermediate)
#Final information to be collected
retMap = {}
retMap['train_bound'] = np.array(bound_train_list)
retMap['valid_bound'] = np.array(bound_valid_list)
retMap['tsbn_bound'] = np.array(bound_tsbn_list)
retMap['valid_nll'] = np.array(nll_valid_list)
if 'synthetic' in self.params['dataset']:
retMap['mu_posterior_train'] = np.concatenate(mu_list_train,
axis=2)
retMap['cov_posterior_train'] = np.concatenate(cov_list_train,
axis=2)
retMap['mu_posterior_valid'] = np.concatenate(mu_list_valid,
axis=2)
retMap['cov_posterior_valid'] = np.concatenate(cov_list_valid,
axis=2)
return retMap
dkf = DKF(params, paramFile=pfile)
displayTime('Building dkf', start_time, time.time())
"""Set save prefix"""
savef = os.path.join(params['savedir'], params['unique_id'])
print 'Savefile: ', savef
start_time = time.time()
"""Learn the model (see stinfmodel/learning.py)"""
savedata = DKF_learn.learn(dkf,
dataset['train'],
dataset['mask_train'],
epoch_start=0,
epoch_end=params['epochs'],
batch_size=params['batch_size'],
savefreq=params['savefreq'],
savefile=savef,
dataset_eval=dataset['valid'],
mask_eval=dataset['mask_valid'],
replicate_K=params['replicate_K'],
shuffle=False)
displayTime('Running DKF', start_time, time.time())
""" Evaluate bound on test set (see stinfmodel/evaluate.py)"""
savedata['bound_test'] = DKF_evaluate.evaluateBound(
dkf,
dataset['test'],
dataset['mask_test'],
batch_size=params['batch_size'])
saveHDF5(savef + '-final.h5', savedata)
print 'Test Bound: ', savedata['bound_test']
import ipdb
ipdb.set_trace()
params = readPickle(pfile)[0]
suffix = '-EP' + str(epochval[mname]) + '-params.npz'
rfile = models[mname] + suffix
assert os.path.exists(rfile), 'not found'
params['EVALUATION'] = True
if 'wikicorp' in mname:
vae = VAE(params,
paramFile=pfile,
reloadFile=rfile,
additional_attrs=additional_attrs_wiki)
trainData = dataset_wiki['train']
validData = dataset_wiki['valid']
Ntrain = trainData.shape[0]
np.random.seed(0)
trainData = trainData[np.random.permutation(Ntrain)[:100000]]
else:
vae = VAE(params,
paramFile=pfile,
reloadFile=rfile,
additional_attrs=additional_attrs_rcv2)
savef = SAVEDIR + mname
train_map_init_final = VAE_evaluate.getInitFinal(vae, trainData, 500)
train_map_init_final['data'] = trainData.toarray()
saveHDF5(savef + '-if_train.h5', train_map_init_final)
eval_map_init_final = VAE_evaluate.getInitFinal(vae, validData, 500)
eval_map_init_final['data'] = validData.toarray()
saveHDF5(savef + '-if_eval.h5', eval_map_init_final)
print 'Saved: ', mname
print 'Done'
def learn(self,
dataset,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=False,
savefile=None,
savefreq=None,
dataset_eval=None,
replicate_K=None):
assert len(dataset.shape) == 2, 'Expecting 2D dataset matrix'
assert dataset.shape[1] == self.params[
'dim_observations'], 'dim observations incorrect'
N = dataset.shape[0]
idxlist = range(N)
if shuffle:
np.random.shuffle(idxlist)
trainbound, validbound, validll, current_lr = [], [], [], self.params[
'lr']
#Training epochs
for epoch in range(epoch_start, epoch_end + 1):
start_time = time.time()
bound, grad_norm, param_norm = 0, [], []
#Update learning rate
current_lr = self.decay_lr()
#Go through dataset
for bnum, st_idx in enumerate(range(0, N, batch_size)):
end_idx = min(st_idx + batch_size, N)
X = dataset[idxlist[st_idx:end_idx]].astype(config.floatX)
Nbatch = X.shape[0]
if replicate_K is not None:
X = X.repeat(replicate_K, 0)
eps = np.random.randn(X.shape[0],
self.params['dim_stochastic']).astype(
config.floatX)
#Forward/Backward pass
if self.params['inference_model'] == 'single':
batch_bound, anneal = self.train(X=X, eps=eps)
else:
assert False, 'Should not be here'
#Divide value of the bound by replicateK
if replicate_K is not None:
batch_bound /= float(replicate_K)
bound += batch_bound
if bnum % 50 == 0:
self._p((
'--Batch: %d, Batch Bound: %.4f, Anneal : %.4f, Lr : %.6e--'
) % (bnum, batch_bound / Nbatch, anneal, current_lr))
bound /= float(N)
trainbound.append((epoch, bound))
end_time = time.time()
self._p(('Ep (%d) Upper Bound: %.4f [Took %.4f seconds]') %
(epoch, bound, (end_time - start_time)))
if savefreq is not None and epoch % savefreq == 0:
self._p(('Saving at epoch %d' % epoch))
self._saveModel(fname=savefile + '-EP' + str(epoch))
if dataset_eval is not None:
v_bound = self.evaluateBound(dataset_eval,
batch_size=batch_size)
validbound.append((epoch, v_bound))
v_ll = self.impSamplingNLL(dataset_eval,
batch_size=batch_size)
validll.append((epoch, v_ll))
self._p(('Ep (%d): Valid Bound: %.4f, Valid LL: %.4f') %
(epoch, v_bound, v_ll))
intermediate = {}
intermediate['train_bound'] = np.array(trainbound)
intermediate['valid_bound'] = np.array(validbound)
intermediate['valid_ll'] = np.array(validll)
intermediate['samples'] = self.sample()
saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5',
intermediate)
ret_map = {}
ret_map['train_bound'] = np.array(trainbound)
ret_map['valid_bound'] = np.array(validbound)
ret_map['valid_ll'] = np.array(validll)
return ret_map
savef = os.path.join(params['savedir'], params['unique_id'])
start_time = time.time()
trainData = dataset['train']
validData = dataset['valid']
savedata = Learn.learn(model,
dataset=trainData,
epoch_start=0,
epoch_end=params['epochs'],
batch_size=params['batch_size'],
savefreq=params['savefreq'],
savefile=savef,
dataset_eval=validData)
displayTime('Running Model', start_time, time.time())
saveHDF5(savef + '-final.h5', savedata)
if 'wiki' not in params['dataset']:
evaluate = {}
test_results = Evaluate.evaluateBound(model,
dataset['test'],
batch_size=params['batch_size'])
evaluate['test_perp_0'] = test_results['perp_0']
evaluate['test_perp_f'] = test_results['perp_f']
print 'Test Bound: ', evaluate['test_perp_f']
kname = 'valid_perp_f'
# Work w/ the best model thus far
epochMin, valMin, idxMin = getLowestError(savedata[kname])
reloadFile = pfile.replace('-config.pkl', '') + '-EP' + str(
int(epochMin)) + '-params.npz'
print 'Loading model from epoch : ', epochMin #reloadFile
additional_attrs_wiki['idf'] = getTF(dataset_wiki)
#additional_attrs_rcv2['idf'] = getTF(dataset_rcv2)
for mname in MODELS_TO_USE:
if 'wikicorp' not in mname:
continue
print 'Model: ', mname
pfile = models[mname].split('uid')[0] + 'uid-config.pkl'
params = readPickle(pfile)[0]
suffix = '-EP' + str(epochval[mname]) + '-params.npz'
rfile = models[mname] + suffix
assert os.path.exists(rfile), 'not found'
params['EVALUATION'] = True
if 'wikicorp' in mname:
vae = VAE(params,
paramFile=pfile,
reloadFile=rfile,
additional_attrs=additional_attrs_wiki)
else:
vae = VAE(params,
paramFile=pfile,
reloadFile=rfile,
additional_attrs=additional_attrs_rcv2)
jacob = expectedJacobian(vae, nsamples=1)
_, s, _ = np.linalg.svd(jacob)
fname = SAVEDIR + mname + '-jacob.h5'
saveHDF5(fname, {'jacob': jacob, 'svals': s})
print 'Saved: ', mname
print 'Done'
def learn(self,
dataset,
mask,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=False,
savefreq=None,
savefile=None,
dataset_eval=None,
mask_eval=None):
"""
Loop through dataset for training
"""
assert not self.params['validate_only'], 'Cannot learn in validate mode'
assert len(dataset.shape) == 3, 'Expecting 3D tensor'
assert dataset.shape[2] == self.params[
'dim_observations'], 'Dimension of observation not valid'
N = dataset.shape[0]
idxlist = range(N)
if shuffle:
np.random.shuffle(idxlist)
nll_train_list, nll_valid_list = [], []
for epoch in range(epoch_start, epoch_end):
start_time = time.time()
nll = 0
for bnum, st_idx in enumerate(range(0, N, batch_size)):
end_idx = min(st_idx + batch_size, N)
X = dataset[idxlist[st_idx:end_idx], :, :].astype(
config.floatX)
M = mask[idxlist[st_idx:end_idx], :].astype(config.floatX)
batch_nll = self.train(X=X, M=M)
nll += batch_nll
self._p(('Bnum:%d, Batch Bound: %.4f') %
(bnum, batch_nll / float(M.sum())))
nll /= float(mask.sum())
nll_train_list.append((epoch, nll))
end_time = time.time()
if bnum % 10 == 0:
self._p(('(Ep %d) NLL: %.4f [Took %.4f seconds]') %
(epoch, nll, end_time - start_time))
if savefreq is not None and epoch % savefreq == 0:
assert savefile is not None, 'expecting savefile'
self._p(('Saving at epoch %d' % epoch))
self._saveModel(fname=savefile + '-EP' + str(epoch))
if dataset_eval is not None and mask_eval is not None:
nll_valid_list.append(
(epoch,
self.evaluateNLL(dataset_eval,
mask_eval,
batch_size=batch_size)))
intermediate = {}
intermediate['train_nll'] = np.array(nll_train_list)
intermediate['valid_nll'] = np.array(nll_valid_list)
saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5',
intermediate)
retMap = {}
retMap['train_nll'] = np.array(nll_train_list)
retMap['valid_nll'] = np.array(nll_valid_list)
return retMap
def learn(dmm,
dataset,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=True,
savefreq=None,
savefile=None,
dataset_eval=None):
""" Train a DMM using data
dmm: DMM object
dataset, dataset_eval: <dict> object with data
epoch_start, epoch_end, batch_size, savefreq: <int>
savefile: <str> Savefile for intermediate results
"""
assert not dmm.params[
'validate_only'], 'cannot run function in validate only mode'
N = dataset['tensor'].shape[0]
idxlist = range(N)
batchlist = np.split(idxlist, range(batch_size, N, batch_size))
bound_train_list, bound_valid_list = [], []
epfreq = 1
dmm.resetDataset(dataset)
for epoch in range(epoch_start, epoch_end):
#Shuffle and reset batches
if shuffle:
np.random.shuffle(idxlist)
batchlist = np.split(idxlist, range(batch_size, N, batch_size))
start_time = time.time()
bound = 0
for bnum, batch_idx in enumerate(batchlist):
batch_idx = batchlist[bnum]
batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal = dmm.train_debug(
idx=batch_idx)
if np.any(
np.isnan(
np.array([
batch_bound, p_norm, g_norm, opt_norm, negCLL, KL,
anneal
]))):
print('Warning: you have encountered a NaN')
import ipdb
ipdb.set_trace()
bound += batch_bound
if epoch % epfreq == 0 and bnum % 10 == 0:
bval = batch_bound / float(dataset['mask'][batch_idx].sum())
dmm._p((
'Bnum: %d, Batch Bound: %.4f, |w|: %.4f, |dw|: %.4f, |w_opt|: %.4f'
) % (bnum, bval, p_norm, g_norm, opt_norm))
dmm._p(('-veCLL:%.4f, KL:%.4f, anneal:%.4f') %
(negCLL, KL, anneal))
bound /= float(dataset['mask'].sum())
bound_train_list.append((epoch, bound))
end_time = time.time()
if epoch % epfreq == 0:
dmm._p(('(Ep %d) Bound: %.4f [Took %.4f seconds] ') %
(epoch, bound, end_time - start_time))
if savefreq is not None and epoch % savefreq == 0:
assert savefile is not None, 'expecting savefile'
dmm._p(('Saving at epoch %d' % epoch))
dmm._saveModel(fname=savefile + '-EP' + str(epoch))
intermediate = {}
if dataset_eval is not None:
tmpMap = {}
bound_valid_list.append(
(epoch,
DMM_evaluate.evaluateBound(dmm,
dataset_eval,
batch_size=batch_size)))
intermediate['valid_bound'] = np.array(bound_valid_list)
intermediate['train_bound'] = np.array(bound_train_list)
saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate)
dmm.resetDataset(dataset)
retMap = {}
retMap['train_bound'] = np.array(bound_train_list)
retMap['valid_bound'] = np.array(bound_valid_list)
return retMap
def learn(vae,
dataset=None,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=False,
savefile=None,
savefreq=None,
dataset_eval=None):
assert dataset is not None, 'Expecting 2D dataset matrix'
assert np.prod(
dataset.shape[1:]
) == vae.params['dim_observations'], 'dim observations incorrect'
N = dataset.shape[0]
idxlist = range(N)
trainbound_0, trainbound_f = [], []
trainperp_0, trainperp_f, validperp_0, validperp_f, svallist = [],[],[],[],[]
gmulist, glcovlist, diff_elbolist, batchtimelist = [], [], [], []
gdifflists = {}
for name in vae.p_names:
gdifflists[name] = []
learnBatch = None
print 'OPT TYPE: ', vae.params['opt_type']
if vae.params['opt_type'] in ['none']:
learnBatch = _optNone
elif vae.params['opt_type'] in ['finopt']:
learnBatch = _optFinopt
elif vae.params['opt_type'] in ['finopt_none', 'none_finopt']:
learnBatch = _optMixed
else:
raise ValueError('Invalid optimization type: ' +
str(vae.params['opt_type']))
for epoch in range(epoch_start, epoch_end + 1):
np.random.shuffle(idxlist)
start_time = time.time()
bd_0, bd_f, gmu, glcov, diff_elbo, time_taken = 0, 0, 0, 0, 0, 0
""" Evaluate more frequently in the initial few epochs """
if epoch > 10:
sfreq = savefreq
tfreq = savefreq
else:
sfreq = 3
tfreq = 3
for bnum, st_idx in enumerate(range(0, N, batch_size)):
end_idx = min(st_idx + batch_size, N)
X = dataset[idxlist[st_idx:end_idx]]
if X.__class__.__name__ == 'csr_matrix' or X.__class__.__name__ == 'csc_matrix':
X = X.toarray()
X = X.astype(config.floatX)
Nbatch = X.shape[0]
update_opt = None
if vae.params['opt_type'] in ['finopt_none', 'none_finopt']:
if vae.params[
'opt_type'] == 'finopt_none': #start w/ optimizing var. params, then stop
if epoch < 10: #(epoch_end/2.):
update_opt = True
else:
update_opt = False
else: #'none_finopt' - start w/out optimizing var. params, then optimize them
if epoch < 10: #(epoch_end/2.):
update_opt = False
else:
update_opt = True
retVal = learnBatch(vae,
bnum,
Nbatch,
X,
calcELBOfinal=(epoch % tfreq == 0),
update_opt=update_opt)
bd_0 += retVal['elbo_0']
bd_f += retVal['elbo_f']
gmu += retVal['gmu']
glcov += retVal['glcov']
diff_elbo += retVal['diff_elbo']
time_taken += retVal['time_taken']
if bnum > 2:
pass #break
bd_0 /= float(N)
bd_f /= float(N)
gmu /= float(N)
glcov /= float(N)
diff_elbo /= float(N)
time_taken /= float(bnum)
batchtimelist.append((epoch, time_taken))
trainbound_0.append((epoch, bd_0))
trainbound_f.append((epoch, bd_f))
gmulist.append((epoch, gmu))
glcovlist.append((epoch, glcov))
diff_elbolist.append((epoch, diff_elbo))
end_time = time.time()
print '\n'
vae._p((
'Ep(%d) ELBO[0]: %.4f, ELBO[f]: %.4f, gmu: %.4f, glcov: %.4f, [%.3f, %.3f] [%.4f seconds]'
) % (epoch, bd_0, bd_f, gmu, glcov, diff_elbo, 0.,
(end_time - start_time)))
if savefreq is not None and epoch % sfreq == 0:
vae._p(('Saving at epoch %d' % epoch))
vae._saveModel(fname=savefile + '-EP' + str(epoch))
if dataset_eval is not None:
if vae.params['data_type'] == 'bow':
k0 = 'perp_0'
kf = 'perp_f'
else:
k0 = 'elbo_0'
kf = 'elbo_f'
eval_retVal = VAE_evaluate.evaluateBound(vae,
dataset_eval,
batch_size=batch_size)
validperp_0.append((epoch, eval_retVal[k0]))
validperp_f.append((epoch, eval_retVal[kf]))
train_retVal = VAE_evaluate.evaluateBound(
vae, dataset, batch_size=batch_size)
trainperp_0.append((epoch, train_retVal[k0]))
trainperp_f.append((epoch, train_retVal[kf]))
vae._p(('\nEp (%d): Valid[0]: %.4f, Valid[f]: %.4f') %
(epoch, eval_retVal[k0], eval_retVal[kf]))
vae._p(('\nEp (%d): Train[0]: %.4f, Train[f]: %.4f') %
(epoch, train_retVal[k0], train_retVal[kf]))
intermediate = {}
intermediate['train_bound_0'] = np.array(trainbound_0)
intermediate['train_bound_f'] = np.array(trainbound_f)
if vae.params['data_type'] == 'bow':
k0 = 'valid_perp_0'
kf = 'valid_perp_f'
else:
k0 = 'valid_bound_0'
kf = 'valid_bound_f'
intermediate[k0] = np.array(validperp_0)
intermediate[kf] = np.array(validperp_f)
intermediate[k0.replace('valid', 'train')] = np.array(trainperp_0)
intermediate[kf.replace('valid', 'train')] = np.array(trainperp_f)
intermediate['batch_time'] = np.array(batchtimelist)
intermediate['gmu'] = np.array(gmulist)
intermediate['glcov'] = np.array(glcovlist)
intermediate['diff_elbo'] = np.array(diff_elbolist)
jacob = vae.jacobian_logprobs(
np.zeros((vae.params['dim_stochastic'], )).astype('float32'))
_, svals, _ = np.linalg.svd(jacob)
epres = np.array([epoch] + svals.ravel().tolist())
svallist.append(epres)
intermediate['svals'] = np.array(svallist)
saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate)
ret_map = {}
ret_map['train_bound_0'] = np.array(trainbound_0)
ret_map['train_bound_f'] = np.array(trainbound_f)
ret_map['batch_time'] = np.array(batchtimelist)
if vae.params['data_type'] == 'bow':
k0 = 'valid_perp_0'
kf = 'valid_perp_f'
else:
k0 = 'valid_bound_0'
kf = 'valid_bound_f'
ret_map[k0] = np.array(validperp_0)
ret_map[kf] = np.array(validperp_f)
ret_map[k0.replace('valid', 'train')] = np.array(trainperp_0)
ret_map[kf.replace('valid', 'train')] = np.array(trainperp_f)
ret_map['gmu'] = np.array(gmulist)
ret_map['glcov'] = np.array(glcovlist)
ret_map['diff_elbo'] = np.array(diff_elbolist)
ret_map['svals'] = np.array(svallist)
return ret_map
def learn(dkf,
dataset,
mask,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=True,
savefreq=None,
savefile=None,
dataset_eval=None,
mask_eval=None,
replicate_K=None,
normalization='frame'):
"""
Train DKF
"""
assert not dkf.params['validate_only'], 'cannot learn in validate only mode'
assert len(dataset.shape) == 3, 'Expecting 3D tensor for data'
assert dataset.shape[2] == dkf.params[
'dim_observations'], 'Dim observations not valid'
N = dataset.shape[0]
idxlist = range(N)
batchlist = np.split(idxlist, range(batch_size, N, batch_size))
bound_train_list,bound_valid_list,bound_tsbn_list,nll_valid_list = [],[],[],[]
p_norm, g_norm, opt_norm = None, None, None
#Lists used to track quantities for synthetic experiments
mu_list_train, cov_list_train, mu_list_valid, cov_list_valid = [],[],[],[]
model_params = {}
#Start of training loop
if 'synthetic' in dkf.params['dataset']:
epfreq = 10
else:
epfreq = 1
#Set data
dkf.resetDataset(dataset, mask)
for epoch in range(epoch_start, epoch_end):
#Shuffle
if shuffle:
np.random.shuffle(idxlist)
batchlist = np.split(idxlist, range(batch_size, N, batch_size))
#Always shuffle order the batches are presented in
np.random.shuffle(batchlist)
start_time = time.time()
bound = 0
for bnum, batch_idx in enumerate(batchlist):
batch_idx = batchlist[bnum]
batch_bound, p_norm, g_norm, opt_norm, negCLL, KL, anneal = dkf.train_debug(
idx=batch_idx)
#Number of frames
M_sum = mask[batch_idx].sum()
#Correction for replicating batch
if replicate_K is not None:
batch_bound, negCLL, KL = batch_bound / replicate_K, negCLL / replicate_K, KL / replicate_K,
M_sum = M_sum / replicate_K
#Update bound
bound += batch_bound
### Display ###
if epoch % epfreq == 0 and bnum % 10 == 0:
if normalization == 'frame':
bval = batch_bound / float(M_sum)
elif normalization == 'sequence':
bval = batch_bound / float(X.shape[0])
else:
assert False, 'Invalid normalization'
dkf._p((
'Bnum: %d, Batch Bound: %.4f, |w|: %.4f, |dw|: %.4f, |w_opt|: %.4f'
) % (bnum, bval, p_norm, g_norm, opt_norm))
dkf._p(('-veCLL:%.4f, KL:%.4f, anneal:%.4f') %
(negCLL, KL, anneal))
if normalization == 'frame':
bound /= (float(mask.sum()) / replicate_K)
elif normalization == 'sequence':
bound /= float(N)
else:
assert False, 'Invalid normalization'
bound_train_list.append((epoch, bound))
end_time = time.time()
if epoch % epfreq == 0:
dkf._p(('(Ep %d) Bound: %.4f [Took %.4f seconds] ') %
(epoch, bound, end_time - start_time))
#Save at intermediate stages
if savefreq is not None and epoch % savefreq == 0:
assert savefile is not None, 'expecting savefile'
dkf._p(('Saving at epoch %d' % epoch))
dkf._saveModel(fname=savefile + '-EP' + str(epoch))
intermediate = {}
if dataset_eval is not None and mask_eval is not None:
tmpMap = {}
bound_valid_list.append(
(epoch,
DKF_evaluate.evaluateBound(dkf,
dataset_eval,
mask_eval,
batch_size=batch_size,
additional=tmpMap,
normalization=normalization)))
bound_tsbn_list.append((epoch, tmpMap['tsbn_bound']))
nll_valid_list.append(
DKF_evaluate.impSamplingNLL(dkf,
dataset_eval,
mask_eval,
batch_size,
normalization=normalization))
intermediate['valid_bound'] = np.array(bound_valid_list)
intermediate['train_bound'] = np.array(bound_train_list)
intermediate['tsbn_bound'] = np.array(bound_tsbn_list)
intermediate['valid_nll'] = np.array(nll_valid_list)
if 'synthetic' in dkf.params['dataset']:
mu_train, cov_train, mu_valid, cov_valid, learned_params = _syntheticProc(
dkf, dataset, mask, dataset_eval, mask_eval)
if dkf.params['dim_stochastic'] == 1:
mu_list_train.append(mu_train)
cov_list_train.append(cov_train)
mu_list_valid.append(mu_valid)
cov_list_valid.append(cov_valid)
intermediate['mu_posterior_train'] = np.concatenate(
mu_list_train, axis=2)
intermediate['cov_posterior_train'] = np.concatenate(
cov_list_train, axis=2)
intermediate['mu_posterior_valid'] = np.concatenate(
mu_list_valid, axis=2)
intermediate['cov_posterior_valid'] = np.concatenate(
cov_list_valid, axis=2)
else:
mu_list_train.append(mu_train[None, :])
cov_list_train.append(cov_train[None, :])
mu_list_valid.append(mu_valid[None, :])
cov_list_valid.append(cov_valid[None, :])
intermediate['mu_posterior_train'] = np.concatenate(
mu_list_train, axis=0)
intermediate['cov_posterior_train'] = np.concatenate(
cov_list_train, axis=0)
intermediate['mu_posterior_valid'] = np.concatenate(
mu_list_valid, axis=0)
intermediate['cov_posterior_valid'] = np.concatenate(
cov_list_valid, axis=0)
for k in dkf.params_synthetic[dkf.params['dataset']]['params']:
if k in model_params:
model_params[k].append(learned_params[k])
else:
model_params[k] = [learned_params[k]]
for k in dkf.params_synthetic[dkf.params['dataset']]['params']:
intermediate[k + '_learned'] = np.array(
model_params[k]).squeeze()
saveHDF5(savefile + '-EP' + str(epoch) + '-stats.h5', intermediate)
### Update X in the computational flow_graph to point to training data
dkf.resetDataset(dataset, mask)
#Final information to be collected
retMap = {}
retMap['train_bound'] = np.array(bound_train_list)
retMap['valid_bound'] = np.array(bound_valid_list)
retMap['tsbn_bound'] = np.array(bound_tsbn_list)
retMap['valid_nll'] = np.array(nll_valid_list)
if 'synthetic' in dkf.params['dataset']:
if dkf.params['dim_stochastic'] == 1:
retMap['mu_posterior_train'] = np.concatenate(mu_list_train,
axis=2)
retMap['cov_posterior_train'] = np.concatenate(cov_list_train,
axis=2)
retMap['mu_posterior_valid'] = np.concatenate(mu_list_valid,
axis=2)
retMap['cov_posterior_valid'] = np.concatenate(cov_list_valid,
axis=2)
else:
retMap['mu_posterior_train'] = np.concatenate(mu_list_train,
axis=0)
retMap['cov_posterior_train'] = np.concatenate(cov_list_train,
axis=0)
retMap['mu_posterior_valid'] = np.concatenate(mu_list_valid,
axis=0)
retMap['cov_posterior_valid'] = np.concatenate(cov_list_valid,
axis=0)
for k in dkf.params_synthetic[dkf.params['dataset']]['params']:
retMap[k + '_learned'] = np.array(model_params[k])
return retMap
def learn(self,
dataset,
epoch_start=0,
epoch_end=1000,
batch_size=200,
shuffle=True,
savefile=None,
savefreq=None,
evalfreq=1,
dataset_eval=None):
assert len(dataset['X'].shape) == 2, 'Expecting 2D dataset matrix'
assert dataset['X'].shape[1] == self.params[
'dim_observations'], 'dim observations incorrect'
assert dataset['X'].shape[0] == dataset['Y'].shape[
0], 'len X and Y do not match'
assert shuffle == True, 'Shuffle should be true, especially when using batchnorm'
N = dataset['X'].shape[0]
idxlist = range(N)
train_crossentropy,train_accuracy,valid_crossentropy,valid_accuracy,current_lr = [],[],[],[],self.params['lr']
#Training epochs
for epoch in range(epoch_start, epoch_end + 1):
start_time = time.time()
crossentropy, ncorrect, grad_norm, param_norm = 0, 0, [], []
#Update learning rate
#current_lr = self.decay_lr()
if shuffle:
np.random.shuffle(idxlist)
#Go through dataset
for bnum, st_idx in enumerate(range(0, N, batch_size)):
end_idx = min(st_idx + batch_size, N)
X = self.sampleDataset(
dataset['X'][idxlist[st_idx:end_idx]].astype(
config.floatX))
Y = dataset['Y'][idxlist[st_idx:end_idx]].astype('int32')
Nbatch = X.shape[0]
#Forward/Backward pass
batch_crossentropy, batch_ncorrect = self.train(X=X, Y=Y)
crossentropy += batch_crossentropy
ncorrect += batch_ncorrect
crossentropy /= float(N)
accuracy = ncorrect / float(N)
train_crossentropy.append((epoch, crossentropy))
train_accuracy.append((epoch, accuracy))
end_time = time.time()
self._p((
'Ep (%d) Train CE_NLL: %.4f, Train Accuracy: %0.4f [Took %.4f seconds]'
) % (epoch, crossentropy, accuracy, (end_time - start_time)))
if evalfreq is not None and epoch % evalfreq == 0:
if dataset_eval is not None:
v_crossentropy, v_accuracy = self.evaluateClassifier(
dataset_eval, batch_size=batch_size)
valid_crossentropy.append((epoch, v_crossentropy))
valid_accuracy.append((epoch, v_accuracy))
self._p(
('Ep (%d): Valid CE_NLL: %.4f, Valid Accuracy: %.4f') %
(epoch, v_crossentropy, v_accuracy))
if savefreq is not None and epoch % savefreq == 0:
self._p(('Saving at epoch %d' % epoch))
self._saveModel(fname=savefile + 'EP' + str(epoch))
intermediate = {}
intermediate['train_crossentropy'] = np.array(
train_crossentropy)
intermediate['train_accuracy'] = np.array(train_accuracy)
intermediate['valid_crossentropy'] = np.array(
valid_crossentropy)
intermediate['valid_accuracy'] = np.array(valid_accuracy)
saveHDF5(savefile + 'EP' + str(epoch) + '-stats.h5',
intermediate)
ret_map = {}
ret_map['train_crossentropy'] = np.array(train_crossentropy)
ret_map['train_accuracy'] = np.array(train_accuracy)
ret_map['valid_crossentropy'] = np.array(valid_crossentropy)
ret_map['valid_accuracy'] = np.array(valid_accuracy)
return ret_map
