def fitModel(self,
trX,
trY,
snapshot_rate=1,
path=None,
epochs=30,
batch_size=50,
learning_rate_decay=0.97,
decay_after=10):
from neuralmodels.loadcheckpoint import save
X_minibatch = []
Y_minibatch = []
num_examples = trX.shape[1]
batches_in_one_epoch = int(num_examples / batch_size)
loss_values = []
for epoch in range(epochs):
t0 = time.time()
permutation = permute(num_examples)
if self.X.ndim > 2:
trX = trX[:, permutation, :]
else:
trX = trX[:, permutation]
if self.Y.ndim > 1:
trY = trY[:, permutation]
else:
trY = trY[permutation]
for j in range(batches_in_one_epoch):
if self.X.ndim > 2:
X_minibatch = trX[:,
j * batch_size:(j + 1) * batch_size, :]
else:
X_minibatch = trX[:, j * batch_size:(j + 1) * batch_size]
if self.Y.ndim > 1:
Y_minibatch = trY[:, j * batch_size:(j + 1) * batch_size]
else:
Y_minibatch = trY[j * batch_size:(j + 1) * batch_size]
loss = self.train(X_minibatch, Y_minibatch)
loss_values.append(loss)
print "epoch={0} loss={1}".format(epoch, loss)
if path and epoch % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(epoch)
save(self, "{0}checkpoint.{1}".format(path, epoch))
f = open('{0}logfile'.format(path), 'w')
for v in loss_values:
f.write('{0}\n'.format(v))
f.close()
t1 = time.time()
print 'Epoch took {0} seconds'.format(t1 - t0)
def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate_decay=0.97,decay_after=10):
from neuralmodels.loadcheckpoint import save
X_minibatch=[]
Y_minibatch=[]
num_examples = trX.shape[1]
batches_in_one_epoch = int(num_examples / batch_size)
loss_values = []
for epoch in range(epochs):
t0 = time.time()
permutation = permute(num_examples)
if self.X.ndim > 2:
trX = trX[:,permutation,:]
else:
trX = trX[:,permutation]
if self.Y.ndim > 1:
trY = trY[:,permutation]
else:
trY = trY[permutation]
for j in range(batches_in_one_epoch):
if self.X.ndim > 2:
X_minibatch = trX[:,j*batch_size:(j+1)*batch_size,:]
else:
X_minibatch = trX[:,j*batch_size:(j+1)*batch_size]
if self.Y.ndim > 1:
Y_minibatch = trY[:,j*batch_size:(j+1)*batch_size]
else:
Y_minibatch = trY[j*batch_size:(j+1)*batch_size]
loss = self.train(X_minibatch,Y_minibatch)
loss_values.append(loss)
print "epoch={0} loss={1}".format(epoch,loss)
if path and epoch % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(epoch)
save(self,"{0}checkpoint.{1}".format(path,epoch))
f = open('{0}logfile'.format(path),'w')
for v in loss_values:
f.write('{0}\n'.format(v))
f.close()
t1 = time.time()
print 'Epoch took {0} seconds'.format(t1-t0)
def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate=1e-3,
learning_rate_decay=0.97,std=1e-5,decay_after=-1,trX_validation=None,trY_validation=None,
trX_forecasting=None,trY_forecasting=None,rng=np.random.RandomState(1234567890),iter_start=None,
decay_type=None,decay_schedule=None,decay_rate_schedule=None,
use_noise=False,noise_schedule=None,noise_rate_schedule=None,maxiter=10000):
from neuralmodels.loadcheckpoint import save
'''Saving first 5 training examples. This is for visualization of training error'''
training_trajectories = trX[:,:5,:]
if path:
fname = 'train_example'
self.saveForecastedMotion(training_trajectories,path,fname)
'''While calculating the loos we ignore these many initial time steps'''
delta_t_ignore = 0 #50
'''If loading an existing model then some of the parameters needs to be restored'''
epoch_count = 0
validation_set = []
loss_after_each_minibatch = []
complete_logger = ''
iterations = 0
if iter_start > 0:
if path:
lines = open('{0}logfile'.format(path)).readlines()
for i in range(iter_start):
line = lines[i]
values = line.strip().split(',')
print values
if len(values) == 1:
loss_after_each_minibatch.append(float(values[0]))
validation_set.append(-1)
elif len(values) == 2:
loss_after_each_minibatch.append(float(values[0]))
validation_set.append(float(values[1]))
#if os.path.exists('{0}complete_log'.format(path)):
# complete_logger = open('{0}complete_log'.format(path)).read()
# complete_logger = complete_logger[:epoch_count]
iterations = iter_start + 1
N = trX.shape[1]
outputDim = trY.ndim
seq_length = trY.shape[0] - delta_t_ignore
feature_dim = trY.shape[2]
batches_in_one_epoch = int(np.ceil(N*1.0 / batch_size))
numrange = np.arange(N)
X = []
Y = []
#iterations = epoch_count * batches_in_one_epoch * 1.0
Tvalidation = 0
Dvalidation = 0
if (trX_validation is not None):
Tvalidation = trX_validation.shape[0] - delta_t_ignore
Dvalidation = trX_validation.shape[2]
epoch = 0
print 'batches in one epoch ',batches_in_one_epoch
#for epoch in range(epoch_count,epochs):
while iterations <= maxiter:
t0 = time.time()
'''Learning rate decay.'''
if decay_type:
if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after:
learning_rate *= learning_rate_decay
elif decay_type == 'schedule' and decay_schedule is not None:
for i in range(len(decay_schedule)):
if decay_schedule[i] > 0 and iterations > decay_schedule[i]:
learning_rate *= decay_rate_schedule[i]
decay_schedule[i] = -1
'''Set noise level.'''
if use_noise and noise_schedule is not None:
for i in range(len(noise_schedule)):
if noise_schedule[i] > 0 and iterations >= noise_schedule[i]:
std = noise_rate_schedule[i]
noise_schedule[i] = -1
'''Permuting before mini-batch iteration'''
shuffle_list = rng.permutation(numrange)
trX = trX[:,shuffle_list,:]
if outputDim == 2:
trY = trY[:,shuffle_list]
elif outputDim == 3:
trY = trY[:,shuffle_list,:]
for j in range(batches_in_one_epoch):
X = trX[:,j*batch_size:min((j+1)*batch_size,N),:]
if outputDim == 2:
Y = trY[:,j*batch_size:min((j+1)*batch_size,N)]
elif outputDim == 3:
Y = trY[:,j*batch_size:min((j+1)*batch_size,N),:]
'''One iteration of training'''
loss = self.train(X,Y,learning_rate,std)
g = self.grad_norm(X,Y,std)
loss_after_each_minibatch.append(loss)
validation_set.append(-1)
iterations += 1
termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} normalized={3} skel_err={6}'.format(loss,epoch,j,(loss*1.0/(seq_length*feature_dim)),g,learning_rate,np.sqrt(loss*1.0/seq_length),std,iterations)
#termout = 'e={1} m={2} lr={5} g_l2={4} noise={7} loss={0} normalized={3} skel_err={6}'.format(loss,epoch,j,(loss*1.0/(feature_dim)),g,learning_rate,np.sqrt(loss*1.0),std)
complete_logger += termout + '\n'
print termout
'''Trajectory forecasting on validation set'''
if (trX_forecasting is not None) and (trY_forecasting is not None) and path and int(iterations) % snapshot_rate == 0:
forecasted_motion = self.predict_sequence(trX_forecasting,sequence_length=trY_forecasting.shape[0])
fname = 'forecast_iteration_{0}'.format(iterations)
self.saveForecastedMotion(forecasted_motion,path,fname)
skel_err = np.mean(np.sqrt(np.sum(np.square((forecasted_motion - trY_forecasting)),axis=2)),axis=1)
err_per_dof = skel_err / trY_forecasting.shape[2]
fname = 'forecast_error_iteration_{0}'.format(iterations)
self.saveForecastError(skel_err,err_per_dof,path,fname)
'''Saving the learned model so far'''
if path and int(iterations) % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(iterations)
save(self,"{0}checkpoint.{1}".format(path,iterations))
'''Computing error on validation set'''
if (trX_validation is not None) and (trY_validation is not None):
validation_error = self.prediction_loss(trX_validation,trY_validation,std)
validation_set[-1] = validation_error
termout = 'Validation: loss={0} normalized={1} skel_err={2}'.format(validation_error,(validation_error*1.0/(Tvalidation*Dvalidation)),np.sqrt(validation_error*1.0/Tvalidation))
complete_logger += termout + '\n'
print termout
if path:
print 'Dir: ',path
'''Writing training error and validation error in a log file'''
f = open('{0}logfile'.format(path),'w')
for l,v in zip(loss_after_each_minibatch,validation_set):
f.write('{0},{1}\n'.format(l,v))
f.close()
f = open('{0}complete_log'.format(path),'w')
f.write(complete_logger)
f.close()
'''Get error on training trajectories'''
#training_prediction = self.predict(training_trajectories,1e-5)
#fname = 'train_error_epoch_{0}'.format(epoch)
#self.saveForecastedMotion(training_prediction,path,fname)
t1 = time.time()
termout = 'Epoch took {0} seconds'.format(t1-t0)
complete_logger += termout + '\n'
print termout
epoch += 1
def fitModel(self,
trX,
trY,
snapshot_rate=1,
path=None,
epochs=30,
batch_size=50,
learning_rate=1e-3,
learning_rate_decay=0.97,
std=1e-5,
decay_after=-1,
trX_validation=None,
trY_validation=None,
trX_forecasting=None,
trY_forecasting=None,
rng=np.random.RandomState(1234567890),
iter_start=None,
decay_type=None,
decay_schedule=None,
decay_rate_schedule=None,
use_noise=False,
noise_schedule=None,
noise_rate_schedule=None,
maxiter=10000):
from neuralmodels.loadcheckpoint import save
'''Saving first 5 training examples. This is for visualization of training error'''
training_trajectories = trX[:, :5, :]
if path:
fname = 'train_example'
self.saveForecastedMotion(training_trajectories, path, fname)
'''While calculating the loos we ignore these many initial time steps'''
delta_t_ignore = 0 #50
'''If loading an existing model then some of the parameters needs to be restored'''
epoch_count = 0
validation_set = []
loss_after_each_minibatch = []
complete_logger = ''
iterations = 0
if iter_start > 0:
if path:
lines = open('{0}logfile'.format(path)).readlines()
for i in range(iter_start):
line = lines[i]
values = line.strip().split(',')
print values
if len(values) == 1:
loss_after_each_minibatch.append(float(values[0]))
validation_set.append(-1)
elif len(values) == 2:
loss_after_each_minibatch.append(float(values[0]))
validation_set.append(float(values[1]))
#if os.path.exists('{0}complete_log'.format(path)):
# complete_logger = open('{0}complete_log'.format(path)).read()
# complete_logger = complete_logger[:epoch_count]
iterations = iter_start + 1
N = trX.shape[1]
outputDim = trY.ndim
seq_length = trY.shape[0] - delta_t_ignore
feature_dim = trY.shape[2]
batches_in_one_epoch = int(np.ceil(N * 1.0 / batch_size))
numrange = np.arange(N)
X = []
Y = []
#iterations = epoch_count * batches_in_one_epoch * 1.0
Tvalidation = 0
Dvalidation = 0
if (trX_validation is not None):
Tvalidation = trX_validation.shape[0] - delta_t_ignore
Dvalidation = trX_validation.shape[2]
epoch = 0
print 'batches in one epoch ', batches_in_one_epoch
#for epoch in range(epoch_count,epochs):
while iterations <= maxiter:
t0 = time.time()
'''Learning rate decay.'''
if decay_type:
if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after:
learning_rate *= learning_rate_decay
elif decay_type == 'schedule' and decay_schedule is not None:
for i in range(len(decay_schedule)):
if decay_schedule[
i] > 0 and iterations > decay_schedule[i]:
learning_rate *= decay_rate_schedule[i]
decay_schedule[i] = -1
'''Set noise level.'''
if use_noise and noise_schedule is not None:
for i in range(len(noise_schedule)):
if noise_schedule[i] > 0 and iterations >= noise_schedule[
i]:
std = noise_rate_schedule[i]
noise_schedule[i] = -1
'''Permuting before mini-batch iteration'''
shuffle_list = rng.permutation(numrange)
trX = trX[:, shuffle_list, :]
if outputDim == 2:
trY = trY[:, shuffle_list]
elif outputDim == 3:
trY = trY[:, shuffle_list, :]
for j in range(batches_in_one_epoch):
X = trX[:, j * batch_size:min((j + 1) * batch_size, N), :]
if outputDim == 2:
Y = trY[:, j * batch_size:min((j + 1) * batch_size, N)]
elif outputDim == 3:
Y = trY[:, j * batch_size:min((j + 1) * batch_size, N), :]
'''One iteration of training'''
loss = self.train(X, Y, learning_rate, std)
g = self.grad_norm(X, Y, std)
loss_after_each_minibatch.append(loss)
validation_set.append(-1)
iterations += 1
termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} normalized={3} skel_err={6}'.format(
loss, epoch, j, (loss * 1.0 / (seq_length * feature_dim)),
g, learning_rate, np.sqrt(loss * 1.0 / seq_length), std,
iterations)
#termout = 'e={1} m={2} lr={5} g_l2={4} noise={7} loss={0} normalized={3} skel_err={6}'.format(loss,epoch,j,(loss*1.0/(feature_dim)),g,learning_rate,np.sqrt(loss*1.0),std)
complete_logger += termout + '\n'
print termout
'''Trajectory forecasting on validation set'''
if (trX_forecasting is not None) and (
trY_forecasting is not None
) and path and int(iterations) % snapshot_rate == 0:
forecasted_motion = self.predict_sequence(
trX_forecasting,
sequence_length=trY_forecasting.shape[0])
fname = 'forecast_iteration_{0}'.format(iterations)
self.saveForecastedMotion(forecasted_motion, path, fname)
skel_err = np.mean(np.sqrt(
np.sum(np.square(
(forecasted_motion - trY_forecasting)),
axis=2)),
axis=1)
err_per_dof = skel_err / trY_forecasting.shape[2]
fname = 'forecast_error_iteration_{0}'.format(iterations)
self.saveForecastError(skel_err, err_per_dof, path, fname)
'''Saving the learned model so far'''
if path and int(iterations) % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(iterations)
save(self, "{0}checkpoint.{1}".format(path, iterations))
'''Computing error on validation set'''
if (trX_validation is not None) and (trY_validation is not None):
validation_error = self.prediction_loss(
trX_validation, trY_validation, std)
validation_set[-1] = validation_error
termout = 'Validation: loss={0} normalized={1} skel_err={2}'.format(
validation_error,
(validation_error * 1.0 / (Tvalidation * Dvalidation)),
np.sqrt(validation_error * 1.0 / Tvalidation))
complete_logger += termout + '\n'
print termout
if path:
print 'Dir: ', path
'''Writing training error and validation error in a log file'''
f = open('{0}logfile'.format(path), 'w')
for l, v in zip(loss_after_each_minibatch, validation_set):
f.write('{0},{1}\n'.format(l, v))
f.close()
f = open('{0}complete_log'.format(path), 'w')
f.write(complete_logger)
f.close()
'''Get error on training trajectories'''
#training_prediction = self.predict(training_trajectories,1e-5)
#fname = 'train_error_epoch_{0}'.format(epoch)
#self.saveForecastedMotion(training_prediction,path,fname)
t1 = time.time()
termout = 'Epoch took {0} seconds'.format(t1 - t0)
complete_logger += termout + '\n'
print termout
epoch += 1
