def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate=1e-3,
learning_rate_decay=0.97,std=1e-5,decay_after=-1,trX_validation=None,trY_validation=None,
trX_forecasting=None,trY_forecasting=None,trX_forecast_nodeFeatures=None,rng=np.random.RandomState(1234567890),iter_start=None,
decay_type=None,decay_schedule=None,decay_rate_schedule=None,
use_noise=False,noise_schedule=None,noise_rate_schedule=None,
new_idx=None,featureRange=None,poseDataset=None,graph=None,maxiter=10000):
from neuralmodels.loadcheckpoint import saveDRA
'''If loading an existing model then some of the parameters needs to be restored'''
epoch_count = 0
iterations = 0
validation_set = []
skel_loss_after_each_minibatch = []
loss_after_each_minibatch = []
complete_logger = ''
if iter_start > 0:
if path:
lines = open('{0}logfile'.format(path)).readlines()
for i in range(iter_start):
line = lines[i]
values = line.strip().split(',')
print values
if len(values) == 1:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(-1)
elif len(values) == 2:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(float(values[1]))
#if os.path.exists('{0}complete_log'.format(path)):
# complete_logger = open('{0}complete_log'.format(path)).read()
# complete_logger = complete_logger[:epoch_count]
iterations = iter_start + 1
tr_X = {}
tr_Y = {}
Nmax = 0
outputDim = 0
unequalSize = False
numExamples = {}
seq_length = 0
skel_dim = 0
nodeTypes = self.nodeRNNs.keys()
print "nodeTypes: ",nodeTypes
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
nodeNames = trX.keys()
for nm in nodeNames:
N = trX[nm].shape[1]
seq_length = trX[nm].shape[0]
skel_dim += trY[nm].shape[2]
outputDim = trY[nm].ndim
numExamples[nm] = N
if Nmax == 0:
Nmax = N
if not Nmax == N:
if N > Nmax:
Nmax = N
unequalSize = True
if trY_forecasting is not None and new_idx is not None:
trY_forecasting = self.convertToSingleVec(trY_forecasting,new_idx,featureRange)
print 'trY_forecasting shape: {0}'.format(trY_forecasting.shape)
assert(skel_dim == trY_forecasting.shape[2])
if unequalSize:
batch_size = Nmax
batches_in_one_epoch = 1
for nm in nodeNames:
N = trX[nm].shape[1]
batches_in_one_epoch = int(np.ceil(N*1.0 / batch_size))
break
print "batches in each epoch ",batches_in_one_epoch
print nodeNames
#iterations = epoch_count * batches_in_one_epoch * 1.0
numrange = np.arange(Nmax)
#for epoch in range(epoch_count,epochs):
epoch = 0
while iterations <= maxiter:
t0 = time.time()
'''Learning rate decay.'''
if decay_type:
if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after:
learning_rate *= learning_rate_decay
elif decay_type == 'schedule' and decay_schedule is not None:
for i in range(len(decay_schedule)):
if decay_schedule[i] > 0 and iterations > decay_schedule[i]:
learning_rate *= decay_rate_schedule[i]
decay_schedule[i] = -1
'''Set noise level.'''
if use_noise and noise_schedule is not None:
for i in range(len(noise_schedule)):
if noise_schedule[i] > 0 and iterations >= noise_schedule[i]:
std = noise_rate_schedule[i]
noise_schedule[i] = -1
'''Loading noisy data'''
noisy_data = graph.readCRFgraph(poseDataset,noise=std)
trX = noisy_data[8]
trY = noisy_data[9]
trX_validation = noisy_data[10]
trY_validation = noisy_data[11]
'''Permuting before mini-batch iteration'''
if not unequalSize:
shuffle_list = rng.permutation(numrange)
for nm in nodeNames:
trX[nm] = trX[nm][:,shuffle_list,:]
if outputDim == 2:
trY[nm] = trY[nm][:,shuffle_list]
elif outputDim == 3:
trY[nm] = trY[nm][:,shuffle_list,:]
for j in range(batches_in_one_epoch):
examples_taken_from_node = 0
for nm in nodeNames:
nt = nm.split(':')[1]
if(len(tr_X[nt])) == 0:
examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size
tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
if outputDim == 2:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])])
elif outputDim == 3:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
else:
tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
if outputDim == 2:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1)
elif outputDim == 3:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
loss = 0.0
skel_loss = 0.0
grad_norms = []
for nt in nodeTypes:
loss_for_current_node = self.train_node[nt](tr_X[nt],tr_Y[nt],learning_rate,std)
g = self.grad_norm[nt](tr_X[nt],tr_Y[nt],std)
grad_norms.append(g)
skel_loss_for_current_node = loss_for_current_node*tr_X[nt].shape[1]*1.0 / examples_taken_from_node
loss += loss_for_current_node
skel_loss += skel_loss_for_current_node
iterations += 1
loss_after_each_minibatch.append(loss)
validation_set.append(-1)
skel_loss_after_each_minibatch.append(skel_loss)
termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} normalized={3} skel_err={6}'.format(loss,epoch,j,(skel_loss*1.0/(seq_length*skel_dim)),grad_norms,learning_rate,np.sqrt(skel_loss*1.0/seq_length),std,iterations)
complete_logger += termout + '\n'
print termout
del tr_X
del tr_Y
tr_X = {}
tr_Y = {}
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
if int(iterations) % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(int(iterations))
saveDRA(self,"{0}checkpoint.{1}".format(path,int(iterations)))
'''Trajectory forecasting on validation set'''
if (trX_forecasting is not None) and (trY_forecasting is not None) and path and (int(iterations) % snapshot_rate == 0):
forecasted_motion = self.predict_sequence(trX_forecasting,trX_forecast_nodeFeatures,sequence_length=trY_forecasting.shape[0],poseDataset=poseDataset,graph=graph)
forecasted_motion = self.convertToSingleVec(forecasted_motion,new_idx,featureRange)
fname = 'forecast_iteration_{0}'.format(int(iterations))
self.saveForecastedMotion(forecasted_motion,path,fname)
skel_err = np.mean(np.sqrt(np.sum(np.square((forecasted_motion - trY_forecasting)),axis=2)),axis=1)
err_per_dof = skel_err / trY_forecasting.shape[2]
fname = 'forecast_error_iteration_{0}'.format(int(iterations))
self.saveForecastError(skel_err,err_per_dof,path,fname)
'''Computing error on validation set'''
if (trX_validation is not None) and (trY_validation is not None):
validation_error = 0.0
Tvalidation = 0
for nm in trX_validation.keys():
nt = nm.split(':')[1]
validation_error += self.predict_node_loss[nt](trX_validation[nm],trY_validation[nm],std)
Tvalidation = trX_validation[nm].shape[0]
validation_set[-1] = validation_error
termout = 'Validation: loss={0} normalized={1} skel_err={2}'.format(validation_error,(validation_error*1.0/(Tvalidation*skel_dim)),np.sqrt(validation_error*1.0/Tvalidation))
complete_logger += termout + '\n'
print termout
'''Saving the learned model so far'''
if path:
print 'Dir: ',path
'''Writing training error and validation error in a log file'''
f = open('{0}logfile'.format(path),'w')
for l,v in zip(skel_loss_after_each_minibatch,validation_set):
f.write('{0},{1}\n'.format(l,v))
f.close()
f = open('{0}complete_log'.format(path),'w')
f.write(complete_logger)
f.close()
t1 = time.time()
termout = 'Epoch took {0} seconds'.format(t1-t0)
complete_logger += termout + '\n'
print termout
epoch += 1
nodeName = nm.split(':')[0]
prediction = self.predict_node[nt](teX[nm],1e-5)
nodeFeatures[nodeName] = prediction[-1:,:,:]
if len(teY[nm]) == 0:
teY[nm] = copy.deepcopy(nodeFeatures[nodeName])
else:
teY[nm] = np.append(teY[nm],nodeFeatures[nodeName],axis=0)
for nm in nodeNames:
nt = nm.split(':')[1]
nodeName = nm.split(':')[0]
nodeRNNFeatures = graph.getNodeFeature(nodeName,nodeFeatures,poseDataset)
teX[nm] = np.append(teX[nm],nodeRNNFeatures,axis=0)
del teX
return teY
'''
'''
losses = []
loss = 0.0
for nm in nodeNames:
nt = nm.split(':')[1]
losses.append(self.predict_node_loss[nt](trX[nm],trY[nm],1e-5))
loss += losses[-1]
w_1 = self.nodeRNNs['torso'][0].params[0].get_value()
o_1 = self.predict_node['torso'](trX['torso:torso'],1e-5)
tlf_1 = self.torso_leg_features(trX['torso:torso'])
taf_1 = self.torso_arm_features(trX['torso:torso'])
tif_1 = self.torso_input_features(trX['torso:torso'])
print '\n ****'
print 'Curent model: loss={0} loss_list={1}'.format(loss,losses)
def fitModel(
self,
trX,
trY,
snapshot_rate=1,
path=None,
epochs=30,
batch_size=50,
learning_rate=1e-3,
learning_rate_decay=0.97,
std=1e-5,
decay_after=-1,
trX_validation=None,
trY_validation=None,
trX_forecasting=None,
trY_forecasting=None,
rng=np.random.RandomState(1234567890),
iter_start=None,
decay_type=None,
decay_schedule=None,
decay_rate_schedule=None,
use_noise=False,
noise_schedule=None,
noise_rate_schedule=None,
new_idx=None,
featureRange=None,
poseDataset=None,
graph=None,
maxiter=10000,
predictfn=None,
train_for="detection",
):
from neuralmodels.loadcheckpoint import saveDRA
"""If loading an existing model then some of the parameters needs to be restored"""
epoch_count = 0
iterations = 0
validation_set = []
skel_loss_after_each_minibatch = []
loss_after_each_minibatch = []
complete_logger = ""
if iter_start > 0:
if path:
lines = open("{0}logfile".format(path)).readlines()
for i in range(iter_start):
line = lines[i]
values = line.strip().split(",")
print values
if len(values) == 1:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(-1)
elif len(values) == 2:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(float(values[1]))
# if os.path.exists('{0}complete_log'.format(path)):
# complete_logger = open('{0}complete_log'.format(path)).read()
# complete_logger = complete_logger[:epoch_count]
iterations = iter_start + 1
tr_X = {}
tr_Y = {}
Nmax = 0
outputDim = 0
unequalSize = False
numExamples = {}
seq_length = 0
skel_dim = 0
nodeTypes = self.nodeRNNs.keys()
print "nodeTypes: ", nodeTypes
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
nodeNames = trX.keys()
for nm in nodeNames:
N = trX[nm].shape[1]
seq_length = trX[nm].shape[0]
outputDim = trY[nm]["detection"].ndim
if outputDim > 2:
skel_dim += trY[nm]["detection"].shape[2]
numExamples[nm] = N
if Nmax == 0:
Nmax = N
if not Nmax == N:
if N > Nmax:
Nmax = N
unequalSize = True
if unequalSize:
batch_size = Nmax
batches_in_one_epoch = 1
for nm in nodeNames:
N = trX[nm].shape[1]
batches_in_one_epoch = int(np.ceil(N * 1.0 / batch_size))
break
print "batches in each epoch ", batches_in_one_epoch
print nodeNames
# iterations = epoch_count * batches_in_one_epoch * 1.0
numrange = np.arange(Nmax)
# for epoch in range(epoch_count,epochs):
validation_file = None
if path is not None:
if train_for == "joint":
validation_file = open("{0}{2}_{1}".format(path, "joint_validation_acc_detection", train_for), "w")
validation_file.close()
validation_file = open("{0}{2}_{1}".format(path, "joint_validation_acc_anticipation", train_for), "w")
validation_file.close()
else:
validation_file = open("{0}{2}_{1}".format(path, "validation_acc", train_for), "w")
validation_file.close()
epoch = 0
while iterations <= maxiter:
t0 = time.time()
"""Learning rate decay."""
if decay_type:
if decay_type == "continuous" and decay_after > 0 and epoch > decay_after:
learning_rate *= learning_rate_decay
elif decay_type == "schedule" and decay_schedule is not None:
for i in range(len(decay_schedule)):
if decay_schedule[i] > 0 and iterations > decay_schedule[i]:
learning_rate *= decay_rate_schedule[i]
decay_schedule[i] = -1
"""Set noise level."""
if use_noise and noise_schedule is not None:
for i in range(len(noise_schedule)):
if noise_schedule[i] > 0 and iterations >= noise_schedule[i]:
std = noise_rate_schedule[i]
noise_schedule[i] = -1
"""Permuting before mini-batch iteration"""
if not unequalSize:
shuffle_list = rng.permutation(numrange)
for nm in nodeNames:
trX[nm] = trX[nm][:, shuffle_list, :]
if outputDim == 2:
trY[nm] = trY[nm][:, shuffle_list]
elif outputDim == 3:
trY[nm] = trY[nm][:, shuffle_list, :]
for j in range(batches_in_one_epoch):
examples_taken_from_node = 0
for nm in nodeNames:
nt = nm.split(":")[1]
tr_X[nt] = trX[nm]
tr_Y[nt] = {}
tr_Y[nt]["detection"] = trY[nm]["detection"]
tr_Y[nt]["anticipation"] = trY[nm]["anticipation"]
"""
if(len(tr_X[nt])) == 0:
examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size
tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
if outputDim == 2:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])])
elif outputDim == 3:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
else:
tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
if outputDim == 2:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1)
elif outputDim == 3:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
"""
loss = 0.0
skel_loss = 0.0
grad_norms = []
losses = {}
for nt in nodeTypes:
loss_for_current_node = 0.0
g = []
if train_for == "joint":
loss_for_current_node = self.train_node[nt](
tr_X[nt], tr_Y[nt]["detection"], tr_Y[nt]["anticipation"], learning_rate, std
)
g = self.grad_norm[nt](tr_X[nt], tr_Y[nt]["detection"], tr_Y[nt]["anticipation"], std)
else:
loss_for_current_node = self.train_node[nt](tr_X[nt], tr_Y[nt][train_for], learning_rate, std)
g = self.grad_norm[nt](tr_X[nt], tr_Y[nt][train_for], std)
losses[nt] = loss_for_current_node
grad_norms.append(g)
loss += loss_for_current_node
iterations += 1
loss_after_each_minibatch.append(loss)
validation_set.append(-1)
termout = "e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} H={3} O={6}".format(
loss,
epoch,
j,
(losses["H"] * 1.0 / (seq_length)),
grad_norms,
learning_rate,
(losses["O"] * 1.0 / seq_length),
std,
iterations,
)
complete_logger += termout + "\n"
print termout
del tr_X
del tr_Y
tr_X = {}
tr_Y = {}
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
if int(iterations) % snapshot_rate == 0:
print "saving snapshot checkpoint.{0}".format(int(iterations))
saveDRA(self, "{0}checkpoint.{1}".format(path, int(iterations)))
"""Computing error on validation set"""
if (
(trX_validation is not None)
and (trY_validation is not None)
and (predictfn is not None)
and (int(iterations) % snapshot_rate == 0)
):
if train_for == "joint":
predict_detection = self.predict_output(
trX_validation, trY_validation["detection"], predictfn, "detection"
)
predict_anticipation = self.predict_output(
trX_validation, trY_validation["anticipation"], predictfn, "anticipation"
)
[detection_belief, detection_labels] = self.predict_nextstep(
trX_validation, trY_validation["detection"], predictfn, "detection"
)
[anticipation_belief, anticipation_labels] = self.predict_nextstep(
trX_validation, trY_validation["anticipation"], predictfn, "anticipation"
)
validation_acc_detection = {}
validation_pr_detection = {}
validation_re_detection = {}
for nm in predict_detection.keys():
validation_acc_detection[nm] = self.microaccuracy(predict_detection[nm])
temp = self.confusionMat(predict_detection[nm])
validation_pr_detection[nm] = temp[-2]
validation_re_detection[nm] = temp[-1]
validation_acc_anticipation = {}
validation_pr_anticipation = {}
validation_re_anticipation = {}
for nm in predict_anticipation.keys():
validation_acc_anticipation[nm] = self.microaccuracy(predict_anticipation[nm])
temp = self.confusionMat(predict_anticipation[nm])
validation_pr_anticipation[nm] = temp[-2]
validation_re_anticipation[nm] = temp[-1]
termout = "Detection Validation: H={0} O={1}".format(
validation_acc_detection["H:H"], validation_acc_detection["O:O"]
)
complete_logger += termout + "\n"
print termout
termout = "Anticipation Validation: H={0} O={1}".format(
validation_acc_anticipation["H:H"], validation_acc_anticipation["O:O"]
)
complete_logger += termout + "\n"
print termout
validation_file = None
if path is not None:
validation_file = open(
"{0}{2}_{1}".format(path, "joint_validation_acc_detection", train_for), "a"
)
validation_file.write(
"iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n".format(
iterations,
validation_acc_detection["H:H"],
validation_acc_detection["O:O"],
validation_pr_detection["H:H"],
validation_re_detection["H:H"],
validation_pr_detection["O:O"],
validation_re_detection["O:O"],
)
)
validation_file.close()
if path is not None:
validation_file = open(
"{0}{2}_{1}".format(path, "joint_validation_acc_anticipation", train_for), "a"
)
validation_file.write(
"iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n".format(
iterations,
validation_acc_anticipation["H:H"],
validation_acc_anticipation["O:O"],
validation_pr_anticipation["H:H"],
validation_re_anticipation["H:H"],
validation_pr_anticipation["O:O"],
validation_re_anticipation["O:O"],
)
)
validation_file.close()
if path is not None:
cc = 1
for b, ypr, y_ in zip(
detection_belief["H:H"], detection_labels["H:H"], trY_validation["detection"]["H:H"]
):
belief_file = open("{0}detection_belief_{1}_{2}".format(path, cc, iterations), "w")
for i in range(b.shape[0]):
st = "{0} {1} ".format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + "{0} ".format(val)
st = st.strip() + "\n"
belief_file.write(st)
belief_file.close()
cc += 1
cc = 1
for b, ypr, y_ in zip(
anticipation_belief["H:H"],
anticipation_labels["H:H"],
trY_validation["anticipation"]["H:H"],
):
belief_file = open("{0}anticipation_belief_{1}_{2}".format(path, cc, iterations), "w")
for i in range(b.shape[0]):
st = "{0} {1} ".format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + "{0} ".format(val)
st = st.strip() + "\n"
belief_file.write(st)
belief_file.close()
cc += 1
cc = 1
for b, ypr, y_ in zip(
detection_belief["O:O"], detection_labels["O:O"], trY_validation["detection"]["O:O"]
):
belief_file = open("{0}detection_objbelief_{1}_{2}".format(path, cc, iterations), "w")
for i in range(b.shape[0]):
st = "{0} {1} ".format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + "{0} ".format(val)
st = st.strip() + "\n"
belief_file.write(st)
belief_file.close()
cc += 1
cc = 1
for b, ypr, y_ in zip(
anticipation_belief["O:O"],
anticipation_labels["O:O"],
trY_validation["anticipation"]["O:O"],
):
belief_file = open(
"{0}anticipation_objbelief_{1}_{2}".format(path, cc, iterations), "w"
)
for i in range(b.shape[0]):
st = "{0} {1} ".format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + "{0} ".format(val)
st = st.strip() + "\n"
belief_file.write(st)
belief_file.close()
cc += 1
else:
predict = self.predict_output(
trX_validation, trY_validation[train_for], predictfn, train_for=train_for
)
validation_acc = {}
validation_pr = {}
validation_re = {}
for nm in predict.keys():
validation_acc[nm] = self.microaccuracy(predict[nm])
temp = self.confusionMat(predict[nm])
validation_pr[nm] = temp[-2]
validation_re[nm] = temp[-1]
termout = "Validation: H={0} O={1}".format(validation_acc["H:H"], validation_acc["O:O"])
complete_logger += termout + "\n"
validation_file = None
if path is not None:
validation_file = open("{0}{2}_{1}".format(path, "validation_acc", train_for), "a")
# validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc['H:H'],validation_acc['O:O']))
validation_file.write(
"iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n".format(
iterations,
validation_acc["H:H"],
validation_acc["O:O"],
validation_pr["H:H"],
validation_re["H:H"],
validation_pr["O:O"],
validation_re["O:O"],
)
)
validation_file.close()
print termout
"""Saving the learned model so far"""
if path:
print "Dir: ", path
"""Writing training error and validation error in a log file"""
f = open("{0}logfile".format(path), "w")
for l, v in zip(skel_loss_after_each_minibatch, validation_set):
f.write("{0},{1}\n".format(l, v))
f.close()
f = open("{0}complete_log".format(path), "w")
f.write(complete_logger)
f.close()
t1 = time.time()
termout = "Epoch took {0} seconds".format(t1 - t0)
complete_logger += termout + "\n"
print termout
epoch += 1
def fitModel(self,
trX,
trY,
snapshot_rate=1,
path=None,
epochs=30,
batch_size=50,
learning_rate=1e-3,
learning_rate_decay=0.97,
std=1e-5,
decay_after=-1,
trX_validation=None,
trY_validation=None,
trX_forecasting=None,
trY_forecasting=None,
rng=np.random.RandomState(1234567890),
iter_start=None,
decay_type=None,
decay_schedule=None,
decay_rate_schedule=None,
use_noise=False,
noise_schedule=None,
noise_rate_schedule=None,
new_idx=None,
featureRange=None,
poseDataset=None,
graph=None,
maxiter=10000,
predictfn=None,
train_for='detection'):
from neuralmodels.loadcheckpoint import saveDRA
'''If loading an existing model then some of the parameters needs to be restored'''
epoch_count = 0
iterations = 0
validation_set = []
skel_loss_after_each_minibatch = []
loss_after_each_minibatch = []
complete_logger = ''
if iter_start > 0:
if path:
lines = open('{0}logfile'.format(path)).readlines()
for i in range(iter_start):
line = lines[i]
values = line.strip().split(',')
print values
if len(values) == 1:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(-1)
elif len(values) == 2:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(float(values[1]))
#if os.path.exists('{0}complete_log'.format(path)):
# complete_logger = open('{0}complete_log'.format(path)).read()
# complete_logger = complete_logger[:epoch_count]
iterations = iter_start + 1
tr_X = {}
tr_Y = {}
Nmax = 0
outputDim = 0
unequalSize = False
numExamples = {}
seq_length = 0
skel_dim = 0
nodeTypes = self.nodeRNNs.keys()
print "nodeTypes: ", nodeTypes
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
nodeNames = trX.keys()
for nm in nodeNames:
N = trX[nm].shape[1]
seq_length = trX[nm].shape[0]
outputDim = trY[nm]['detection'].ndim
if outputDim > 2:
skel_dim += trY[nm]['detection'].shape[2]
numExamples[nm] = N
if Nmax == 0:
Nmax = N
if not Nmax == N:
if N > Nmax:
Nmax = N
unequalSize = True
if unequalSize:
batch_size = Nmax
batches_in_one_epoch = 1
for nm in nodeNames:
N = trX[nm].shape[1]
batches_in_one_epoch = int(np.ceil(N * 1.0 / batch_size))
break
print "batches in each epoch ", batches_in_one_epoch
print nodeNames
#iterations = epoch_count * batches_in_one_epoch * 1.0
numrange = np.arange(Nmax)
#for epoch in range(epoch_count,epochs):
validation_file = None
if path is not None:
if train_for == 'joint':
validation_file = open(
'{0}{2}_{1}'.format(path, 'joint_validation_acc_detection',
train_for), 'w')
validation_file.close()
validation_file = open(
'{0}{2}_{1}'.format(path,
'joint_validation_acc_anticipation',
train_for), 'w')
validation_file.close()
else:
validation_file = open(
'{0}{2}_{1}'.format(path, 'validation_acc', train_for),
'w')
validation_file.close()
epoch = 0
while iterations <= maxiter:
t0 = time.time()
'''Learning rate decay.'''
if decay_type:
if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after:
learning_rate *= learning_rate_decay
elif decay_type == 'schedule' and decay_schedule is not None:
for i in range(len(decay_schedule)):
if decay_schedule[
i] > 0 and iterations > decay_schedule[i]:
learning_rate *= decay_rate_schedule[i]
decay_schedule[i] = -1
'''Set noise level.'''
if use_noise and noise_schedule is not None:
for i in range(len(noise_schedule)):
if noise_schedule[i] > 0 and iterations >= noise_schedule[
i]:
std = noise_rate_schedule[i]
noise_schedule[i] = -1
'''Permuting before mini-batch iteration'''
if not unequalSize:
shuffle_list = rng.permutation(numrange)
for nm in nodeNames:
trX[nm] = trX[nm][:, shuffle_list, :]
if outputDim == 2:
trY[nm] = trY[nm][:, shuffle_list]
elif outputDim == 3:
trY[nm] = trY[nm][:, shuffle_list, :]
for j in range(batches_in_one_epoch):
examples_taken_from_node = 0
for nm in nodeNames:
nt = nm.split(':')[1]
tr_X[nt] = trX[nm]
tr_Y[nt] = {}
tr_Y[nt]['detection'] = trY[nm]['detection']
tr_Y[nt]['anticipation'] = trY[nm]['anticipation']
'''
if(len(tr_X[nt])) == 0:
examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size
tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
if outputDim == 2:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])])
elif outputDim == 3:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
else:
tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
if outputDim == 2:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1)
elif outputDim == 3:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
'''
loss = 0.0
skel_loss = 0.0
grad_norms = []
losses = {}
for nt in nodeTypes:
loss_for_current_node = 0.0
g = []
if train_for == 'joint':
loss_for_current_node = self.train_node[nt](
tr_X[nt], tr_Y[nt]['detection'],
tr_Y[nt]['anticipation'], learning_rate, std)
g = self.grad_norm[nt](tr_X[nt], tr_Y[nt]['detection'],
tr_Y[nt]['anticipation'], std)
else:
loss_for_current_node = self.train_node[nt](
tr_X[nt], tr_Y[nt][train_for], learning_rate, std)
g = self.grad_norm[nt](tr_X[nt], tr_Y[nt][train_for],
std)
losses[nt] = loss_for_current_node
grad_norms.append(g)
loss += loss_for_current_node
iterations += 1
loss_after_each_minibatch.append(loss)
validation_set.append(-1)
termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} H={3} O={6}'.format(
loss, epoch, j, (losses['H'] * 1.0 / (seq_length)),
grad_norms, learning_rate,
(losses['O'] * 1.0 / seq_length), std, iterations)
complete_logger += termout + '\n'
print termout
del tr_X
del tr_Y
tr_X = {}
tr_Y = {}
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
if int(iterations) % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(
int(iterations))
saveDRA(self,
"{0}checkpoint.{1}".format(path, int(iterations)))
'''Computing error on validation set'''
if (trX_validation
is not None) and (trY_validation is not None) and (
predictfn is not None) and (int(iterations) %
snapshot_rate == 0):
if train_for == 'joint':
predict_detection = self.predict_output(
trX_validation, trY_validation['detection'],
predictfn, 'detection')
predict_anticipation = self.predict_output(
trX_validation, trY_validation['anticipation'],
predictfn, 'anticipation')
[detection_belief, detection_labels
] = self.predict_nextstep(trX_validation,
trY_validation['detection'],
predictfn, 'detection')
[anticipation_belief,
anticipation_labels] = self.predict_nextstep(
trX_validation, trY_validation['anticipation'],
predictfn, 'anticipation')
validation_acc_detection = {}
validation_pr_detection = {}
validation_re_detection = {}
for nm in predict_detection.keys():
validation_acc_detection[nm] = self.microaccuracy(
predict_detection[nm])
temp = self.confusionMat(predict_detection[nm])
validation_pr_detection[nm] = temp[-2]
validation_re_detection[nm] = temp[-1]
validation_acc_anticipation = {}
validation_pr_anticipation = {}
validation_re_anticipation = {}
for nm in predict_anticipation.keys():
validation_acc_anticipation[
nm] = self.microaccuracy(
predict_anticipation[nm])
temp = self.confusionMat(predict_anticipation[nm])
validation_pr_anticipation[nm] = temp[-2]
validation_re_anticipation[nm] = temp[-1]
termout = 'Detection Validation: H={0} O={1}'.format(
validation_acc_detection['H:H'],
validation_acc_detection['O:O'])
complete_logger += termout + '\n'
print termout
termout = 'Anticipation Validation: H={0} O={1}'.format(
validation_acc_anticipation['H:H'],
validation_acc_anticipation['O:O'])
complete_logger += termout + '\n'
print termout
validation_file = None
if path is not None:
validation_file = open(
'{0}{2}_{1}'.format(
path, 'joint_validation_acc_detection',
train_for), 'a')
validation_file.write(
'iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n'.
format(iterations,
validation_acc_detection['H:H'],
validation_acc_detection['O:O'],
validation_pr_detection['H:H'],
validation_re_detection['H:H'],
validation_pr_detection['O:O'],
validation_re_detection['O:O']))
validation_file.close()
if path is not None:
validation_file = open(
'{0}{2}_{1}'.format(
path, 'joint_validation_acc_anticipation',
train_for), 'a')
validation_file.write(
'iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n'.
format(iterations,
validation_acc_anticipation['H:H'],
validation_acc_anticipation['O:O'],
validation_pr_anticipation['H:H'],
validation_re_anticipation['H:H'],
validation_pr_anticipation['O:O'],
validation_re_anticipation['O:O']))
validation_file.close()
if path is not None:
cc = 1
for b, ypr, y_ in zip(
detection_belief['H:H'],
detection_labels['H:H'],
trY_validation['detection']['H:H']):
belief_file = open(
'{0}detection_belief_{1}_{2}'.format(
path, cc, iterations), 'w')
for i in range(b.shape[0]):
st = '{0} {1} '.format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + '{0} '.format(val)
st = st.strip() + '\n'
belief_file.write(st)
belief_file.close()
cc += 1
cc = 1
for b, ypr, y_ in zip(
anticipation_belief['H:H'],
anticipation_labels['H:H'],
trY_validation['anticipation']['H:H']):
belief_file = open(
'{0}anticipation_belief_{1}_{2}'.format(
path, cc, iterations), 'w')
for i in range(b.shape[0]):
st = '{0} {1} '.format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + '{0} '.format(val)
st = st.strip() + '\n'
belief_file.write(st)
belief_file.close()
cc += 1
cc = 1
for b, ypr, y_ in zip(
detection_belief['O:O'],
detection_labels['O:O'],
trY_validation['detection']['O:O']):
belief_file = open(
'{0}detection_objbelief_{1}_{2}'.format(
path, cc, iterations), 'w')
for i in range(b.shape[0]):
st = '{0} {1} '.format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + '{0} '.format(val)
st = st.strip() + '\n'
belief_file.write(st)
belief_file.close()
cc += 1
cc = 1
for b, ypr, y_ in zip(
anticipation_belief['O:O'],
anticipation_labels['O:O'],
trY_validation['anticipation']['O:O']):
belief_file = open(
'{0}anticipation_objbelief_{1}_{2}'.format(
path, cc, iterations), 'w')
for i in range(b.shape[0]):
st = '{0} {1} '.format(y_[i, 0], ypr[i, 0])
for val in b[i, :]:
st = st + '{0} '.format(val)
st = st.strip() + '\n'
belief_file.write(st)
belief_file.close()
cc += 1
else:
predict = self.predict_output(
trX_validation,
trY_validation[train_for],
predictfn,
train_for=train_for)
validation_acc = {}
validation_pr = {}
validation_re = {}
for nm in predict.keys():
validation_acc[nm] = self.microaccuracy(
predict[nm])
temp = self.confusionMat(predict[nm])
validation_pr[nm] = temp[-2]
validation_re[nm] = temp[-1]
termout = 'Validation: H={0} O={1}'.format(
validation_acc['H:H'], validation_acc['O:O'])
complete_logger += termout + '\n'
validation_file = None
if path is not None:
validation_file = open(
'{0}{2}_{1}'.format(path, 'validation_acc',
train_for), 'a')
#validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc['H:H'],validation_acc['O:O']))
validation_file.write(
'iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n'.
format(iterations, validation_acc['H:H'],
validation_acc['O:O'],
validation_pr['H:H'],
validation_re['H:H'],
validation_pr['O:O'],
validation_re['O:O']))
validation_file.close()
print termout
'''Saving the learned model so far'''
if path:
print 'Dir: ', path
'''Writing training error and validation error in a log file'''
f = open('{0}logfile'.format(path), 'w')
for l, v in zip(skel_loss_after_each_minibatch,
validation_set):
f.write('{0},{1}\n'.format(l, v))
f.close()
f = open('{0}complete_log'.format(path), 'w')
f.write(complete_logger)
f.close()
t1 = time.time()
termout = 'Epoch took {0} seconds'.format(t1 - t0)
complete_logger += termout + '\n'
print termout
epoch += 1
def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate=1e-3,
learning_rate_decay=0.97,std=1e-5,decay_after=-1,trX_validation=None,trY_validation=None,
trX_forecasting=None,trY_forecasting=None,rng=np.random.RandomState(1234567890),iter_start=None,
decay_type=None,decay_schedule=None,decay_rate_schedule=None,
use_noise=False,noise_schedule=None,noise_rate_schedule=None,
new_idx=None,featureRange=None,poseDataset=None,graph=None,maxiter=10000,predictfn=None,train_for='detection'):
from neuralmodels.loadcheckpoint import saveDRA
'''If loading an existing model then some of the parameters needs to be restored'''
epoch_count = 0
iterations = 0
validation_set = []
skel_loss_after_each_minibatch = []
loss_after_each_minibatch = []
complete_logger = ''
if iter_start > 0:
if path:
lines = open('{0}logfile'.format(path)).readlines()
for i in range(iter_start):
line = lines[i]
values = line.strip().split(',')
print values
if len(values) == 1:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(-1)
elif len(values) == 2:
skel_loss_after_each_minibatch.append(float(values[0]))
validation_set.append(float(values[1]))
#if os.path.exists('{0}complete_log'.format(path)):
# complete_logger = open('{0}complete_log'.format(path)).read()
# complete_logger = complete_logger[:epoch_count]
iterations = iter_start + 1
tr_X = {}
tr_Y = {}
Nmax = 0
outputDim = 0
unequalSize = False
numExamples = {}
seq_length = 0
skel_dim = 0
nodeTypes = self.nodeRNNs.keys()
print "nodeTypes: ",nodeTypes
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
nodeNames = trX.keys()
for nm in nodeNames:
N = trX[nm].shape[1]
seq_length = trX[nm].shape[0]
outputDim = trY[nm]['detection'].ndim
if outputDim > 2:
skel_dim += trY[nm]['detection'].shape[2]
numExamples[nm] = N
if Nmax == 0:
Nmax = N
if not Nmax == N:
if N > Nmax:
Nmax = N
unequalSize = True
if unequalSize:
batch_size = Nmax
batches_in_one_epoch = 1
for nm in nodeNames:
N = trX[nm].shape[1]
batches_in_one_epoch = int(np.ceil(N*1.0 / batch_size))
break
print "batches in each epoch ",batches_in_one_epoch
print nodeNames
#iterations = epoch_count * batches_in_one_epoch * 1.0
numrange = np.arange(Nmax)
#for epoch in range(epoch_count,epochs):
validation_file = None
if path is not None:
if train_for == 'joint':
validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_detection',train_for),'w')
validation_file.close()
validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_anticipation',train_for),'w')
validation_file.close()
else:
validation_file = open('{0}{2}_{1}'.format(path,'validation_acc',train_for),'w')
validation_file.close()
epoch = 0
while iterations <= maxiter:
t0 = time.time()
'''Learning rate decay.'''
if decay_type:
if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after:
learning_rate *= learning_rate_decay
elif decay_type == 'schedule' and decay_schedule is not None:
for i in range(len(decay_schedule)):
if decay_schedule[i] > 0 and iterations > decay_schedule[i]:
learning_rate *= decay_rate_schedule[i]
decay_schedule[i] = -1
'''Set noise level.'''
if use_noise and noise_schedule is not None:
for i in range(len(noise_schedule)):
if noise_schedule[i] > 0 and iterations >= noise_schedule[i]:
std = noise_rate_schedule[i]
noise_schedule[i] = -1
'''Permuting before mini-batch iteration'''
if not unequalSize:
shuffle_list = rng.permutation(numrange)
for nm in nodeNames:
trX[nm] = trX[nm][:,shuffle_list,:]
if outputDim == 2:
trY[nm] = trY[nm][:,shuffle_list]
elif outputDim == 3:
trY[nm] = trY[nm][:,shuffle_list,:]
for j in range(batches_in_one_epoch):
examples_taken_from_node = 0
for nm in nodeNames:
nt = nm.split(':')[1]
tr_X[nt] = trX[nm]
tr_Y[nt] = {}
tr_Y[nt]['detection'] = trY[nm]['detection']
tr_Y[nt]['anticipation'] = trY[nm]['anticipation']
'''
if(len(tr_X[nt])) == 0:
examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size
tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
if outputDim == 2:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])])
elif outputDim == 3:
tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:])
else:
tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
if outputDim == 2:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1)
elif outputDim == 3:
tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1)
'''
loss = 0.0
skel_loss = 0.0
grad_norms = []
losses = {}
for nt in nodeTypes:
loss_for_current_node = 0.0
g = []
if train_for == 'joint':
loss_for_current_node = self.train_node[nt](tr_X[nt],tr_Y[nt]['detection'],tr_Y[nt]['anticipation'],learning_rate,std)
g = self.grad_norm[nt](tr_X[nt],tr_Y[nt]['detection'],tr_Y[nt]['anticipation'],std)
else:
loss_for_current_node = self.train_node[nt](tr_X[nt],tr_Y[nt][train_for],learning_rate,std)
g = self.grad_norm[nt](tr_X[nt],tr_Y[nt][train_for],std)
losses[nt] = loss_for_current_node
grad_norms.append(g)
loss += loss_for_current_node
iterations += 1
loss_after_each_minibatch.append(loss)
validation_set.append(-1)
termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} H={3} O={6}'.format(loss,epoch,j,(losses['H']*1.0/(seq_length)),grad_norms,learning_rate,(losses['O']*1.0/seq_length),std,iterations)
complete_logger += termout + '\n'
print termout
del tr_X
del tr_Y
tr_X = {}
tr_Y = {}
for nt in nodeTypes:
tr_X[nt] = []
tr_Y[nt] = []
if int(iterations) % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(int(iterations))
saveDRA(self,"{0}checkpoint.{1}".format(path,int(iterations)))
'''Computing error on validation set'''
if (trX_validation is not None) and (trY_validation is not None) and (predictfn is not None) and (int(iterations) % snapshot_rate == 0):
if train_for == 'joint':
predict_detection = self.predict_output(trX_validation,trY_validation['detection'],predictfn,'detection')
predict_anticipation = self.predict_output(trX_validation,trY_validation['anticipation'],predictfn,'anticipation')
validation_acc_detection = {}
for nm in predict_detection.keys():
validation_acc_detection[nm] = self.microaccuracy(predict_detection[nm])
validation_acc_anticipation = {}
for nm in predict_anticipation.keys():
validation_acc_anticipation[nm] = self.microaccuracy(predict_anticipation[nm])
termout = 'Detection Validation: H={0} O={1}'.format(validation_acc_detection['H:H'],validation_acc_detection['O:O'])
complete_logger += termout + '\n'
print termout
termout = 'Anticipation Validation: H={0} O={1}'.format(validation_acc_anticipation['H:H'],validation_acc_anticipation['O:O'])
complete_logger += termout + '\n'
print termout
validation_file = None
if path is not None:
validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_detection',train_for),'a')
validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc_detection['H:H'],validation_acc_detection['O:O']))
validation_file.close()
if path is not None:
validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_anticipation',train_for),'a')
validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc_anticipation['H:H'],validation_acc_anticipation['O:O']))
validation_file.close()
else:
predict = self.predict_output(trX_validation,trY_validation[train_for],predictfn,train_for=train_for)
validation_acc = {}
for nm in predict.keys():
validation_acc[nm] = self.microaccuracy(predict[nm])
termout = 'Validation: H={0} O={1}'.format(validation_acc['H:H'],validation_acc['O:O'])
complete_logger += termout + '\n'
validation_file = None
if path is not None:
validation_file = open('{0}{2}_{1}'.format(path,'validation_acc',train_for),'a')
validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc['H:H'],validation_acc['O:O']))
validation_file.close()
print termout
'''Saving the learned model so far'''
if path:
print 'Dir: ',path
'''Writing training error and validation error in a log file'''
f = open('{0}logfile'.format(path),'w')
for l,v in zip(skel_loss_after_each_minibatch,validation_set):
f.write('{0},{1}\n'.format(l,v))
f.close()
f = open('{0}complete_log'.format(path),'w')
f.write(complete_logger)
f.close()
t1 = time.time()
termout = 'Epoch took {0} seconds'.format(t1-t0)
complete_logger += termout + '\n'
print termout
epoch += 1
