def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate=1e-3, learning_rate_decay=0.97,std=1e-5,decay_after=-1,trX_validation=None,trY_validation=None, trX_forecasting=None,trY_forecasting=None,trX_forecast_nodeFeatures=None,rng=np.random.RandomState(1234567890),iter_start=None, decay_type=None,decay_schedule=None,decay_rate_schedule=None, use_noise=False,noise_schedule=None,noise_rate_schedule=None, new_idx=None,featureRange=None,poseDataset=None,graph=None,maxiter=10000): from neuralmodels.loadcheckpoint import saveDRA '''If loading an existing model then some of the parameters needs to be restored''' epoch_count = 0 iterations = 0 validation_set = [] skel_loss_after_each_minibatch = [] loss_after_each_minibatch = [] complete_logger = '' if iter_start > 0: if path: lines = open('{0}logfile'.format(path)).readlines() for i in range(iter_start): line = lines[i] values = line.strip().split(',') print values if len(values) == 1: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(-1) elif len(values) == 2: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(float(values[1])) #if os.path.exists('{0}complete_log'.format(path)): # complete_logger = open('{0}complete_log'.format(path)).read() # complete_logger = complete_logger[:epoch_count] iterations = iter_start + 1 tr_X = {} tr_Y = {} Nmax = 0 outputDim = 0 unequalSize = False numExamples = {} seq_length = 0 skel_dim = 0 nodeTypes = self.nodeRNNs.keys() print "nodeTypes: ",nodeTypes for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] nodeNames = trX.keys() for nm in nodeNames: N = trX[nm].shape[1] seq_length = trX[nm].shape[0] skel_dim += trY[nm].shape[2] outputDim = trY[nm].ndim numExamples[nm] = N if Nmax == 0: Nmax = N if not Nmax == N: if N > Nmax: Nmax = N unequalSize = True if trY_forecasting is not None and new_idx is not None: trY_forecasting = self.convertToSingleVec(trY_forecasting,new_idx,featureRange) print 'trY_forecasting shape: {0}'.format(trY_forecasting.shape) assert(skel_dim == trY_forecasting.shape[2]) if unequalSize: batch_size = Nmax batches_in_one_epoch = 1 for nm in nodeNames: N = trX[nm].shape[1] batches_in_one_epoch = int(np.ceil(N*1.0 / batch_size)) break print "batches in each epoch ",batches_in_one_epoch print nodeNames #iterations = epoch_count * batches_in_one_epoch * 1.0 numrange = np.arange(Nmax) #for epoch in range(epoch_count,epochs): epoch = 0 while iterations <= maxiter: t0 = time.time() '''Learning rate decay.''' if decay_type: if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after: learning_rate *= learning_rate_decay elif decay_type == 'schedule' and decay_schedule is not None: for i in range(len(decay_schedule)): if decay_schedule[i] > 0 and iterations > decay_schedule[i]: learning_rate *= decay_rate_schedule[i] decay_schedule[i] = -1 '''Set noise level.''' if use_noise and noise_schedule is not None: for i in range(len(noise_schedule)): if noise_schedule[i] > 0 and iterations >= noise_schedule[i]: std = noise_rate_schedule[i] noise_schedule[i] = -1 '''Loading noisy data''' noisy_data = graph.readCRFgraph(poseDataset,noise=std) trX = noisy_data[8] trY = noisy_data[9] trX_validation = noisy_data[10] trY_validation = noisy_data[11] '''Permuting before mini-batch iteration''' if not unequalSize: shuffle_list = rng.permutation(numrange) for nm in nodeNames: trX[nm] = trX[nm][:,shuffle_list,:] if outputDim == 2: trY[nm] = trY[nm][:,shuffle_list] elif outputDim == 3: trY[nm] = trY[nm][:,shuffle_list,:] for j in range(batches_in_one_epoch): examples_taken_from_node = 0 for nm in nodeNames: nt = nm.split(':')[1] if(len(tr_X[nt])) == 0: examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) if outputDim == 2: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]) elif outputDim == 3: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) else: tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) if outputDim == 2: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1) elif outputDim == 3: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) loss = 0.0 skel_loss = 0.0 grad_norms = [] for nt in nodeTypes: loss_for_current_node = self.train_node[nt](tr_X[nt],tr_Y[nt],learning_rate,std) g = self.grad_norm[nt](tr_X[nt],tr_Y[nt],std) grad_norms.append(g) skel_loss_for_current_node = loss_for_current_node*tr_X[nt].shape[1]*1.0 / examples_taken_from_node loss += loss_for_current_node skel_loss += skel_loss_for_current_node iterations += 1 loss_after_each_minibatch.append(loss) validation_set.append(-1) skel_loss_after_each_minibatch.append(skel_loss) termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} normalized={3} skel_err={6}'.format(loss,epoch,j,(skel_loss*1.0/(seq_length*skel_dim)),grad_norms,learning_rate,np.sqrt(skel_loss*1.0/seq_length),std,iterations) complete_logger += termout + '\n' print termout del tr_X del tr_Y tr_X = {} tr_Y = {} for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] if int(iterations) % snapshot_rate == 0: print 'saving snapshot checkpoint.{0}'.format(int(iterations)) saveDRA(self,"{0}checkpoint.{1}".format(path,int(iterations))) '''Trajectory forecasting on validation set''' if (trX_forecasting is not None) and (trY_forecasting is not None) and path and (int(iterations) % snapshot_rate == 0): forecasted_motion = self.predict_sequence(trX_forecasting,trX_forecast_nodeFeatures,sequence_length=trY_forecasting.shape[0],poseDataset=poseDataset,graph=graph) forecasted_motion = self.convertToSingleVec(forecasted_motion,new_idx,featureRange) fname = 'forecast_iteration_{0}'.format(int(iterations)) self.saveForecastedMotion(forecasted_motion,path,fname) skel_err = np.mean(np.sqrt(np.sum(np.square((forecasted_motion - trY_forecasting)),axis=2)),axis=1) err_per_dof = skel_err / trY_forecasting.shape[2] fname = 'forecast_error_iteration_{0}'.format(int(iterations)) self.saveForecastError(skel_err,err_per_dof,path,fname) '''Computing error on validation set''' if (trX_validation is not None) and (trY_validation is not None): validation_error = 0.0 Tvalidation = 0 for nm in trX_validation.keys(): nt = nm.split(':')[1] validation_error += self.predict_node_loss[nt](trX_validation[nm],trY_validation[nm],std) Tvalidation = trX_validation[nm].shape[0] validation_set[-1] = validation_error termout = 'Validation: loss={0} normalized={1} skel_err={2}'.format(validation_error,(validation_error*1.0/(Tvalidation*skel_dim)),np.sqrt(validation_error*1.0/Tvalidation)) complete_logger += termout + '\n' print termout '''Saving the learned model so far''' if path: print 'Dir: ',path '''Writing training error and validation error in a log file''' f = open('{0}logfile'.format(path),'w') for l,v in zip(skel_loss_after_each_minibatch,validation_set): f.write('{0},{1}\n'.format(l,v)) f.close() f = open('{0}complete_log'.format(path),'w') f.write(complete_logger) f.close() t1 = time.time() termout = 'Epoch took {0} seconds'.format(t1-t0) complete_logger += termout + '\n' print termout epoch += 1
nodeName = nm.split(':')[0] prediction = self.predict_node[nt](teX[nm],1e-5) nodeFeatures[nodeName] = prediction[-1:,:,:] if len(teY[nm]) == 0: teY[nm] = copy.deepcopy(nodeFeatures[nodeName]) else: teY[nm] = np.append(teY[nm],nodeFeatures[nodeName],axis=0) for nm in nodeNames: nt = nm.split(':')[1] nodeName = nm.split(':')[0] nodeRNNFeatures = graph.getNodeFeature(nodeName,nodeFeatures,poseDataset) teX[nm] = np.append(teX[nm],nodeRNNFeatures,axis=0) del teX return teY ''' ''' losses = [] loss = 0.0 for nm in nodeNames: nt = nm.split(':')[1] losses.append(self.predict_node_loss[nt](trX[nm],trY[nm],1e-5)) loss += losses[-1] w_1 = self.nodeRNNs['torso'][0].params[0].get_value() o_1 = self.predict_node['torso'](trX['torso:torso'],1e-5) tlf_1 = self.torso_leg_features(trX['torso:torso']) taf_1 = self.torso_arm_features(trX['torso:torso']) tif_1 = self.torso_input_features(trX['torso:torso']) print '\n ****' print 'Curent model: loss={0} loss_list={1}'.format(loss,losses)
def fitModel( self, trX, trY, snapshot_rate=1, path=None, epochs=30, batch_size=50, learning_rate=1e-3, learning_rate_decay=0.97, std=1e-5, decay_after=-1, trX_validation=None, trY_validation=None, trX_forecasting=None, trY_forecasting=None, rng=np.random.RandomState(1234567890), iter_start=None, decay_type=None, decay_schedule=None, decay_rate_schedule=None, use_noise=False, noise_schedule=None, noise_rate_schedule=None, new_idx=None, featureRange=None, poseDataset=None, graph=None, maxiter=10000, predictfn=None, train_for="detection", ): from neuralmodels.loadcheckpoint import saveDRA """If loading an existing model then some of the parameters needs to be restored""" epoch_count = 0 iterations = 0 validation_set = [] skel_loss_after_each_minibatch = [] loss_after_each_minibatch = [] complete_logger = "" if iter_start > 0: if path: lines = open("{0}logfile".format(path)).readlines() for i in range(iter_start): line = lines[i] values = line.strip().split(",") print values if len(values) == 1: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(-1) elif len(values) == 2: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(float(values[1])) # if os.path.exists('{0}complete_log'.format(path)): # complete_logger = open('{0}complete_log'.format(path)).read() # complete_logger = complete_logger[:epoch_count] iterations = iter_start + 1 tr_X = {} tr_Y = {} Nmax = 0 outputDim = 0 unequalSize = False numExamples = {} seq_length = 0 skel_dim = 0 nodeTypes = self.nodeRNNs.keys() print "nodeTypes: ", nodeTypes for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] nodeNames = trX.keys() for nm in nodeNames: N = trX[nm].shape[1] seq_length = trX[nm].shape[0] outputDim = trY[nm]["detection"].ndim if outputDim > 2: skel_dim += trY[nm]["detection"].shape[2] numExamples[nm] = N if Nmax == 0: Nmax = N if not Nmax == N: if N > Nmax: Nmax = N unequalSize = True if unequalSize: batch_size = Nmax batches_in_one_epoch = 1 for nm in nodeNames: N = trX[nm].shape[1] batches_in_one_epoch = int(np.ceil(N * 1.0 / batch_size)) break print "batches in each epoch ", batches_in_one_epoch print nodeNames # iterations = epoch_count * batches_in_one_epoch * 1.0 numrange = np.arange(Nmax) # for epoch in range(epoch_count,epochs): validation_file = None if path is not None: if train_for == "joint": validation_file = open("{0}{2}_{1}".format(path, "joint_validation_acc_detection", train_for), "w") validation_file.close() validation_file = open("{0}{2}_{1}".format(path, "joint_validation_acc_anticipation", train_for), "w") validation_file.close() else: validation_file = open("{0}{2}_{1}".format(path, "validation_acc", train_for), "w") validation_file.close() epoch = 0 while iterations <= maxiter: t0 = time.time() """Learning rate decay.""" if decay_type: if decay_type == "continuous" and decay_after > 0 and epoch > decay_after: learning_rate *= learning_rate_decay elif decay_type == "schedule" and decay_schedule is not None: for i in range(len(decay_schedule)): if decay_schedule[i] > 0 and iterations > decay_schedule[i]: learning_rate *= decay_rate_schedule[i] decay_schedule[i] = -1 """Set noise level.""" if use_noise and noise_schedule is not None: for i in range(len(noise_schedule)): if noise_schedule[i] > 0 and iterations >= noise_schedule[i]: std = noise_rate_schedule[i] noise_schedule[i] = -1 """Permuting before mini-batch iteration""" if not unequalSize: shuffle_list = rng.permutation(numrange) for nm in nodeNames: trX[nm] = trX[nm][:, shuffle_list, :] if outputDim == 2: trY[nm] = trY[nm][:, shuffle_list] elif outputDim == 3: trY[nm] = trY[nm][:, shuffle_list, :] for j in range(batches_in_one_epoch): examples_taken_from_node = 0 for nm in nodeNames: nt = nm.split(":")[1] tr_X[nt] = trX[nm] tr_Y[nt] = {} tr_Y[nt]["detection"] = trY[nm]["detection"] tr_Y[nt]["anticipation"] = trY[nm]["anticipation"] """ if(len(tr_X[nt])) == 0: examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) if outputDim == 2: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]) elif outputDim == 3: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) else: tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) if outputDim == 2: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1) elif outputDim == 3: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) """ loss = 0.0 skel_loss = 0.0 grad_norms = [] losses = {} for nt in nodeTypes: loss_for_current_node = 0.0 g = [] if train_for == "joint": loss_for_current_node = self.train_node[nt]( tr_X[nt], tr_Y[nt]["detection"], tr_Y[nt]["anticipation"], learning_rate, std ) g = self.grad_norm[nt](tr_X[nt], tr_Y[nt]["detection"], tr_Y[nt]["anticipation"], std) else: loss_for_current_node = self.train_node[nt](tr_X[nt], tr_Y[nt][train_for], learning_rate, std) g = self.grad_norm[nt](tr_X[nt], tr_Y[nt][train_for], std) losses[nt] = loss_for_current_node grad_norms.append(g) loss += loss_for_current_node iterations += 1 loss_after_each_minibatch.append(loss) validation_set.append(-1) termout = "e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} H={3} O={6}".format( loss, epoch, j, (losses["H"] * 1.0 / (seq_length)), grad_norms, learning_rate, (losses["O"] * 1.0 / seq_length), std, iterations, ) complete_logger += termout + "\n" print termout del tr_X del tr_Y tr_X = {} tr_Y = {} for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] if int(iterations) % snapshot_rate == 0: print "saving snapshot checkpoint.{0}".format(int(iterations)) saveDRA(self, "{0}checkpoint.{1}".format(path, int(iterations))) """Computing error on validation set""" if ( (trX_validation is not None) and (trY_validation is not None) and (predictfn is not None) and (int(iterations) % snapshot_rate == 0) ): if train_for == "joint": predict_detection = self.predict_output( trX_validation, trY_validation["detection"], predictfn, "detection" ) predict_anticipation = self.predict_output( trX_validation, trY_validation["anticipation"], predictfn, "anticipation" ) [detection_belief, detection_labels] = self.predict_nextstep( trX_validation, trY_validation["detection"], predictfn, "detection" ) [anticipation_belief, anticipation_labels] = self.predict_nextstep( trX_validation, trY_validation["anticipation"], predictfn, "anticipation" ) validation_acc_detection = {} validation_pr_detection = {} validation_re_detection = {} for nm in predict_detection.keys(): validation_acc_detection[nm] = self.microaccuracy(predict_detection[nm]) temp = self.confusionMat(predict_detection[nm]) validation_pr_detection[nm] = temp[-2] validation_re_detection[nm] = temp[-1] validation_acc_anticipation = {} validation_pr_anticipation = {} validation_re_anticipation = {} for nm in predict_anticipation.keys(): validation_acc_anticipation[nm] = self.microaccuracy(predict_anticipation[nm]) temp = self.confusionMat(predict_anticipation[nm]) validation_pr_anticipation[nm] = temp[-2] validation_re_anticipation[nm] = temp[-1] termout = "Detection Validation: H={0} O={1}".format( validation_acc_detection["H:H"], validation_acc_detection["O:O"] ) complete_logger += termout + "\n" print termout termout = "Anticipation Validation: H={0} O={1}".format( validation_acc_anticipation["H:H"], validation_acc_anticipation["O:O"] ) complete_logger += termout + "\n" print termout validation_file = None if path is not None: validation_file = open( "{0}{2}_{1}".format(path, "joint_validation_acc_detection", train_for), "a" ) validation_file.write( "iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n".format( iterations, validation_acc_detection["H:H"], validation_acc_detection["O:O"], validation_pr_detection["H:H"], validation_re_detection["H:H"], validation_pr_detection["O:O"], validation_re_detection["O:O"], ) ) validation_file.close() if path is not None: validation_file = open( "{0}{2}_{1}".format(path, "joint_validation_acc_anticipation", train_for), "a" ) validation_file.write( "iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n".format( iterations, validation_acc_anticipation["H:H"], validation_acc_anticipation["O:O"], validation_pr_anticipation["H:H"], validation_re_anticipation["H:H"], validation_pr_anticipation["O:O"], validation_re_anticipation["O:O"], ) ) validation_file.close() if path is not None: cc = 1 for b, ypr, y_ in zip( detection_belief["H:H"], detection_labels["H:H"], trY_validation["detection"]["H:H"] ): belief_file = open("{0}detection_belief_{1}_{2}".format(path, cc, iterations), "w") for i in range(b.shape[0]): st = "{0} {1} ".format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + "{0} ".format(val) st = st.strip() + "\n" belief_file.write(st) belief_file.close() cc += 1 cc = 1 for b, ypr, y_ in zip( anticipation_belief["H:H"], anticipation_labels["H:H"], trY_validation["anticipation"]["H:H"], ): belief_file = open("{0}anticipation_belief_{1}_{2}".format(path, cc, iterations), "w") for i in range(b.shape[0]): st = "{0} {1} ".format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + "{0} ".format(val) st = st.strip() + "\n" belief_file.write(st) belief_file.close() cc += 1 cc = 1 for b, ypr, y_ in zip( detection_belief["O:O"], detection_labels["O:O"], trY_validation["detection"]["O:O"] ): belief_file = open("{0}detection_objbelief_{1}_{2}".format(path, cc, iterations), "w") for i in range(b.shape[0]): st = "{0} {1} ".format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + "{0} ".format(val) st = st.strip() + "\n" belief_file.write(st) belief_file.close() cc += 1 cc = 1 for b, ypr, y_ in zip( anticipation_belief["O:O"], anticipation_labels["O:O"], trY_validation["anticipation"]["O:O"], ): belief_file = open( "{0}anticipation_objbelief_{1}_{2}".format(path, cc, iterations), "w" ) for i in range(b.shape[0]): st = "{0} {1} ".format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + "{0} ".format(val) st = st.strip() + "\n" belief_file.write(st) belief_file.close() cc += 1 else: predict = self.predict_output( trX_validation, trY_validation[train_for], predictfn, train_for=train_for ) validation_acc = {} validation_pr = {} validation_re = {} for nm in predict.keys(): validation_acc[nm] = self.microaccuracy(predict[nm]) temp = self.confusionMat(predict[nm]) validation_pr[nm] = temp[-2] validation_re[nm] = temp[-1] termout = "Validation: H={0} O={1}".format(validation_acc["H:H"], validation_acc["O:O"]) complete_logger += termout + "\n" validation_file = None if path is not None: validation_file = open("{0}{2}_{1}".format(path, "validation_acc", train_for), "a") # validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc['H:H'],validation_acc['O:O'])) validation_file.write( "iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n".format( iterations, validation_acc["H:H"], validation_acc["O:O"], validation_pr["H:H"], validation_re["H:H"], validation_pr["O:O"], validation_re["O:O"], ) ) validation_file.close() print termout """Saving the learned model so far""" if path: print "Dir: ", path """Writing training error and validation error in a log file""" f = open("{0}logfile".format(path), "w") for l, v in zip(skel_loss_after_each_minibatch, validation_set): f.write("{0},{1}\n".format(l, v)) f.close() f = open("{0}complete_log".format(path), "w") f.write(complete_logger) f.close() t1 = time.time() termout = "Epoch took {0} seconds".format(t1 - t0) complete_logger += termout + "\n" print termout epoch += 1
def fitModel(self, trX, trY, snapshot_rate=1, path=None, epochs=30, batch_size=50, learning_rate=1e-3, learning_rate_decay=0.97, std=1e-5, decay_after=-1, trX_validation=None, trY_validation=None, trX_forecasting=None, trY_forecasting=None, rng=np.random.RandomState(1234567890), iter_start=None, decay_type=None, decay_schedule=None, decay_rate_schedule=None, use_noise=False, noise_schedule=None, noise_rate_schedule=None, new_idx=None, featureRange=None, poseDataset=None, graph=None, maxiter=10000, predictfn=None, train_for='detection'): from neuralmodels.loadcheckpoint import saveDRA '''If loading an existing model then some of the parameters needs to be restored''' epoch_count = 0 iterations = 0 validation_set = [] skel_loss_after_each_minibatch = [] loss_after_each_minibatch = [] complete_logger = '' if iter_start > 0: if path: lines = open('{0}logfile'.format(path)).readlines() for i in range(iter_start): line = lines[i] values = line.strip().split(',') print values if len(values) == 1: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(-1) elif len(values) == 2: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(float(values[1])) #if os.path.exists('{0}complete_log'.format(path)): # complete_logger = open('{0}complete_log'.format(path)).read() # complete_logger = complete_logger[:epoch_count] iterations = iter_start + 1 tr_X = {} tr_Y = {} Nmax = 0 outputDim = 0 unequalSize = False numExamples = {} seq_length = 0 skel_dim = 0 nodeTypes = self.nodeRNNs.keys() print "nodeTypes: ", nodeTypes for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] nodeNames = trX.keys() for nm in nodeNames: N = trX[nm].shape[1] seq_length = trX[nm].shape[0] outputDim = trY[nm]['detection'].ndim if outputDim > 2: skel_dim += trY[nm]['detection'].shape[2] numExamples[nm] = N if Nmax == 0: Nmax = N if not Nmax == N: if N > Nmax: Nmax = N unequalSize = True if unequalSize: batch_size = Nmax batches_in_one_epoch = 1 for nm in nodeNames: N = trX[nm].shape[1] batches_in_one_epoch = int(np.ceil(N * 1.0 / batch_size)) break print "batches in each epoch ", batches_in_one_epoch print nodeNames #iterations = epoch_count * batches_in_one_epoch * 1.0 numrange = np.arange(Nmax) #for epoch in range(epoch_count,epochs): validation_file = None if path is not None: if train_for == 'joint': validation_file = open( '{0}{2}_{1}'.format(path, 'joint_validation_acc_detection', train_for), 'w') validation_file.close() validation_file = open( '{0}{2}_{1}'.format(path, 'joint_validation_acc_anticipation', train_for), 'w') validation_file.close() else: validation_file = open( '{0}{2}_{1}'.format(path, 'validation_acc', train_for), 'w') validation_file.close() epoch = 0 while iterations <= maxiter: t0 = time.time() '''Learning rate decay.''' if decay_type: if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after: learning_rate *= learning_rate_decay elif decay_type == 'schedule' and decay_schedule is not None: for i in range(len(decay_schedule)): if decay_schedule[ i] > 0 and iterations > decay_schedule[i]: learning_rate *= decay_rate_schedule[i] decay_schedule[i] = -1 '''Set noise level.''' if use_noise and noise_schedule is not None: for i in range(len(noise_schedule)): if noise_schedule[i] > 0 and iterations >= noise_schedule[ i]: std = noise_rate_schedule[i] noise_schedule[i] = -1 '''Permuting before mini-batch iteration''' if not unequalSize: shuffle_list = rng.permutation(numrange) for nm in nodeNames: trX[nm] = trX[nm][:, shuffle_list, :] if outputDim == 2: trY[nm] = trY[nm][:, shuffle_list] elif outputDim == 3: trY[nm] = trY[nm][:, shuffle_list, :] for j in range(batches_in_one_epoch): examples_taken_from_node = 0 for nm in nodeNames: nt = nm.split(':')[1] tr_X[nt] = trX[nm] tr_Y[nt] = {} tr_Y[nt]['detection'] = trY[nm]['detection'] tr_Y[nt]['anticipation'] = trY[nm]['anticipation'] ''' if(len(tr_X[nt])) == 0: examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) if outputDim == 2: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]) elif outputDim == 3: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) else: tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) if outputDim == 2: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1) elif outputDim == 3: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) ''' loss = 0.0 skel_loss = 0.0 grad_norms = [] losses = {} for nt in nodeTypes: loss_for_current_node = 0.0 g = [] if train_for == 'joint': loss_for_current_node = self.train_node[nt]( tr_X[nt], tr_Y[nt]['detection'], tr_Y[nt]['anticipation'], learning_rate, std) g = self.grad_norm[nt](tr_X[nt], tr_Y[nt]['detection'], tr_Y[nt]['anticipation'], std) else: loss_for_current_node = self.train_node[nt]( tr_X[nt], tr_Y[nt][train_for], learning_rate, std) g = self.grad_norm[nt](tr_X[nt], tr_Y[nt][train_for], std) losses[nt] = loss_for_current_node grad_norms.append(g) loss += loss_for_current_node iterations += 1 loss_after_each_minibatch.append(loss) validation_set.append(-1) termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} H={3} O={6}'.format( loss, epoch, j, (losses['H'] * 1.0 / (seq_length)), grad_norms, learning_rate, (losses['O'] * 1.0 / seq_length), std, iterations) complete_logger += termout + '\n' print termout del tr_X del tr_Y tr_X = {} tr_Y = {} for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] if int(iterations) % snapshot_rate == 0: print 'saving snapshot checkpoint.{0}'.format( int(iterations)) saveDRA(self, "{0}checkpoint.{1}".format(path, int(iterations))) '''Computing error on validation set''' if (trX_validation is not None) and (trY_validation is not None) and ( predictfn is not None) and (int(iterations) % snapshot_rate == 0): if train_for == 'joint': predict_detection = self.predict_output( trX_validation, trY_validation['detection'], predictfn, 'detection') predict_anticipation = self.predict_output( trX_validation, trY_validation['anticipation'], predictfn, 'anticipation') [detection_belief, detection_labels ] = self.predict_nextstep(trX_validation, trY_validation['detection'], predictfn, 'detection') [anticipation_belief, anticipation_labels] = self.predict_nextstep( trX_validation, trY_validation['anticipation'], predictfn, 'anticipation') validation_acc_detection = {} validation_pr_detection = {} validation_re_detection = {} for nm in predict_detection.keys(): validation_acc_detection[nm] = self.microaccuracy( predict_detection[nm]) temp = self.confusionMat(predict_detection[nm]) validation_pr_detection[nm] = temp[-2] validation_re_detection[nm] = temp[-1] validation_acc_anticipation = {} validation_pr_anticipation = {} validation_re_anticipation = {} for nm in predict_anticipation.keys(): validation_acc_anticipation[ nm] = self.microaccuracy( predict_anticipation[nm]) temp = self.confusionMat(predict_anticipation[nm]) validation_pr_anticipation[nm] = temp[-2] validation_re_anticipation[nm] = temp[-1] termout = 'Detection Validation: H={0} O={1}'.format( validation_acc_detection['H:H'], validation_acc_detection['O:O']) complete_logger += termout + '\n' print termout termout = 'Anticipation Validation: H={0} O={1}'.format( validation_acc_anticipation['H:H'], validation_acc_anticipation['O:O']) complete_logger += termout + '\n' print termout validation_file = None if path is not None: validation_file = open( '{0}{2}_{1}'.format( path, 'joint_validation_acc_detection', train_for), 'a') validation_file.write( 'iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n'. format(iterations, validation_acc_detection['H:H'], validation_acc_detection['O:O'], validation_pr_detection['H:H'], validation_re_detection['H:H'], validation_pr_detection['O:O'], validation_re_detection['O:O'])) validation_file.close() if path is not None: validation_file = open( '{0}{2}_{1}'.format( path, 'joint_validation_acc_anticipation', train_for), 'a') validation_file.write( 'iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n'. format(iterations, validation_acc_anticipation['H:H'], validation_acc_anticipation['O:O'], validation_pr_anticipation['H:H'], validation_re_anticipation['H:H'], validation_pr_anticipation['O:O'], validation_re_anticipation['O:O'])) validation_file.close() if path is not None: cc = 1 for b, ypr, y_ in zip( detection_belief['H:H'], detection_labels['H:H'], trY_validation['detection']['H:H']): belief_file = open( '{0}detection_belief_{1}_{2}'.format( path, cc, iterations), 'w') for i in range(b.shape[0]): st = '{0} {1} '.format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + '{0} '.format(val) st = st.strip() + '\n' belief_file.write(st) belief_file.close() cc += 1 cc = 1 for b, ypr, y_ in zip( anticipation_belief['H:H'], anticipation_labels['H:H'], trY_validation['anticipation']['H:H']): belief_file = open( '{0}anticipation_belief_{1}_{2}'.format( path, cc, iterations), 'w') for i in range(b.shape[0]): st = '{0} {1} '.format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + '{0} '.format(val) st = st.strip() + '\n' belief_file.write(st) belief_file.close() cc += 1 cc = 1 for b, ypr, y_ in zip( detection_belief['O:O'], detection_labels['O:O'], trY_validation['detection']['O:O']): belief_file = open( '{0}detection_objbelief_{1}_{2}'.format( path, cc, iterations), 'w') for i in range(b.shape[0]): st = '{0} {1} '.format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + '{0} '.format(val) st = st.strip() + '\n' belief_file.write(st) belief_file.close() cc += 1 cc = 1 for b, ypr, y_ in zip( anticipation_belief['O:O'], anticipation_labels['O:O'], trY_validation['anticipation']['O:O']): belief_file = open( '{0}anticipation_objbelief_{1}_{2}'.format( path, cc, iterations), 'w') for i in range(b.shape[0]): st = '{0} {1} '.format(y_[i, 0], ypr[i, 0]) for val in b[i, :]: st = st + '{0} '.format(val) st = st.strip() + '\n' belief_file.write(st) belief_file.close() cc += 1 else: predict = self.predict_output( trX_validation, trY_validation[train_for], predictfn, train_for=train_for) validation_acc = {} validation_pr = {} validation_re = {} for nm in predict.keys(): validation_acc[nm] = self.microaccuracy( predict[nm]) temp = self.confusionMat(predict[nm]) validation_pr[nm] = temp[-2] validation_re[nm] = temp[-1] termout = 'Validation: H={0} O={1}'.format( validation_acc['H:H'], validation_acc['O:O']) complete_logger += termout + '\n' validation_file = None if path is not None: validation_file = open( '{0}{2}_{1}'.format(path, 'validation_acc', train_for), 'a') #validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc['H:H'],validation_acc['O:O'])) validation_file.write( 'iter={0} H={1} [{3};{4}] O={2} [{5};{6}]\n'. format(iterations, validation_acc['H:H'], validation_acc['O:O'], validation_pr['H:H'], validation_re['H:H'], validation_pr['O:O'], validation_re['O:O'])) validation_file.close() print termout '''Saving the learned model so far''' if path: print 'Dir: ', path '''Writing training error and validation error in a log file''' f = open('{0}logfile'.format(path), 'w') for l, v in zip(skel_loss_after_each_minibatch, validation_set): f.write('{0},{1}\n'.format(l, v)) f.close() f = open('{0}complete_log'.format(path), 'w') f.write(complete_logger) f.close() t1 = time.time() termout = 'Epoch took {0} seconds'.format(t1 - t0) complete_logger += termout + '\n' print termout epoch += 1
def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate=1e-3, learning_rate_decay=0.97,std=1e-5,decay_after=-1,trX_validation=None,trY_validation=None, trX_forecasting=None,trY_forecasting=None,rng=np.random.RandomState(1234567890),iter_start=None, decay_type=None,decay_schedule=None,decay_rate_schedule=None, use_noise=False,noise_schedule=None,noise_rate_schedule=None, new_idx=None,featureRange=None,poseDataset=None,graph=None,maxiter=10000,predictfn=None,train_for='detection'): from neuralmodels.loadcheckpoint import saveDRA '''If loading an existing model then some of the parameters needs to be restored''' epoch_count = 0 iterations = 0 validation_set = [] skel_loss_after_each_minibatch = [] loss_after_each_minibatch = [] complete_logger = '' if iter_start > 0: if path: lines = open('{0}logfile'.format(path)).readlines() for i in range(iter_start): line = lines[i] values = line.strip().split(',') print values if len(values) == 1: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(-1) elif len(values) == 2: skel_loss_after_each_minibatch.append(float(values[0])) validation_set.append(float(values[1])) #if os.path.exists('{0}complete_log'.format(path)): # complete_logger = open('{0}complete_log'.format(path)).read() # complete_logger = complete_logger[:epoch_count] iterations = iter_start + 1 tr_X = {} tr_Y = {} Nmax = 0 outputDim = 0 unequalSize = False numExamples = {} seq_length = 0 skel_dim = 0 nodeTypes = self.nodeRNNs.keys() print "nodeTypes: ",nodeTypes for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] nodeNames = trX.keys() for nm in nodeNames: N = trX[nm].shape[1] seq_length = trX[nm].shape[0] outputDim = trY[nm]['detection'].ndim if outputDim > 2: skel_dim += trY[nm]['detection'].shape[2] numExamples[nm] = N if Nmax == 0: Nmax = N if not Nmax == N: if N > Nmax: Nmax = N unequalSize = True if unequalSize: batch_size = Nmax batches_in_one_epoch = 1 for nm in nodeNames: N = trX[nm].shape[1] batches_in_one_epoch = int(np.ceil(N*1.0 / batch_size)) break print "batches in each epoch ",batches_in_one_epoch print nodeNames #iterations = epoch_count * batches_in_one_epoch * 1.0 numrange = np.arange(Nmax) #for epoch in range(epoch_count,epochs): validation_file = None if path is not None: if train_for == 'joint': validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_detection',train_for),'w') validation_file.close() validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_anticipation',train_for),'w') validation_file.close() else: validation_file = open('{0}{2}_{1}'.format(path,'validation_acc',train_for),'w') validation_file.close() epoch = 0 while iterations <= maxiter: t0 = time.time() '''Learning rate decay.''' if decay_type: if decay_type == 'continuous' and decay_after > 0 and epoch > decay_after: learning_rate *= learning_rate_decay elif decay_type == 'schedule' and decay_schedule is not None: for i in range(len(decay_schedule)): if decay_schedule[i] > 0 and iterations > decay_schedule[i]: learning_rate *= decay_rate_schedule[i] decay_schedule[i] = -1 '''Set noise level.''' if use_noise and noise_schedule is not None: for i in range(len(noise_schedule)): if noise_schedule[i] > 0 and iterations >= noise_schedule[i]: std = noise_rate_schedule[i] noise_schedule[i] = -1 '''Permuting before mini-batch iteration''' if not unequalSize: shuffle_list = rng.permutation(numrange) for nm in nodeNames: trX[nm] = trX[nm][:,shuffle_list,:] if outputDim == 2: trY[nm] = trY[nm][:,shuffle_list] elif outputDim == 3: trY[nm] = trY[nm][:,shuffle_list,:] for j in range(batches_in_one_epoch): examples_taken_from_node = 0 for nm in nodeNames: nt = nm.split(':')[1] tr_X[nt] = trX[nm] tr_Y[nt] = {} tr_Y[nt]['detection'] = trY[nm]['detection'] tr_Y[nt]['anticipation'] = trY[nm]['anticipation'] ''' if(len(tr_X[nt])) == 0: examples_taken_from_node = min((j+1)*batch_size,numExamples[nm]) - j*batch_size tr_X[nt] = copy.deepcopy(trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) if outputDim == 2: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]) elif outputDim == 3: tr_Y[nt] = copy.deepcopy(trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]) else: tr_X[nt] = np.concatenate((tr_X[nt],trX[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) if outputDim == 2: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm])]),axis=1) elif outputDim == 3: tr_Y[nt] = np.concatenate((tr_Y[nt],trY[nm][:,j*batch_size:min((j+1)*batch_size,numExamples[nm]),:]),axis=1) ''' loss = 0.0 skel_loss = 0.0 grad_norms = [] losses = {} for nt in nodeTypes: loss_for_current_node = 0.0 g = [] if train_for == 'joint': loss_for_current_node = self.train_node[nt](tr_X[nt],tr_Y[nt]['detection'],tr_Y[nt]['anticipation'],learning_rate,std) g = self.grad_norm[nt](tr_X[nt],tr_Y[nt]['detection'],tr_Y[nt]['anticipation'],std) else: loss_for_current_node = self.train_node[nt](tr_X[nt],tr_Y[nt][train_for],learning_rate,std) g = self.grad_norm[nt](tr_X[nt],tr_Y[nt][train_for],std) losses[nt] = loss_for_current_node grad_norms.append(g) loss += loss_for_current_node iterations += 1 loss_after_each_minibatch.append(loss) validation_set.append(-1) termout = 'e={1} iter={8} m={2} lr={5} g_l2={4} noise={7} loss={0} H={3} O={6}'.format(loss,epoch,j,(losses['H']*1.0/(seq_length)),grad_norms,learning_rate,(losses['O']*1.0/seq_length),std,iterations) complete_logger += termout + '\n' print termout del tr_X del tr_Y tr_X = {} tr_Y = {} for nt in nodeTypes: tr_X[nt] = [] tr_Y[nt] = [] if int(iterations) % snapshot_rate == 0: print 'saving snapshot checkpoint.{0}'.format(int(iterations)) saveDRA(self,"{0}checkpoint.{1}".format(path,int(iterations))) '''Computing error on validation set''' if (trX_validation is not None) and (trY_validation is not None) and (predictfn is not None) and (int(iterations) % snapshot_rate == 0): if train_for == 'joint': predict_detection = self.predict_output(trX_validation,trY_validation['detection'],predictfn,'detection') predict_anticipation = self.predict_output(trX_validation,trY_validation['anticipation'],predictfn,'anticipation') validation_acc_detection = {} for nm in predict_detection.keys(): validation_acc_detection[nm] = self.microaccuracy(predict_detection[nm]) validation_acc_anticipation = {} for nm in predict_anticipation.keys(): validation_acc_anticipation[nm] = self.microaccuracy(predict_anticipation[nm]) termout = 'Detection Validation: H={0} O={1}'.format(validation_acc_detection['H:H'],validation_acc_detection['O:O']) complete_logger += termout + '\n' print termout termout = 'Anticipation Validation: H={0} O={1}'.format(validation_acc_anticipation['H:H'],validation_acc_anticipation['O:O']) complete_logger += termout + '\n' print termout validation_file = None if path is not None: validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_detection',train_for),'a') validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc_detection['H:H'],validation_acc_detection['O:O'])) validation_file.close() if path is not None: validation_file = open('{0}{2}_{1}'.format(path,'joint_validation_acc_anticipation',train_for),'a') validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc_anticipation['H:H'],validation_acc_anticipation['O:O'])) validation_file.close() else: predict = self.predict_output(trX_validation,trY_validation[train_for],predictfn,train_for=train_for) validation_acc = {} for nm in predict.keys(): validation_acc[nm] = self.microaccuracy(predict[nm]) termout = 'Validation: H={0} O={1}'.format(validation_acc['H:H'],validation_acc['O:O']) complete_logger += termout + '\n' validation_file = None if path is not None: validation_file = open('{0}{2}_{1}'.format(path,'validation_acc',train_for),'a') validation_file.write('iter={0} H={1} O={2}\n'.format(iterations,validation_acc['H:H'],validation_acc['O:O'])) validation_file.close() print termout '''Saving the learned model so far''' if path: print 'Dir: ',path '''Writing training error and validation error in a log file''' f = open('{0}logfile'.format(path),'w') for l,v in zip(skel_loss_after_each_minibatch,validation_set): f.write('{0},{1}\n'.format(l,v)) f.close() f = open('{0}complete_log'.format(path),'w') f.write(complete_logger) f.close() t1 = time.time() termout = 'Epoch took {0} seconds'.format(t1-t0) complete_logger += termout + '\n' print termout epoch += 1