def test_data(sess, X, Y, index_list, S_list, R_L_list, F_list, e, pre_test,
n_batches):
LStateList_F_t = ut.get_zero_state(params)
LStateList_F_pre = ut.get_zero_state(params)
LStateList_K_t = ut.get_zero_state(params, t='K')
LStateList_K_pre = ut.get_zero_state(params, t='K')
state_reset_counter_lst = [0 for i in range(batch_size)]
total_loss = 0.0
total_n_count = 0.0
for minibatch_index in xrange(n_batches):
state_reset_counter_lst = [s + 1 for s in state_reset_counter_lst]
(LStateList_F_pre,LStateList_K_pre,_,x,y,r,f,state_reset_counter_lst)=\
dut.prepare_kfl_QRFf_batch(index_list, minibatch_index, batch_size,
S_list, LStateList_F_t, LStateList_F_pre, LStateList_K_t, LStateList_K_pre,
None, None, params, Y, X, R_L_list,F_list,state_reset_counter_lst)
gt = y
mes = x
# print(r)
feed = {
tracker._z: mes,
tracker.target_data: gt,
tracker.repeat_data: r,
tracker.initial_state: LStateList_F_pre,
tracker.initial_state_Q_noise: LStateList_K_pre,
tracker.output_keep_prob: 1
}
# feed = {tracker._z: mes, tracker.target_data: gt, tracker.initial_state: LStateList_F_pre
# , tracker._P_inp: P, tracker._I: I}
LStateList_F_t,LStateList_K_t,final_output,y = \
sess.run([tracker.final_state_F,tracker.final_state_K,
tracker.final_output,tracker.y], feed)
tmp_lst = []
for item in LStateList_F_t:
tmp_lst.append(item.c)
tmp_lst.append(item.h)
LStateList_F_t = tmp_lst
tmp_lst = []
for item in LStateList_K_t:
tmp_lst.append(item.c)
tmp_lst.append(item.h)
LStateList_K_t = tmp_lst
# print(y)
# print(y.shape)
# print(final_output.shape)
if params["normalise_data"] == 3 or params["normalise_data"] == 2:
final_output = ut.unNormalizeData(final_output, params["y_men"],
params["y_std"])
y = ut.unNormalizeData(y, params["y_men"], params["y_std"])
test_loss, n_count = ut.get_loss(params, gt=y, est=final_output)
total_loss += test_loss * n_count
total_n_count += n_count
# if (minibatch_index%show_every==0):
# print pre_test+" test batch loss: (%i / %i / %i) %f"%(e,minibatch_index,n_train_batches,test_loss)
total_loss = total_loss / total_n_count
s = pre_test + ' Loss --> epoch %i | error %f' % (e, total_loss)
ut.log_write(s, params)
return total_loss
def eval(params):
batch_size = params['batch_size']
num_examples = len(params['test_files'][0])
with tf.Graph().as_default():
batch = dut.distorted_inputs(params,is_training=is_training)
with slim.arg_scope(vgg.vgg_arg_scope()):
logits, end_points = vgg.vgg_19(batch[0], num_classes=params['n_output'], is_training=is_training)
init_fn=ut.get_init_fn(slim,params)
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = params['per_process_gpu_memory_fraction']
with tf.Session(config=config) as sess:
# sess.run(tf.initialize_all_variables())
sess.run(tf.initialize_local_variables())
coord = tf.train.Coordinator()
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(sess, coord=coord, daemon=True, start=True))
init_fn(sess)
num_iter = int(math.ceil(num_examples / batch_size))
print('%s: Testing started.' % (datetime.now()))
step = 0
loss_lst=[]
run_lst=[]
run_lst.append(logits)
[run_lst.append(lst) for lst in batch[1:len(batch)]]
while step < num_iter and not coord.should_stop():
try:
batch_res= sess.run(run_lst)
except tf.errors.OutOfRangeError:
print ('Testing finished....%d'%step)
break
if(params['write_est']==True):
ut.write_est(params,batch_res)
est=batch_res[0]
gt=batch_res[1]
loss= ut.get_loss(params,gt,est)
loss_lst.append(loss)
s ='VAL --> batch %i/%i | error %f'%(step,num_iter,loss)
ut.log_write(s,params)
# joint_list=['/'.join(p1.split('/')[0:-1]).replace('joints','img').replace('.cdf','')+'/frame_'+(p1.split('/')[-1].replace('.txt','')).zfill(5)+'.png' for p1 in image_names]
# print ('List equality check:')
# print len(label_names) == len(set(label_names))
# print sum(joint_list==label_names)==(len(est))
# print(len(label_names))
step += 1
coord.request_stop()
coord.join(threads)
return np.mean(loss_lst)
def test_data(sess,params,X,Y,index_list,S_list,R_L_list,F_list,e, pre_test,n_batches):
is_test=1
dic_state=ut.get_state_list(params)
I= np.asarray([np.diag([1.0]*params['n_output']) for i in range(params["batch_size"])],dtype=np.float32)
params["reset_state"]=-1 #Never reset
state_reset_counter_lst=[0 for i in range(batch_size)]
total_loss=0.0
total_pred_loss=0.0
total_meas_loss=0.0
total_n_count=0.0
for minibatch_index in xrange(n_batches):
state_reset_counter_lst=[s+1 for s in state_reset_counter_lst]
# print state_reset_counter_lst
(dic_state,x,y,r,f,_,state_reset_counter_lst,_)= \
th.prepare_batch(is_test,index_list, minibatch_index, batch_size,
S_list, dic_state, params, Y, X, R_L_list,F_list,state_reset_counter_lst)
feed=th.get_feed(Model,params,r,x,y,I,dic_state, is_training=0)
states,final_output,final_pred_output,final_meas_output,y =sess.run([Model.states,Model.final_output,Model.final_pred_output,Model.final_meas_output,Model.y], feed)
for k in states.keys():
dic_state[k] = states[k]
if params["normalise_data"]==3 or params["normalise_data"]==2:
final_output=ut.unNormalizeData(final_output,params["y_men"],params["y_std"])
final_pred_output=ut.unNormalizeData(final_pred_output,params["y_men"],params["y_std"])
final_meas_output=ut.unNormalizeData(final_meas_output,params["x_men"],params["x_std"])
y=ut.unNormalizeData(y,params["y_men"],params["y_std"])
if params["normalise_data"]==4:
final_output=ut.unNormalizeData(final_output,params["x_men"],params["x_std"])
final_pred_output=ut.unNormalizeData(final_pred_output,params["x_men"],params["x_std"])
final_meas_output=ut.unNormalizeData(final_meas_output,params["x_men"],params["x_std"])
y=ut.unNormalizeData(y,params["x_men"],params["x_std"])
test_loss,n_count=ut.get_loss(params,gt=y,est=final_output,r=r)
test_pred_loss,n_count=ut.get_loss(params,gt=y,est=final_pred_output,r=r)
test_meas_loss,n_count=ut.get_loss(params,gt=y,est=final_meas_output,r=r)
total_loss+=test_loss*n_count
total_pred_loss+=test_pred_loss*n_count
total_meas_loss+=test_meas_loss*n_count
total_n_count+=n_count
# if (minibatch_index%show_every==0):
# print pre_test+" test batch loss: (%i / %i / %i) %f"%(e,minibatch_index,n_train_batches,test_loss)
total_loss=total_loss/total_n_count
total_pred_loss=total_pred_loss/total_n_count
total_meas_loss=total_meas_loss/total_n_count
s =pre_test+' Loss --> epoch %i | error %f, %f, %f'%(e,total_loss,total_pred_loss,total_meas_loss)
ut.log_write(s,params)
return total_loss
def test_data(sess, X, Y, index_list, S_list, R_L_list, F_list, e, pre_test,
n_batches):
state_reset_counter_lst = [0 for i in range(batch_size)]
total_loss = 0.0
total_losss = 0.0
total_n_count = 0.0
total_n_countt = 0.0
for minibatch_index in xrange(n_batches):
state_reset_counter_lst = [s + 1 for s in state_reset_counter_lst]
x = X[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
feed = {
model.input_data: x,
model.target_data: y,
model.is_training: False,
model.output_keep_prob: 1.0
}
final_output = sess.run([model.final_output], feed)
final_output = final_output[0]
test_loss, n_count = ut.get_loss(params, gt=y, est=final_output)
test_losss, n_countt = ut.get_loss(params, gt=y, est=x)
total_loss += test_loss * n_count
total_losss += test_losss * n_countt
total_n_count += n_count
total_n_countt += n_countt
if (minibatch_index % show_every == 0):
print pre_test + " test batch loss: (%i / %i / %i) %f" % (
e, minibatch_index, n_train_batches, test_loss)
total_loss = total_loss / total_n_count
total_losss = total_losss / total_n_countt
s = pre_test + ' Loss --> epoch %i | error %f, %f' % (e, total_loss,
total_losss)
ut.log_write(s, params)
return total_loss
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train, Y_train, S_Train_list, F_list_train, G_list_train, X_test,
Y_test, S_Test_list, F_list_test, G_list_test) = du.load_pose(params)
params["len_train"] = len(X_train)
params["len_test"] = len(X_test)
u.start_log(params)
batch_size = params['batch_size']
n_train_batches = params["len_train"]
n_train_batches /= batch_size
n_test_batches = params["len_test"]
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['run_mode'] == 1:
model = model_provider.get_model_pretrained(params, rng)
u.log_write("Pretrained loaded: %s" % (params['mfile']), params)
else:
model = model_provider.get_model(params, rng)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
sid = 0
is_train = 1
for minibatch_index in range(n_train_batches):
x = X_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y, is_train)
batch_loss += loss
if params['shufle_data'] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 3 == 0):
print("Model testing")
batch_loss3d = []
is_train = 0
for minibatch_index in range(n_test_batches):
x = X_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
pred = model.predictions(x, is_train)
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
batch_loss3d = np.nanmean(batch_loss3d)
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(epoch_counter) + "_" + str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 #0.08
s = 'VAL--> epoch %i | error %f, %f' % (val_counter, batch_loss3d,
n_test_batches)
u.log_write(s, params)
def train():
model = Model(params)
num_epochs = 1000
decay_rate = 0.4
show_every = 100
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = params[
'per_process_gpu_memory_fraction']
with tf.Session(config=config) as sess:
tf.initialize_all_variables().run()
saver = tf.train.Saver(tf.all_variables())
merged = tf.summary.merge_all()
summary_writer = tf.train.SummaryWriter(params["sm"], sess.graph)
for e in xrange(num_epochs):
sess.run(tf.assign(model.lr, params['lr'] * (decay_rate**e)))
LStateList_t = [
np.zeros(shape=(batch_size, params['n_hidden']),
dtype=np.float32) for i in range(params['nlayer'] * 2)
] # initial hidden state
LStateList_pre = [
np.zeros(shape=(batch_size, params['n_hidden']),
dtype=np.float32) for i in range(params['nlayer'] * 2)
] # initial hidden sta
state_reset_counter_lst = [0 for i in range(batch_size)]
total_train_loss = 0
for minibatch_index in xrange(n_train_batches):
start = time.time()
state_reset_counter_lst = [
s + 1 for s in state_reset_counter_lst
]
(LStateList_b, x, y,
state_reset_counter_lst) = dut.prepare_lstm_batch_joints(
index_train_list, minibatch_index, batch_size,
S_Train_list, LStateList_t, LStateList_pre, params,
F_names_training, state_reset_counter_lst)
LStateList_pre = LStateList_b
y = y.reshape(batch_size * params["seq_length"],
params["n_output"])
feed = {
model.input_data: x,
model.input_zero: np.ceil(x),
model.target_data: y,
model.initial_state: LStateList_b,
model.is_training: True,
model.output_keep_prob: 0.5
}
summary,train_loss, LStateList_t,_ =\
sess.run([merged,model.cost, model.final_state, model.train_op], feed)
summary_writer.add_summary(summary, minibatch_index)
tmp_lst = []
for item in LStateList_t:
tmp_lst.append(item.c)
tmp_lst.append(item.h)
LStateList_t = tmp_lst
total_train_loss += train_loss
if (minibatch_index % show_every == 0):
print "Training batch loss: (%i / %i / %i) %f" % (
e, minibatch_index, n_train_batches, train_loss)
total_train_loss = total_train_loss / n_train_batches
s = 'TRAIN --> epoch %i | error %f' % (e, total_train_loss)
ut.log_write(s, params)
LStateList_t = [
np.zeros(shape=(batch_size, params['n_hidden']),
dtype=np.float32) for i in range(params['nlayer'] * 2)
] # initial hidden state
LStateList_pre = [
np.zeros(shape=(batch_size, params['n_hidden']),
dtype=np.float32) for i in range(params['nlayer'] * 2)
] # initial hidden sta
state_reset_counter_lst = [0 for i in range(batch_size)]
total_test_loss = 0
for minibatch_index in xrange(n_test_batches):
state_reset_counter_lst = [
s + 1 for s in state_reset_counter_lst
]
(LStateList_b, x, y,
state_reset_counter_lst) = dut.prepare_lstm_batch(
index_test_list, minibatch_index, batch_size, S_Test_list,
LStateList_t, LStateList_pre, params, Y_test, X_test,
state_reset_counter_lst)
LStateList_pre = LStateList_b
y = y.reshape(batch_size * params["seq_length"],
params["n_output"])
feed = {
model.input_data: x,
model.target_data: y,
model.initial_state: LStateList_b,
model.is_training: False,
model.output_keep_prob: 1.0
}
LStateList_t, final_output = sess.run(
[model.final_state, model.final_output], feed)
test_loss = ut.get_loss(params, gt=y, est=final_output)
tmp_lst = []
for item in LStateList_t:
tmp_lst.append(item.c)
tmp_lst.append(item.h)
LStateList_t = tmp_lst
total_test_loss += test_loss
if (minibatch_index % show_every == 0):
print "Test batch loss: (%i / %i / %i) %f" % (
e, minibatch_index, n_test_batches, test_loss)
total_test_loss = total_test_loss / n_test_batches
print "Total test loss %f" % total_test_loss
s = 'VAL --> epoch %i | error %f' % (e, total_test_loss)
ut.log_write(s, params)
(F_names_training, S_Train_list, F_names_test,
S_Test_list) = dut.prepare_training_set_fnames(params)
index_train_list, S_Train_list = dut.get_seq_indexes(params, S_Train_list)
index_test_list, S_Test_list = dut.get_seq_indexes(params, S_Test_list)
batch_size = params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
params['training_size'] = len(F_names_training) * params['seq_length']
params['test_size'] = len(F_names_test) * params['seq_length']
ut.start_log(params)
ut.log_write("Model training started", params)
# summary_writer = tf.train.SummaryWriter(params["sm"])
def train():
model = Model(params)
num_epochs = 1000
decay_rate = 0.4
show_every = 100
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = params[
'per_process_gpu_memory_fraction']
with tf.Session(config=config) as sess:
tf.initialize_all_variables().run()
saver = tf.train.Saver(tf.all_variables())
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
F_list_train,Y_train=du.shuffle_in_unison_inplace(F_list_train,Y_train)
params["len_train"]=len(F_list_train)
params["len_test"]=len(F_list_test)
u.start_log(params)
batch_size=params['batch_size']
n_train_batches = len(F_list_train)
n_train_batches /= batch_size
n_test_batches = len(F_list_test)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
is_train=1
for minibatch_index in range(n_train_batches):
if(minibatch_index==0):
x,y=du.prepare_cnn_batch(minibatch_index, batch_size, F_list_train, Y_train)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.train, (x, y,is_train))
async_b = pool.apply_async(du.prepare_cnn_batch, (minibatch_index, batch_size, F_list_train, Y_train))
pool.close()
pool.join()
loss = async_t.get() # get the return value from your function.
x=[]
y=[]
(x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
loss= model.train(x, y,is_train)
batch_loss += loss
if params['shufle_data']==1:
F_list_train,Y_train=du.shuffle_in_unison_inplace(F_list_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%1==0):
print("Model testing")
batch_loss3d = []
is_train=0
x=[]
y=[]
for minibatch_index in range(n_test_batches):
if(minibatch_index==0):
x,y=du.prepare_cnn_batch(minibatch_index, batch_size, F_list_test, Y_test)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.predictions, (x,is_train))
async_b = pool.apply_async(du.prepare_cnn_batch, (minibatch_index, batch_size, F_list_test, Y_test))
pool.close()
pool.join()
pred = async_t.get() # get the return value from your function.
loss3d =np.mean(np.linalg.norm((np.asarray(pred) - y)))
x=[]
y=[]
batch_loss3d.append(loss3d)
(x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
pred= model.predictions(x,is_train)
loss3d =np.mean(np.linalg.norm((np.asarray(pred) - y)))
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train():
batch_size = params["batch_size"]
num_epochs = 1000
decay_rate = 0.5
show_every = 100
deca_start = 2
with tf.Session(config=gpu_config) as sess:
tf.global_variables_initializer().run()
# sess.run(tracker.predict())
print 'Training Noise KLSTM'
noise_std = params['noise_std']
new_noise_std = 0.0
for e in range(num_epochs):
if e > (deca_start - 1):
sess.run(
tf.assign(
tracker.lr,
params['lr'] * (decay_rate**((e - deca_start) / 2))))
else:
sess.run(tf.assign(tracker.lr, params['lr']))
total_train_loss = 0
LStateList_F_t = ut.get_zero_state(params)
LStateList_F_pre = ut.get_zero_state(params)
LStateList_K_t = ut.get_zero_state(params, t='K')
LStateList_K_pre = ut.get_zero_state(params, t='K')
state_reset_counter_lst = [0 for i in range(batch_size)]
index_train_list_s = index_train_list
if params["shufle_data"] == 1 and params['reset_state'] == 1:
index_train_list_s = ut.shufle_data(index_train_list)
for minibatch_index in xrange(n_train_batches):
state_reset_counter_lst = [
s + 1 for s in state_reset_counter_lst
]
(LStateList_F_pre,LStateList_K_pre,_,x,y,r,f,state_reset_counter_lst)=\
dut.prepare_kfl_QRFf_batch(index_train_list_s, minibatch_index, batch_size,
S_Train_list, LStateList_F_t, LStateList_F_pre, LStateList_K_t, LStateList_K_pre,
None, None, params, Y_train, X_train, R_L_Train_list,F_list_train,state_reset_counter_lst)
if noise_std > 0.0:
u_cnt = e * n_train_batches + minibatch_index
if u_cnt in params['noise_schedule']:
new_noise_std = noise_std * (
u_cnt / (params['noise_schedule'][0]))
s = 'NOISE --> u_cnt %i | error %f' % (u_cnt,
new_noise_std)
ut.log_write(s, params)
if new_noise_std > 0.0:
noise = np.random.normal(0.0, new_noise_std, x.shape)
x = noise + x
gt = y
mes = x
feed = {
tracker._z: mes,
tracker.target_data: gt,
tracker.repeat_data: r,
tracker.initial_state: LStateList_F_pre,
tracker.initial_state_K: LStateList_K_pre,
tracker.output_keep_prob: params['rnn_keep_prob']
}
# feed = {tracker._z: mes, tracker.target_data: gt, tracker.initial_state: LStateList_F_pre
# , tracker._P_inp: P, tracker._I: I}
train_loss,LStateList_F_t,LStateList_K_t,_ = \
sess.run([tracker.cost,tracker.final_state_F,tracker.final_state_Q,
tracker.train_op], feed)
tmp_lst = []
for item in LStateList_F_t:
tmp_lst.append(item.c)
tmp_lst.append(item.h)
LStateList_F_t = tmp_lst
tmp_lst = []
for item in LStateList_K_t:
tmp_lst.append(item.c)
tmp_lst.append(item.h)
LStateList_K_t = tmp_lst
total_train_loss += train_loss
if (minibatch_index % show_every == 0):
print "Training batch loss: (%i / %i / %i) %f" % (
e, minibatch_index, n_train_batches, train_loss)
total_train_loss = total_train_loss / n_train_batches
s = 'TRAIN --> epoch %i | error %f' % (e, total_train_loss)
ut.log_write(s, params)
pre_test = "TEST_Data"
total_loss = test_data(sess, X_test, Y_test, index_test_list,
S_Test_list, R_L_Test_list, F_list_test, e,
pre_test, n_test_batches)
def train_model(params):
rn_id=params["rn_id"]
im_type=params["im_type"]
batch_size =params["batch_size"]
n_epochs =params["n_epochs"]
datasets = data_loader.load_data(params)
utils.start_log(datasets,params)
X_train, y_train,overlaps_train = datasets[0]
X_val, y_val,overlaps_val = datasets[1]
X_test, y_test,overlaps_test = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = len(X_train)
n_valid_batches = len(X_val)
n_test_batches = len(X_test)
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
y_val_mean=np.mean(y_val)
y_val_abs_mean=np.mean(np.abs(y_val))
utils.log_write("Model build started",params)
model= model_provider.get_model(params)
utils.log_write("Number of parameters: %s"%(model.count_params()),params)
run_mode=params["run_mode"]
utils.log_write("Model build ended",params)
utils.log_write("Training started",params)
best_validation_loss=np.inf
epoch_counter = 0
n_patch=params["n_patch"]
n_repeat=params["n_repeat"]#instead of extracting many batches for each epoch, we are repeating epoch since we are ensuring that output changes for each patch
while (epoch_counter < n_epochs):
epoch_counter = epoch_counter + 1
print("Training model...")
map_list=range(n_train_batches*n_repeat)
random.shuffle(map_list)
for index in xrange(n_train_batches*n_repeat):
minibatch_index=index%n_train_batches
map_index=map_list[index]%n_repeat
#We are shuffling data at each batch, we are shufling here because we already finished one epoch just repeating for the extract different batch
if(index>0 and minibatch_index==0):#we are checking weather we finish all dataset
ext=params["model_file"]+params["model"]+"_"+im_type+"_m_"+str(index%5)+".hdf5"
model.save_weights(ext, overwrite=True)
X_train,y_train=dt_utils.shuffle_in_unison_inplace(X_train,y_train)
iter = (epoch_counter - 1) * n_train_batches + index
if iter % 100 == 0:
print 'training @ iter = ', iter
batch_loss=0
Fx = X_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size]
data_y = y_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size]
for patch_index in xrange(n_patch):
patch_loc=utils.get_patch_loc(params)
argu= [(params,"F", Fx,patch_loc,map_index),(params,"S", Fx,patch_loc,map_index)]
results = dt_utils.asyn_load_batch_images(argu)
data_Fx = results[0]
data_Sx = results[1]
if(params["model_type"]==4):
data=data_Sx-data_Fx
loss =model.train_on_batch(data, data_y)
else:
loss =model.train_on_batch([data_Fx, data_Sx], data_y)
if isinstance(loss,list):
batch_loss+=loss[0]
else:
batch_loss+=loss
batch_loss/=n_patch
s='TRAIN--> epoch %i | batch_index %i/%i | error %f'%(epoch_counter, index + 1, n_train_batches*n_repeat, batch_loss)
utils.log_write(s,params)
if(run_mode==1):
break
#we are shufling for to be sure
X_train,y_train=dt_utils.shuffle_in_unison_inplace(X_train,y_train)
ext=params["model_file"]+params["model"]+"_"+im_type+"_e_"+str(rn_id)+"_"+str(epoch_counter % 10)+".hdf5"
model.save_weights(ext, overwrite=True)
if params['validate']==0:
print("Validation skipped...")
if(run_mode==1):
break
continue
print("Validating model...")
this_validation_loss = 0
map_list=range(n_valid_batches*n_repeat)
random.shuffle(map_list)
for index in xrange(n_valid_batches*n_repeat):
i = index%n_valid_batches
map_index=map_list[index]%n_repeat
epoch_loss=0
Fx = X_val[i * batch_size: (i + 1) * batch_size]
data_y = y_val[i * batch_size: (i + 1) * batch_size]
for patch_index in xrange(n_patch):
patch_loc=utils.get_patch_loc(params)
argu= [(params,"F", Fx,patch_loc,map_index),(params,"S", Fx,patch_loc,map_index)]
results = dt_utils.asyn_load_batch_images(argu)
data_Fx = results[0]
data_Sx = results[1]
if(params["model_type"]==4):
data=data_Sx-data_Fx
loss =model.test_on_batch(data, data_y)
else:
loss= model.test_on_batch([data_Fx, data_Sx],data_y)
if isinstance(loss,list):
epoch_loss+=loss[0]
else:
epoch_loss+=loss
epoch_loss/=n_patch
this_validation_loss +=epoch_loss
if(run_mode==1):
break
this_validation_loss /= (n_valid_batches*n_repeat)
s ='VAL--> epoch %i | error %f | data mean/abs %f/%f'%(epoch_counter, this_validation_loss,y_val_mean,y_val_abs_mean)
utils.log_write(s,params)
if this_validation_loss < best_validation_loss:
best_validation_loss = this_validation_loss
ext=params["model_file"]+params["model"]+"_"+im_type+"_"+"_best_"+str(rn_id)+"_"+str(epoch_counter)+".hdf5"
model.save_weights(ext, overwrite=True)
if(run_mode==1):
break
utils.log_write("Training ended",params)
def train_rnn(params):
data = []
for sindex in range(0, params['seq_length'], 5):
(X_train, Y_train, X_test, Y_test) = du.load_pose(params,
sindex=sindex)
data.append((X_train, Y_train, X_test, Y_test))
(X_train, Y_train, X_test, Y_test) = data[0]
params["len_train"] = X_train.shape[0] * X_train.shape[1]
params["len_test"] = X_test.shape[0] * X_test.shape[1]
u.start_log(params)
batch_size = params['batch_size']
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['resume'] == 1:
model = model_provider.get_model_pretrained(params)
else:
model = model_provider.get_model(params)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 10000
for epoch_counter in range(nb_epochs):
(X_train, Y_train, X_test,
Y_test) = data[np.mod(epoch_counter, len(data))]
if params['shufle_data'] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
batch_loss = 0.
for minibatch_index in range(n_train_batches):
x = X_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*54
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y)
batch_loss += loss
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 10 == 0):
print("Model testing")
batch_loss = 0.
batch_loss3d = 0.
for minibatch_index in range(n_test_batches):
x = X_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
y = Y_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
pred = model.predictions(x, x_b)
else:
pred = model.predictions(x)
loss = np.nanmean(np.abs(pred - y)**2)
loss3d = u.get_loss(y, pred)
batch_loss += loss
batch_loss3d += loss3d
batch_loss /= n_test_batches
batch_loss3d /= n_test_batches
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 #0.08
s = 'VAL--> epoch %i | error %f, %f %f' % (
val_counter, batch_loss, batch_loss3d, n_test_batches)
u.log_write(s, params)
for tr in lst_bool:
loss = 0.
total_cnt = 0.
for action in lst_action:
params["action"] = action
params['test_files'] = dt.load_files(params, is_training=tr)
test_loss = inception_output.eval(params)
cnt = len(params['test_files'][0])
loss = loss + test_loss * cnt
total_cnt = total_cnt + cnt
if tr == False:
s = 'TEST Set --> Action: %s, Frame Count: %i Final error %f' % (
action, cnt, test_loss)
else:
s = 'Train Set --> Action: %s, Frame Count: %i Final error %f' % (
action, cnt, test_loss)
ut.log_write(s, params)
loss = loss / total_cnt
if tr == False:
s = 'Total Test Frame Count: %i Final error %f' % (total_cnt, loss)
else:
s = 'Total Training Frame Count: %i Final error %f' % (total_cnt, loss)
ut.log_write(s, params)
# ut.start_log(params)
# ut.log_write("Model testing started",params)
# test_loss=inception_eval.eval(params)
# s ='VAL --> Final error %f'%(test_loss)
# ut.log_write(s,params)
def train(tracker, params):
I = np.asarray([
np.diag([1.0] * params['n_output'])
for i in range(params["batch_size"])
],
dtype=np.float32)
batch_size = params["batch_size"]
decay_rate = 0.95
# show_every=100
deca_start = 10
# pre_best_loss=10000
with tf.Session(config=gpu_config) as sess:
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# sess.run(tracker.predict())
print 'Training model:' + params["model"]
noise_std = params['noise_std']
new_noise_std = 0.0
median_result_lst = []
mean_result_lst = []
for e in range(num_epochs):
if e == 2:
params['lr'] = params['lr']
if e > (deca_start - 1):
sess.run(
tf.assign(tracker.lr, params['lr'] * (decay_rate**(e))))
else:
sess.run(tf.assign(tracker.lr, params['lr']))
total_train_loss = 0
state_reset_counter_lst = [0 for i in range(batch_size)]
index_train_list_s = index_train_list
dic_state = ut.get_state_list(params)
if params["shufle_data"] == 1 and params['reset_state'] == 1:
index_train_list_s = ut.shufle_data(index_train_list)
for minibatch_index in xrange(n_train_batches):
is_test = 0
state_reset_counter_lst = [
s + 1 for s in state_reset_counter_lst
]
(dic_state,x,y,r,f,_,state_reset_counter_lst,_)= \
th.prepare_batch(is_test,index_train_list_s, minibatch_index, batch_size,
S_Train_list, dic_state, params, Y_train, X_train, R_L_Train_list,F_list_train,state_reset_counter_lst)
if noise_std > 0.0:
u_cnt = e * n_train_batches + minibatch_index
if u_cnt in params['noise_schedule']:
new_noise_std = noise_std * (
u_cnt / (params['noise_schedule'][0]))
s = 'NOISE --> u_cnt %i | error %f' % (u_cnt,
new_noise_std)
ut.log_write(s, params)
if new_noise_std > 0.0:
noise = np.random.normal(0.0, new_noise_std, x.shape)
x = noise + x
feed = th.get_feed(tracker,
params,
r,
x,
y,
I,
dic_state,
is_training=1)
train_loss, states, _ = sess.run(
[tracker.cost, tracker.states, tracker.train_op], feed)
# print last_pred.shape
# print states.shape
for k in states.keys():
dic_state[k] = states[k]
total_train_loss += train_loss
# if e%5==0:
# print total_train_loss
pre_test = "TEST_Data"
total_loss, median_result, mean_result, final_output_lst, file_lst, noise_lst = test_data(
sess, params, X_test, Y_test, index_test_list, S_Test_list,
R_L_Test_list, F_list_test, e, pre_test, n_test_batches)
if len(full_median_result_lst) > 1:
if median_result[0] < np.min(full_median_result_lst,
axis=0)[0]:
# ut.write_slam_est(est_file=params["est_file"],est=final_output_lst,file_names=file_lst)
# ut.write_slam_est(est_file=params["noise_file"],est=noise_lst,file_names=file_lst)
# save_path=params["cp_file"]+params['msg']
# saver.save(sess,save_path)
print 'Writing estimations....'
full_median_result_lst.append(median_result)
median_result_lst.append(median_result)
mean_result_lst.append(mean_result)
# base_cp_path = params["cp_file"] + "/"
#
# lss_str = '%.5f' % total_loss
# model_name = lss_str + "_" + str(e) + "_" + str(params["rn_id"]) + params["model"] + "_model.ckpt"
# save_path = base_cp_path + model_name
# saved_path = False
# if pre_best_loss > total_loss:
# pre_best_loss = total_loss
# model_name = lss_str + "_" + str(e) + "_" + str(params["rn_id"]) + params["model"] + "_best_model.ckpt"
# save_path = base_cp_path + model_name
# saved_path = saver.save(sess, save_path)
# else:
# if e % 3.0 == 0:
# saved_path = saver.save(sess, save_path)
# if saved_path != "":
# s = 'MODEL_Saved --> epoch %i | error %f path %s' % (e, total_loss, saved_path)
# ut.log_write(s, params)
return median_result_lst, mean_result_lst
def test_data(sess, params, X, Y, index_list, S_list, R_L_list, F_list, e,
pre_test, n_batches):
dic_state = ut.get_state_list(params)
I = np.asarray([
np.diag([1.0] * params['n_output'])
for i in range(params["batch_size"])
],
dtype=np.float32)
is_test = 1
state_reset_counter_lst = [0 for i in range(batch_size)]
total_loss = 0.0
total_n_count = 0.0
for minibatch_index in xrange(n_batches):
state_reset_counter_lst = [s + 1 for s in state_reset_counter_lst]
(dic_state,x,y,r,f,_,state_reset_counter_lst,_)= \
th.prepare_batch(is_test,index_list, minibatch_index, batch_size,
S_list, dic_state, params, Y, X, R_L_list,F_list,state_reset_counter_lst)
feed = th.get_feed(tracker,
params,
r,
x,
y,
I,
dic_state,
is_training=0)
if mode == 'klstm':
states,final_output,final_pred_output,final_meas_output,q_mat,r_mat,k_mat,y =\
sess.run([tracker.states,tracker.final_output,tracker.final_pred_output,tracker.final_meas_output,
tracker.final_q_output,tracker.final_r_output,tracker.final_k_output,tracker.y], feed)
else:
states, final_output, y = \
sess.run([tracker.states, tracker.final_output, tracker.y], feed)
for k in states.keys():
dic_state[k] = states[k]
if params["normalise_data"] == 3 or params["normalise_data"] == 2:
final_output = ut.unNormalizeData(final_output, params["y_men"],
params["y_std"])
y = ut.unNormalizeData(y, params["y_men"], params["y_std"])
if params["normalise_data"] == 4:
final_output = ut.unNormalizeData(final_output, params["x_men"],
params["x_std"])
y = ut.unNormalizeData(y, params["x_men"], params["x_std"])
if mode == 'klstm':
final_pred_output = ut.unNormalizeData(final_pred_output,
params["x_men"],
params["x_std"])
final_meas_output = ut.unNormalizeData(final_meas_output,
params["x_men"],
params["x_std"])
test_loss, n_count = ut.get_loss(params,
gt=y,
est=final_output,
r=None)
f = f.reshape((-1, 2))
y_f = y.reshape(final_output.shape)
r = r.flatten()
fnames = f[np.nonzero(r)]
# e=final_output[np.nonzero(r)]
if mode == 'klstm':
ut.write_est(est_file=params["est_file"] + "/kal_est/",
est=final_output,
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/kal_est_dif/",
est=np.abs(final_output - y_f),
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/kal_pred/",
est=final_pred_output,
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/kal_pred_dif/",
est=np.abs(final_pred_output - y_f),
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/meas/",
est=final_meas_output,
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/q_mat/",
est=q_mat,
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/r_mat/",
est=r_mat,
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/k_mat/",
est=k_mat,
file_names=fnames)
ut.write_est(est_file=params["est_file"] + "/y_f/",
est=y_f,
file_names=fnames)
else:
ut.write_est(est_file=params["est_file"],
est=final_output,
file_names=fnames)
# print test/_loss
total_loss += test_loss * n_count
total_n_count += n_count
print total_loss / total_n_count
# if (minibatch_index%show_every==0):
# print pre_test+" test batch loss: (%i / %i / %i) %f"%(e,minibatch_index,n_train_batches,test_loss)
total_loss = total_loss / total_n_count
s = pre_test + ' Loss --> epoch %i | error %f' % (e, total_loss)
ut.log_write(s, params)
return total_loss
params['write_est'] = True
params["ds_training"] = "crop350"
params["ds_test"] = "crop350"
(trainer, evaller, params) = model_provider.get_model(params)
#Number of steps per epoch
params['training_files'] = dt.load_files(params, is_training=True)
# params['training_files']=([],[])
params['test_files'] = dt.load_files(params, is_training=False)
# params['run_mode']=0 #Load previuesly trained model
if args.mode == 3:
assert params['model_file'] != ""
params['model_file'] = args.model_file
params['run_mode'] = 3
ut.start_log(params)
ut.log_write("Testing given model", params)
params["batch_size"] = 100
params["est_file"] = params["est_file"] + str(
args.model_file.split('/')[-1]) + '/'
test_loss = evaller.eval(params)
s = 'VAL --> Model %s | error %f' % (args.model_file.split('/')[-1],
test_loss)
ut.log_write(s, params)
elif args.mode == 1:
ut.start_log(params)
for epoch_counter in range(args.epoch_counter_start, 100):
params["sm"] = params["sm"] + '/' + ut.get_time()
# if epoch_counter> 0:.
params['run_mode'] = 2
params["batch_size"] = 30
def train(X_train, Y_train, X_test, Y_test):
num_epochs = 1000
decay_rate = 0.5
pre_best_loss = 10000.0
# config = tf.ConfigProto(device_count = {'GPU': 0})
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.95
with tf.Session(config=config) as sess:
tf.initialize_all_variables().run()
saver = tf.train.Saver()
merged = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter(params["sm"], sess.graph)
for e in xrange(num_epochs):
X_train, Y_train = ut.unison_shuffled_copies(X_train, Y_train)
if e > 1:
sess.run(
tf.assign(model.lr, params['lr'] * (decay_rate**(e - 1))))
else:
sess.run(tf.assign(model.lr, params['lr']))
state_reset_counter_lst = [0 for i in range(batch_size)]
total_train_loss = 0
for minibatch_index in xrange(n_train_batches):
start = time.time()
state_reset_counter_lst = [
s + 1 for s in state_reset_counter_lst
]
x = X_train[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_train[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
feed = {
model.input_data: x,
model.target_data: y,
model.is_training: True,
model.output_keep_prob: 0.8
}
summary,train_loss,_ =\
sess.run([merged,model.cost, model.train_op], feed)
summary_writer.add_summary(summary, minibatch_index)
total_train_loss += train_loss
if (minibatch_index % show_every == 0):
print "Training batch loss: (%i / %i / %i) %f" % (
e, minibatch_index, n_train_batches, train_loss)
total_train_loss = total_train_loss / n_train_batches
s = 'TRAIN --> epoch %i | error %f' % (e, total_train_loss)
ut.log_write(s, params)
pre_test = "TRAINING_Data"
test_data(sess, X_train, Y_train, index_train_list, S_Train_list,
R_L_Train_list, F_list_train, e, pre_test,
n_train_batches)
pre_test = "TEST_Data"
total_loss = test_data(sess, X_test, Y_test, index_test_list,
S_Test_list, R_L_Test_list, F_list_test, e,
pre_test, n_test_batches)
base_cp_path = params["cp_file"] + "/"
lss_str = '%.5f' % total_loss
model_name = lss_str + "_" + str(e) + "_" + str(
params["rn_id"]) + params["model"] + "_model.ckpt"
save_path = base_cp_path + model_name
saved_path = False
if pre_best_loss > total_loss:
pre_best_loss = total_loss
model_name = lss_str + "_" + str(e) + "_" + str(
params["rn_id"]) + params["model"] + "_best_model.ckpt"
save_path = base_cp_path + model_name
saved_path = saver.save(sess, save_path)
else:
if e % 3.0 == 0:
saved_path = saver.save(sess, save_path)
if saved_path != False:
s = 'MODEL_Saved --> epoch %i | error %f path %s' % (
e, total_loss, saved_path)
ut.log_write(s, params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
params["len_train"]=len(F_list_train)
params["len_test"]=len(F_list_test)
u.start_log(params)
batch_size=params['batch_size']
n_train_batches = len(F_list_train)
n_train_batches /= batch_size
n_test_batches = len(F_list_test)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # initial hidden state
sid=0
for minibatch_index in range(n_train_batches):
x_lst=F_list_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
y_lst=G_list_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
x,y=du.load_batch(params,x_lst,y_lst)
# x=X_train[id_lst] #60*20*1024
# y=Y_train[id_lst]#60*20*54
is_train=1
if(params["model"]=="blstmnp"):
x_b=np.asarray(map(np.flipud,x))
loss = model.train(x,x_b,y)
else:
loss= model.train(x, y,is_train)
batch_loss += loss
if params['shufle_data']==1:
X_train,Y_train=du.shuffle_in_unison_inplace(X_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%3==0):
print("Model testing")
batch_loss3d = []
H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
sid=0
for minibatch_index in range(n_test_batches):
x_lst=F_list_test[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
y_lst=G_list_test[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
x,y=du.load_batch(params,x_lst,y_lst)
# tmp_sid=S_Test_list[(minibatch_index + 1) * batch_size-1]
# if(sid==0):
# sid=tmp_sid
# if(tmp_sid!=sid):
# sid=tmp_sid
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
# x=X_test[id_lst] #60*20*1024
# y=Y_test[id_lst]#60*20*54
is_train=0
if(params["model"]=="blstmnp"):
x_b=np.asarray(map(np.flipud,x))
pred = model.predictions(x,x_b)
else:
pred= model.predictions(x,is_train)
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train, Y_train, S_Train_list, F_list_train, G_list_train, X_test,
Y_test, S_Test_list, F_list_test, G_list_test) = du.load_pose(params)
params["len_train"] = Y_train.shape[0] * Y_train.shape[1]
params["len_test"] = Y_test.shape[0] * Y_test.shape[1]
u.start_log(params)
index_train_list, S_Train_list = du.get_batch_indexes(
S_Train_list) #This will prepare bacth indexes
index_test_list, S_Test_list = du.get_batch_indexes(S_Test_list)
batch_size = params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['run_mode'] == 1:
model = model_provider.get_model_pretrained(params, rng)
u.log_write("Pretrained loaded: %s" % (params['mfile']), params)
else:
model = model_provider.get_model(params, rng)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
H = C = np.zeros(shape=(batch_size, params['n_hidden']),
dtype=dtype) # initial hidden state
sid = 0
is_train = 1
x = []
y = []
for minibatch_index in range(n_train_batches):
if (minibatch_index == 0):
(sid, H, C, x, y) = du.prepare_cnn_lstm_batch(
index_train_list, minibatch_index, batch_size,
S_Train_list, sid, H, C, F_list_train, params, Y_train,
X_train)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.train, (x, y, is_train, H, C))
async_b = pool.apply_async(
du.prepare_cnn_lstm_batch,
(index_train_list, minibatch_index, batch_size, S_Train_list,
sid, H, C, F_list_train, params, Y_train, X_train))
pool.close()
pool.join()
(loss, H,
C) = async_t.get() # get the return value from your function.
x = []
y = []
(sid, H, C, x,
y) = async_b.get() # get the return value from your function.
if (minibatch_index == n_train_batches - 1):
loss, H, C = model.train(x, y, is_train, H, C)
batch_loss += loss
if params['shufle_data'] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 1 == 0):
print("Model testing")
batch_loss3d = []
H = C = np.zeros(
shape=(batch_size, params['n_hidden']),
dtype=dtype) # resetting initial state, since seq change
sid = 0
for minibatch_index in range(n_test_batches):
if (minibatch_index == 0):
(sid, H, C, x, y) = du.prepare_cnn_lstm_batch(
index_test_list, minibatch_index, batch_size,
S_Test_list, sid, H, C, F_list_test, params, Y_test,
X_test)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.predictions,
(x, is_train, H, C))
async_b = pool.apply_async(
du.prepare_cnn_lstm_batch,
(index_test_list, minibatch_index, batch_size, S_Test_list,
sid, H, C, F_list_test, params, Y_test, X_test))
pool.close()
pool.join()
(pred, H,
C) = async_t.get() # get the return value from your function.
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
(sid, H, C, x,
y) = async_b.get() # get the return value from your function.
if (minibatch_index == n_train_batches - 1):
pred, H, C = model.predictions(x, is_train, H, C)
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
batch_loss3d = np.nanmean(batch_loss3d)
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(epoch_counter) + "_" + str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1
s = 'VAL--> epoch %i | error %f, %f' % (val_counter, batch_loss3d,
n_test_batches)
u.log_write(s, params)
def train_rnn(params):
data = []
for sindex in range(0, params["seq_length"], 5):
(X_train, Y_train, X_test, Y_test) = du.load_pose(params, sindex=sindex)
data.append((X_train, Y_train, X_test, Y_test))
(X_train, Y_train, X_test, Y_test) = data[0]
params["len_train"] = X_train.shape[0] * X_train.shape[1]
params["len_test"] = X_test.shape[0] * X_test.shape[1]
u.start_log(params)
batch_size = params["batch_size"]
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
nb_epochs = params["n_epochs"]
print("Batch size: %i, train batch size: %i, test batch size: %i" % (batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params["resume"] == 1:
model = model_provider.get_model_pretrained(params)
else:
model = model_provider.get_model(params)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 10000
for epoch_counter in range(nb_epochs):
(X_train, Y_train, X_test, Y_test) = data[np.mod(epoch_counter, len(data))]
if params["shufle_data"] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
batch_loss = 0.0
for minibatch_index in range(n_train_batches):
x = X_train[minibatch_index * batch_size : (minibatch_index + 1) * batch_size] # 60*20*1024
y = Y_train[minibatch_index * batch_size : (minibatch_index + 1) * batch_size] # 60*20*54
if params["model"] == "blstmnp":
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y)
batch_loss += loss
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = "TRAIN--> epoch %i | error %f" % (epoch_counter, batch_loss)
u.log_write(s, params)
if epoch_counter % 10 == 0:
print("Model testing")
batch_loss = 0.0
batch_loss3d = 0.0
for minibatch_index in range(n_test_batches):
x = X_test[minibatch_index * batch_size : (minibatch_index + 1) * batch_size]
y = Y_test[minibatch_index * batch_size : (minibatch_index + 1) * batch_size]
if params["model"] == "blstmnp":
x_b = np.asarray(map(np.flipud, x))
pred = model.predictions(x, x_b)
else:
pred = model.predictions(x)
loss = np.nanmean(np.abs(pred - y) ** 2)
loss3d = u.get_loss(y, pred)
batch_loss += loss
batch_loss3d += loss3d
batch_loss /= n_test_batches
batch_loss3d /= n_test_batches
if batch_loss3d < best_loss:
best_loss = batch_loss3d
ext = str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 # 0.08
s = "VAL--> epoch %i | error %f, %f %f" % (val_counter, batch_loss, batch_loss3d, n_test_batches)
u.log_write(s, params)
index_train_list, S_Train_list = dut.get_seq_indexes(params, S_Train_list)
index_test_list, S_Test_list = dut.get_seq_indexes(params, S_Test_list)
batch_size = params['batch_size']
n_train_batches = X_train.shape[0]
n_train_batches /= batch_size
n_test_batches = X_test.shape[0]
n_test_batches /= batch_size
params['training_size'] = X_train.shape[0]
params['test_size'] = X_test.shape[0]
ut.start_log(params)
ut.log_write(
"Train mean:%f/%f, max:%f/%f, min:%f/%f ; Test mean:%f/%f, max:%f/%f, min:%f/%f "
";" %
(np.mean(X_train), np.mean(Y_train), np.max(X_train), np.max(Y_train),
np.min(X_train), np.min(Y_train), np.mean(X_test), np.mean(Y_test),
np.max(X_test), np.max(Y_test), np.min(X_test), np.min(Y_test)), params)
ut.log_write("Model training started", params)
# summary_writer = tf.train.SummaryWriter(params["sm"])
show_every = 100000.0
def test_data(sess, X, Y, index_list, S_list, R_L_list, F_list, e, pre_test,
n_batches):
state_reset_counter_lst = [0 for i in range(batch_size)]
total_loss = 0.0
total_losss = 0.0
total_n_count = 0.0
total_n_countt = 0.0
for minibatch_index in xrange(n_batches):
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train, Y_train, S_Train_list, F_list_train, G_list_train, X_test,
Y_test, S_Test_list, F_list_test, G_list_test) = du.load_pose(params)
F_list_train, G_list_train = du.shuffle_in_unison_inplace(
F_list_train, G_list_train)
params["len_train"] = len(F_list_train)
params["len_test"] = len(F_list_test)
u.start_log(params)
batch_size = params['batch_size']
n_train_batches = len(F_list_train)
n_train_batches /= batch_size
n_test_batches = len(F_list_test)
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['run_mode'] == 1:
model = model_provider.get_model_pretrained(params, rng)
u.log_write("Pretrained loaded: %s" % (params['mfile']), params)
else:
model = model_provider.get_model(params, rng)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # initial hidden state
sid = 0
for minibatch_index in range(n_train_batches):
x_lst = F_list_train[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y_lst = G_list_train[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
x, y = du.load_batch(params, x_lst, y_lst)
# x=X_train[id_lst] #60*20*1024
# y=Y_train[id_lst]#60*20*54
is_train = 1
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y, is_train)
batch_loss += loss
if params['shufle_data'] == 1:
F_list_train, G_list_train = du.shuffle_in_unison_inplace(
F_list_train, G_list_train)
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 1 == 0):
print("Model testing")
batch_loss3d = []
H = C = np.zeros(
shape=(batch_size, params['n_hidden']),
dtype=dtype) # resetting initial state, since seq change
sid = 0
for minibatch_index in range(n_test_batches):
x_lst = F_list_test[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y_lst = G_list_test[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
x, y = du.load_batch(params, x_lst, y_lst)
# tmp_sid=S_Test_list[(minibatch_index + 1) * batch_size-1]
# if(sid==0):
# sid=tmp_sid
# if(tmp_sid!=sid):
# sid=tmp_sid
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
# x=X_test[id_lst] #60*20*1024
# y=Y_test[id_lst]#60*20*54
is_train = 0
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
pred = model.predictions(x, x_b)
else:
pred = model.predictions(x, is_train)
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
batch_loss3d = np.nanmean(batch_loss3d)
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(epoch_counter) + "_" + str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 #0.08
s = 'VAL--> epoch %i | error %f, %f' % (val_counter, batch_loss3d,
n_test_batches)
u.log_write(s, params)
def train(Model,params):
I= np.asarray([np.diag([1.0]*params['n_output']) for i in range(params["batch_size"])],dtype=np.float32)
batch_size=params["batch_size"]
num_epochs=100000
decay_rate=0.9
show_every=100
deca_start=3
pre_best_loss=10000
with tf.Session() as sess:#config=gpu_config
tf.global_variables_initializer().run()
#saver = tf.train.Saver()
# if params["model"] == "kfl_QRf":
# ckpt = tf.train.get_checkpoint_state(params["mfile"])
# if ckpt and ckpt.model_checkpoint_path:
# saver.restore(sess, ckpt.model_checkpoint_path)
# mfile = ckpt.model_checkpoint_path
# params["est_file"] = params["est_file"] + mfile.split('/')[-1].replace('.ckpt', '') + '/'
# print "Loaded Model: %s" % ckpt.model_checkpoint_path
# if params["model"] == "kfl_QRf":
# for var in Model.tvars:
# path = '/mnt/Data1/hc/tt/cp/weights/' + var.name.replace('transitionF/','')
# if os.path.exists(path+'.npy'):
# val=np.load(path+'.npy')
# sess.run(tf.assign(var, val))
# print 'PreTrained LSTM model loaded...'
# sess.run(Model.predict())
print ('Training model:'+params["model"])
noise_std = params['noise_std']
new_noise_std=0.0
for e in range(num_epochs):
if e>(deca_start-1):
sess.run(tf.assign(Model.lr, params['lr'] * (decay_rate ** (e))))
else:
sess.run(tf.assign(Model.lr, params['lr']))
total_train_loss=0
state_reset_counter_lst=[0 for i in range(batch_size)]
index_train_list_s=index_train_list
dic_state = ut.get_state_list(params)
# total_loss = test_data(sess, params, X_test, Y_test, index_test_list, S_Test_list, R_L_Test_list,
# F_list_test, e, 'Test Check', n_test_batches)
if params["shufle_data"]==1 and params['reset_state']==1:
index_train_list_s = ut.shufle_data(index_train_list)
for minibatch_index in xrange(n_train_batches):
is_test = 0
state_reset_counter_lst=[s+1 for s in state_reset_counter_lst]
(dic_state,x,y,r,f,_,state_reset_counter_lst,_)= \
th.prepare_batch(is_test,index_train_list_s, minibatch_index, batch_size,
S_Train_list, dic_state, params, Y_train, X_train, R_L_Train_list,F_list_train,state_reset_counter_lst)
if noise_std >0.0:
u_cnt= e*n_train_batches + minibatch_index
if u_cnt in params['noise_schedule']:
if u_cnt==params['noise_schedule'][0]:
new_noise_std=noise_std
else:
new_noise_std = noise_std * (u_cnt / (params['noise_schedule'][1]))
s = 'NOISE --> u_cnt %i | error %f' % (u_cnt, new_noise_std)
ut.log_write(s, params)
if new_noise_std>0.0:
noise=np.random.normal(0.0,new_noise_std,x.shape)
x=noise+x
feed = th.get_feed(Model, params, r, x, y, I, dic_state, is_training=1)
train_loss,states,_ = sess.run([Model.cost,Model.states,Model.train_op], feed)
for k in states.keys():
dic_state[k] = states[k]
total_train_loss+=train_loss
if (minibatch_index%show_every==0):
print "Training batch loss: (%i / %i / %i) %f"%(e,minibatch_index,n_train_batches,
train_loss)
total_train_loss=total_train_loss/n_train_batches
s='TRAIN --> epoch %i | error %f'%(e, total_train_loss)
ut.log_write(s,params)
pre_test = "TRAINING_Data"
total_loss = test_data(sess, params, X_train, Y_train, index_train_list, S_Train_list, R_L_Train_list,
F_list_train, e, pre_test, n_train_batches)
pre_test="TEST_Data"
total_loss= test_data(sess,params,X_test,Y_test,index_test_list,S_Test_list,R_L_Test_list,F_list_test,e, pre_test,n_test_batches)
base_cp_path = params["cp_file"] + "/"
lss_str = '%.5f' % total_loss
model_name = lss_str + "_" + str(e) + "_" + str(params["rn_id"]) + params["model"] + "_model.ckpt"
save_path = base_cp_path + model_name
saved_path = False
if pre_best_loss > total_loss:
pre_best_loss = total_loss
model_name = lss_str + "_" + str(e) + "_" + str(params["rn_id"]) + params["model"] + "_best_model.ckpt"
save_path = base_cp_path + model_name
saved_path = saver.save(sess, save_path)
else:
if e % 3.0 == 0:
saved_path = saver.save(sess, save_path)
if saved_path != "":
s = 'MODEL_Saved --> epoch %i | error %f path %s' % (e, total_loss, saved_path)
ut.log_write(s, params)
ext=params["model_file"]+params["model"]+"_"+im_type+"_"+"_best_"+str(rn_id)+"_"+str(epoch_counter)+".hdf5"
model.save_weights(ext, overwrite=True)
if(run_mode==1):
break
utils.log_write("Training ended",params)
if __name__ == "__main__":
global params
params= config.get_params()
parser = argparse.ArgumentParser(description='Training the module')
parser.add_argument('-rm','--run_mode',type=int, help='Training mode:0 full train, 1 system check, 2 simle train',
required=True)
parser.add_argument('-m','--model',help='Model: kcnnr, dccnr, current('+params["model"]+')',default=params["model"])
parser.add_argument('-i','--im_type',help='Image type: pre_depth, depth, gray, current('+params["im_type"]+')',
default=params["im_type"])
args = vars(parser.parse_args())
params["run_mode"]=args["run_mode"]
params["model"]=args["model"]
params["im_type"]=args["im_type"]
params["is_exit"]=0
params=config.update_params(params)
try:
train_model(params)
except KeyboardInterrupt:
utils.log_write("Exiting program",params)
is_exit=1
params["is_exit"]=1
except Exception, e:
utils.log_write('got exception: %r, terminating the pool' % (e,),params)
params["is_exit"]=1
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
params["len_train"]=Y_train.shape[0]*Y_train.shape[1]
params["len_test"]=Y_test.shape[0]*Y_test.shape[1]
u.start_log(params)
index_train_list,S_Train_list=du.get_seq_indexes(params,S_Train_list)
index_test_list,S_Test_list=du.get_seq_indexes(params,S_Test_list)
batch_size=params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
LStateList_t=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden state
LStateList_pre=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden state
state_reset_counter_lst=[0 for i in range(batch_size)]
is_train=1
for minibatch_index in range(n_train_batches):
state_reset_counter_lst=[s+1 for s in state_reset_counter_lst]
(LStateList_b,x,y,state_reset_counter_lst)=du.prepare_lstm_batch(index_train_list, minibatch_index, batch_size, S_Train_list,LStateList_t,LStateList_pre, F_list_train, params, Y_train, X_train,state_reset_counter_lst)
LStateList_pre=LStateList_b
args=(x, y,is_train)+tuple(LStateList_b)
result= model.train(*args)
loss=result[0]
LStateList_t=result[1:len(result)]
batch_loss += loss
if params['shufle_data']==1:
X_train,Y_train=du.shuffle_in_unison_inplace(X_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%1==0):
is_train=0
print("Model testing")
state_reset_counter=0
batch_loss3d = []
LStateList_t=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden state
LStateList_pre=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden sta
state_reset_counter_lst=[0 for i in range(batch_size)]
for minibatch_index in range(n_test_batches):
state_reset_counter_lst=[s+1 for s in state_reset_counter_lst]
(LStateList_b,x,y,state_reset_counter_lst)=du.prepare_lstm_batch(index_test_list, minibatch_index, batch_size, S_Test_list, LStateList_t,LStateList_pre, F_list_test, params, Y_test, X_test,state_reset_counter_lst)
LStateList_pre=LStateList_b
args=(x,is_train)+tuple(LStateList_b)
result = model.predictions(*args)
pred=result[0]
LStateList_t=result[1:len(result)]
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
params["len_train"]=Y_train.shape[0]*Y_train.shape[1]
params["len_test"]=Y_test.shape[0]*Y_test.shape[1]
u.start_log(params)
index_train_list,S_Train_list=du.get_batch_indexes(params,S_Train_list)
index_test_list,S_Test_list=du.get_batch_indexes(params,S_Test_list)
batch_size=params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # initial hidden state
sid=0
is_train=1
x=[]
y=[]
for minibatch_index in range(n_train_batches):
if(minibatch_index==0):
(sid,H,C,x,y)=du.prepare_cnn_lstm_batch(index_train_list, minibatch_index, batch_size, S_Train_list, sid, H, C, F_list_train, params, Y_train, X_train)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.train, (x, y,is_train,H,C))
async_b = pool.apply_async(du.prepare_cnn_lstm_batch, (index_train_list, minibatch_index, batch_size, S_Train_list, sid, H, C, F_list_train, params, Y_train, X_train))
pool.close()
pool.join()
(loss,H,C) = async_t.get() # get the return value from your function.
x=[]
y=[]
(sid,H,C,x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
loss,H,C= model.train(x, y,is_train,H,C)
batch_loss += loss
if params['shufle_data']==1:
X_train,Y_train=du.shuffle_in_unison_inplace(X_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%1==0):
print("Model testing")
batch_loss3d = []
H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
sid=0
for minibatch_index in range(n_test_batches):
if(minibatch_index==0):
(sid,H,C,x,y)=du.prepare_cnn_lstm_batch(index_test_list, minibatch_index, batch_size, S_Test_list, sid, H, C, F_list_test, params, Y_test, X_test)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.predictions, (x,is_train,H,C))
async_b = pool.apply_async(du.prepare_cnn_lstm_batch, (index_test_list, minibatch_index, batch_size, S_Test_list, sid, H, C, F_list_test, params, Y_test, X_test))
pool.close()
pool.join()
(pred,H,C) = async_t.get() # get the return value from your function.
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
x=[]
y=[]
(sid,H,C,x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
pred,H,C= model.predictions(x,is_train,H,C)
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def eval(params):
batch_size = params['batch_size']
params['write_est']=False
num_examples = len(params['test_files'][0])
with tf.Graph().as_default():
batch = dut.distorted_inputs(params,is_training=is_training)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits,aux, end_points = inception_resnet_v2.inception_resnet_v2(batch[0],
num_classes=params['n_output'],
is_training=is_training)
# Obtain the trainable variables and a saver
# variables_to_restore = slim.get_variables_to_restore()
init_fn=ut.get_init_fn(slim,params)
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = params['per_process_gpu_memory_fraction']
with tf.Session() as sess:
init_op = tf.group(tf.initialize_all_variables(), tf.initialize_local_variables())
# init_op = tf.global_variables_initializer()
sess.run(init_op)
init_fn(sess)
coord = tf.train.Coordinator()
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(sess, coord=coord, daemon=True, start=True))
num_iter = int(math.ceil(num_examples / batch_size))
print('%s: Testing started.' % (datetime.now()))
step = 0
loss_lst=[]
run_lst=[]
run_lst.append(logits)
# run_lst.append(end_points['PreLogitsFlatten'])
# run_lst.append(end_points['PrePool'])
[run_lst.append(lst) for lst in batch[1:len(batch)]]
while step < num_iter and not coord.should_stop():
try:
batch_res= sess.run(run_lst)
except tf.errors.OutOfRangeError:
print ('Testing finished....%d'%step)
break
if(params['write_est']==True):
ut.write_mid_est(params,batch_res)
est=batch_res[0]
gt=batch_res[1]
# print(est.shape)
# print(gt.shape)
prepool=batch_res[-1]
loss,_= ut.get_loss(params,gt,est)
loss_lst.append(loss)
s ='VAL --> batch %i/%i | error %f'%(step,num_iter,loss)
if step%10==0:
ut.log_write(s,params,screen_print=True)
print "Current number of examples / mean err: %i / %f"%(step*gt.shape[0],np.mean(loss_lst))
else:
ut.log_write(s, params, screen_print=False)
# joint_list=['/'.join(p1.split('/')[0:-1]).replace('joints','img').replace('.cdf','')+'/frame_'+(p1.split('/')[-1].replace('.txt','')).zfill(5)+'.png' for p1 in image_names]
# print ('List equality check:')
# print len(label_names) == len(set(label_names))
# print sum(joint_list==label_names)==(len(est))
# print(len(label_names))
step += 1
coord.request_stop()
coord.join(threads)
return np.mean(loss_lst)
