def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
params["len_train"]=Y_train.shape[0]*Y_train.shape[1]
params["len_test"]=Y_test.shape[0]*Y_test.shape[1]
u.start_log(params)
index_train_list,S_Train_list=du.get_batch_indexes(params,S_Train_list)
index_test_list,S_Test_list=du.get_batch_indexes(params,S_Test_list)
batch_size=params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # initial hidden state
sid=0
is_train=1
x=[]
y=[]
for minibatch_index in range(n_train_batches):
if(minibatch_index==0):
(sid,H,C,x,y)=du.prepare_cnn_lstm_batch(index_train_list, minibatch_index, batch_size, S_Train_list, sid, H, C, F_list_train, params, Y_train, X_train)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.train, (x, y,is_train,H,C))
async_b = pool.apply_async(du.prepare_cnn_lstm_batch, (index_train_list, minibatch_index, batch_size, S_Train_list, sid, H, C, F_list_train, params, Y_train, X_train))
pool.close()
pool.join()
(loss,H,C) = async_t.get() # get the return value from your function.
x=[]
y=[]
(sid,H,C,x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
loss,H,C= model.train(x, y,is_train,H,C)
batch_loss += loss
if params['shufle_data']==1:
X_train,Y_train=du.shuffle_in_unison_inplace(X_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%1==0):
print("Model testing")
batch_loss3d = []
H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
sid=0
for minibatch_index in range(n_test_batches):
if(minibatch_index==0):
(sid,H,C,x,y)=du.prepare_cnn_lstm_batch(index_test_list, minibatch_index, batch_size, S_Test_list, sid, H, C, F_list_test, params, Y_test, X_test)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.predictions, (x,is_train,H,C))
async_b = pool.apply_async(du.prepare_cnn_lstm_batch, (index_test_list, minibatch_index, batch_size, S_Test_list, sid, H, C, F_list_test, params, Y_test, X_test))
pool.close()
pool.join()
(pred,H,C) = async_t.get() # get the return value from your function.
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
x=[]
y=[]
(sid,H,C,x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
pred,H,C= model.predictions(x,is_train,H,C)
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train, Y_train, S_Train_list, F_list_train, G_list_train, X_test,
Y_test, S_Test_list, F_list_test, G_list_test) = du.load_pose(params)
params["len_train"] = Y_train.shape[0] * Y_train.shape[1]
params["len_test"] = Y_test.shape[0] * Y_test.shape[1]
u.start_log(params)
index_train_list, S_Train_list = du.get_batch_indexes(
S_Train_list) #This will prepare bacth indexes
index_test_list, S_Test_list = du.get_batch_indexes(S_Test_list)
batch_size = params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['run_mode'] == 1:
model = model_provider.get_model_pretrained(params, rng)
u.log_write("Pretrained loaded: %s" % (params['mfile']), params)
else:
model = model_provider.get_model(params, rng)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
H = C = np.zeros(shape=(batch_size, params['n_hidden']),
dtype=dtype) # initial hidden state
sid = 0
is_train = 1
x = []
y = []
for minibatch_index in range(n_train_batches):
if (minibatch_index == 0):
(sid, H, C, x, y) = du.prepare_cnn_lstm_batch(
index_train_list, minibatch_index, batch_size,
S_Train_list, sid, H, C, F_list_train, params, Y_train,
X_train)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.train, (x, y, is_train, H, C))
async_b = pool.apply_async(
du.prepare_cnn_lstm_batch,
(index_train_list, minibatch_index, batch_size, S_Train_list,
sid, H, C, F_list_train, params, Y_train, X_train))
pool.close()
pool.join()
(loss, H,
C) = async_t.get() # get the return value from your function.
x = []
y = []
(sid, H, C, x,
y) = async_b.get() # get the return value from your function.
if (minibatch_index == n_train_batches - 1):
loss, H, C = model.train(x, y, is_train, H, C)
batch_loss += loss
if params['shufle_data'] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 1 == 0):
print("Model testing")
batch_loss3d = []
H = C = np.zeros(
shape=(batch_size, params['n_hidden']),
dtype=dtype) # resetting initial state, since seq change
sid = 0
for minibatch_index in range(n_test_batches):
if (minibatch_index == 0):
(sid, H, C, x, y) = du.prepare_cnn_lstm_batch(
index_test_list, minibatch_index, batch_size,
S_Test_list, sid, H, C, F_list_test, params, Y_test,
X_test)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.predictions,
(x, is_train, H, C))
async_b = pool.apply_async(
du.prepare_cnn_lstm_batch,
(index_test_list, minibatch_index, batch_size, S_Test_list,
sid, H, C, F_list_test, params, Y_test, X_test))
pool.close()
pool.join()
(pred, H,
C) = async_t.get() # get the return value from your function.
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
(sid, H, C, x,
y) = async_b.get() # get the return value from your function.
if (minibatch_index == n_train_batches - 1):
pred, H, C = model.predictions(x, is_train, H, C)
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
batch_loss3d = np.nanmean(batch_loss3d)
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(epoch_counter) + "_" + str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1
s = 'VAL--> epoch %i | error %f, %f' % (val_counter, batch_loss3d,
n_test_batches)
u.log_write(s, params)
def train_model(params):
rn_id=params["rn_id"]
im_type=params["im_type"]
batch_size =params["batch_size"]
n_epochs =params["n_epochs"]
datasets = data_loader.load_data(params)
utils.start_log(datasets,params)
X_train, y_train,overlaps_train = datasets[0]
X_val, y_val,overlaps_val = datasets[1]
X_test, y_test,overlaps_test = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = len(X_train)
n_valid_batches = len(X_val)
n_test_batches = len(X_test)
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
y_val_mean=np.mean(y_val)
y_val_abs_mean=np.mean(np.abs(y_val))
utils.log_write("Model build started",params)
model= model_provider.get_model(params)
utils.log_write("Number of parameters: %s"%(model.count_params()),params)
run_mode=params["run_mode"]
utils.log_write("Model build ended",params)
utils.log_write("Training started",params)
best_validation_loss=np.inf
epoch_counter = 0
n_patch=params["n_patch"]
n_repeat=params["n_repeat"]#instead of extracting many batches for each epoch, we are repeating epoch since we are ensuring that output changes for each patch
while (epoch_counter < n_epochs):
epoch_counter = epoch_counter + 1
print("Training model...")
map_list=range(n_train_batches*n_repeat)
random.shuffle(map_list)
for index in xrange(n_train_batches*n_repeat):
minibatch_index=index%n_train_batches
map_index=map_list[index]%n_repeat
#We are shuffling data at each batch, we are shufling here because we already finished one epoch just repeating for the extract different batch
if(index>0 and minibatch_index==0):#we are checking weather we finish all dataset
ext=params["model_file"]+params["model"]+"_"+im_type+"_m_"+str(index%5)+".hdf5"
model.save_weights(ext, overwrite=True)
X_train,y_train=dt_utils.shuffle_in_unison_inplace(X_train,y_train)
iter = (epoch_counter - 1) * n_train_batches + index
if iter % 100 == 0:
print 'training @ iter = ', iter
batch_loss=0
Fx = X_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size]
data_y = y_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size]
for patch_index in xrange(n_patch):
patch_loc=utils.get_patch_loc(params)
argu= [(params,"F", Fx,patch_loc,map_index),(params,"S", Fx,patch_loc,map_index)]
results = dt_utils.asyn_load_batch_images(argu)
data_Fx = results[0]
data_Sx = results[1]
if(params["model_type"]==4):
data=data_Sx-data_Fx
loss =model.train_on_batch(data, data_y)
else:
loss =model.train_on_batch([data_Fx, data_Sx], data_y)
if isinstance(loss,list):
batch_loss+=loss[0]
else:
batch_loss+=loss
batch_loss/=n_patch
s='TRAIN--> epoch %i | batch_index %i/%i | error %f'%(epoch_counter, index + 1, n_train_batches*n_repeat, batch_loss)
utils.log_write(s,params)
if(run_mode==1):
break
#we are shufling for to be sure
X_train,y_train=dt_utils.shuffle_in_unison_inplace(X_train,y_train)
ext=params["model_file"]+params["model"]+"_"+im_type+"_e_"+str(rn_id)+"_"+str(epoch_counter % 10)+".hdf5"
model.save_weights(ext, overwrite=True)
if params['validate']==0:
print("Validation skipped...")
if(run_mode==1):
break
continue
print("Validating model...")
this_validation_loss = 0
map_list=range(n_valid_batches*n_repeat)
random.shuffle(map_list)
for index in xrange(n_valid_batches*n_repeat):
i = index%n_valid_batches
map_index=map_list[index]%n_repeat
epoch_loss=0
Fx = X_val[i * batch_size: (i + 1) * batch_size]
data_y = y_val[i * batch_size: (i + 1) * batch_size]
for patch_index in xrange(n_patch):
patch_loc=utils.get_patch_loc(params)
argu= [(params,"F", Fx,patch_loc,map_index),(params,"S", Fx,patch_loc,map_index)]
results = dt_utils.asyn_load_batch_images(argu)
data_Fx = results[0]
data_Sx = results[1]
if(params["model_type"]==4):
data=data_Sx-data_Fx
loss =model.test_on_batch(data, data_y)
else:
loss= model.test_on_batch([data_Fx, data_Sx],data_y)
if isinstance(loss,list):
epoch_loss+=loss[0]
else:
epoch_loss+=loss
epoch_loss/=n_patch
this_validation_loss +=epoch_loss
if(run_mode==1):
break
this_validation_loss /= (n_valid_batches*n_repeat)
s ='VAL--> epoch %i | error %f | data mean/abs %f/%f'%(epoch_counter, this_validation_loss,y_val_mean,y_val_abs_mean)
utils.log_write(s,params)
if this_validation_loss < best_validation_loss:
best_validation_loss = this_validation_loss
ext=params["model_file"]+params["model"]+"_"+im_type+"_"+"_best_"+str(rn_id)+"_"+str(epoch_counter)+".hdf5"
model.save_weights(ext, overwrite=True)
if(run_mode==1):
break
utils.log_write("Training ended",params)
def train_rnn(params):
data = []
for sindex in range(0, params['seq_length'], 5):
(X_train, Y_train, X_test, Y_test) = du.load_pose(params,
sindex=sindex)
data.append((X_train, Y_train, X_test, Y_test))
(X_train, Y_train, X_test, Y_test) = data[0]
params["len_train"] = X_train.shape[0] * X_train.shape[1]
params["len_test"] = X_test.shape[0] * X_test.shape[1]
u.start_log(params)
batch_size = params['batch_size']
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['resume'] == 1:
model = model_provider.get_model_pretrained(params)
else:
model = model_provider.get_model(params)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 10000
for epoch_counter in range(nb_epochs):
(X_train, Y_train, X_test,
Y_test) = data[np.mod(epoch_counter, len(data))]
if params['shufle_data'] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
batch_loss = 0.
for minibatch_index in range(n_train_batches):
x = X_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*54
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y)
batch_loss += loss
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 10 == 0):
print("Model testing")
batch_loss = 0.
batch_loss3d = 0.
for minibatch_index in range(n_test_batches):
x = X_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
y = Y_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
pred = model.predictions(x, x_b)
else:
pred = model.predictions(x)
loss = np.nanmean(np.abs(pred - y)**2)
loss3d = u.get_loss(y, pred)
batch_loss += loss
batch_loss3d += loss3d
batch_loss /= n_test_batches
batch_loss3d /= n_test_batches
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 #0.08
s = 'VAL--> epoch %i | error %f, %f %f' % (
val_counter, batch_loss, batch_loss3d, n_test_batches)
u.log_write(s, params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train, Y_train, S_Train_list, F_list_train, G_list_train, X_test,
Y_test, S_Test_list, F_list_test, G_list_test) = du.load_pose(params)
F_list_train, G_list_train = du.shuffle_in_unison_inplace(
F_list_train, G_list_train)
params["len_train"] = len(F_list_train)
params["len_test"] = len(F_list_test)
u.start_log(params)
batch_size = params['batch_size']
n_train_batches = len(F_list_train)
n_train_batches /= batch_size
n_test_batches = len(F_list_test)
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['run_mode'] == 1:
model = model_provider.get_model_pretrained(params, rng)
u.log_write("Pretrained loaded: %s" % (params['mfile']), params)
else:
model = model_provider.get_model(params, rng)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # initial hidden state
sid = 0
for minibatch_index in range(n_train_batches):
x_lst = F_list_train[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y_lst = G_list_train[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
x, y = du.load_batch(params, x_lst, y_lst)
# x=X_train[id_lst] #60*20*1024
# y=Y_train[id_lst]#60*20*54
is_train = 1
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y, is_train)
batch_loss += loss
if params['shufle_data'] == 1:
F_list_train, G_list_train = du.shuffle_in_unison_inplace(
F_list_train, G_list_train)
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 1 == 0):
print("Model testing")
batch_loss3d = []
H = C = np.zeros(
shape=(batch_size, params['n_hidden']),
dtype=dtype) # resetting initial state, since seq change
sid = 0
for minibatch_index in range(n_test_batches):
x_lst = F_list_test[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y_lst = G_list_test[minibatch_index *
batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
x, y = du.load_batch(params, x_lst, y_lst)
# tmp_sid=S_Test_list[(minibatch_index + 1) * batch_size-1]
# if(sid==0):
# sid=tmp_sid
# if(tmp_sid!=sid):
# sid=tmp_sid
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
# x=X_test[id_lst] #60*20*1024
# y=Y_test[id_lst]#60*20*54
is_train = 0
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
pred = model.predictions(x, x_b)
else:
pred = model.predictions(x, is_train)
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
batch_loss3d = np.nanmean(batch_loss3d)
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(epoch_counter) + "_" + str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 #0.08
s = 'VAL--> epoch %i | error %f, %f' % (val_counter, batch_loss3d,
n_test_batches)
u.log_write(s, params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
params["len_train"]=len(F_list_train)
params["len_test"]=len(F_list_test)
u.start_log(params)
batch_size=params['batch_size']
n_train_batches = len(F_list_train)
n_train_batches /= batch_size
n_test_batches = len(F_list_test)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # initial hidden state
sid=0
for minibatch_index in range(n_train_batches):
x_lst=F_list_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
y_lst=G_list_train[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
x,y=du.load_batch(params,x_lst,y_lst)
# x=X_train[id_lst] #60*20*1024
# y=Y_train[id_lst]#60*20*54
is_train=1
if(params["model"]=="blstmnp"):
x_b=np.asarray(map(np.flipud,x))
loss = model.train(x,x_b,y)
else:
loss= model.train(x, y,is_train)
batch_loss += loss
if params['shufle_data']==1:
X_train,Y_train=du.shuffle_in_unison_inplace(X_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%3==0):
print("Model testing")
batch_loss3d = []
H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
sid=0
for minibatch_index in range(n_test_batches):
x_lst=F_list_test[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
y_lst=G_list_test[minibatch_index * batch_size: (minibatch_index + 1) * batch_size] #60*20*1024
x,y=du.load_batch(params,x_lst,y_lst)
# tmp_sid=S_Test_list[(minibatch_index + 1) * batch_size-1]
# if(sid==0):
# sid=tmp_sid
# if(tmp_sid!=sid):
# sid=tmp_sid
# H=C=np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) # resetting initial state, since seq change
# x=X_test[id_lst] #60*20*1024
# y=Y_test[id_lst]#60*20*54
is_train=0
if(params["model"]=="blstmnp"):
x_b=np.asarray(map(np.flipud,x))
pred = model.predictions(x,x_b)
else:
pred= model.predictions(x,is_train)
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train, Y_train, S_Train_list, F_list_train, G_list_train, X_test,
Y_test, S_Test_list, F_list_test, G_list_test) = du.load_pose(params)
params["len_train"] = len(X_train)
params["len_test"] = len(X_test)
u.start_log(params)
batch_size = params['batch_size']
n_train_batches = params["len_train"]
n_train_batches /= batch_size
n_test_batches = params["len_test"]
n_test_batches /= batch_size
nb_epochs = params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i" %
(batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params['run_mode'] == 1:
model = model_provider.get_model_pretrained(params, rng)
u.log_write("Pretrained loaded: %s" % (params['mfile']), params)
else:
model = model_provider.get_model(params, rng)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
sid = 0
is_train = 1
for minibatch_index in range(n_train_batches):
x = X_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_train[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
if (params["model"] == "blstmnp"):
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y, is_train)
batch_loss += loss
if params['shufle_data'] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = 'TRAIN--> epoch %i | error %f' % (epoch_counter, batch_loss)
u.log_write(s, params)
if (epoch_counter % 3 == 0):
print("Model testing")
batch_loss3d = []
is_train = 0
for minibatch_index in range(n_test_batches):
x = X_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
y = Y_test[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size] #60*20*1024
pred = model.predictions(x, is_train)
loss3d = u.get_loss(params, y, pred)
batch_loss3d.append(loss3d)
batch_loss3d = np.nanmean(batch_loss3d)
if (batch_loss3d < best_loss):
best_loss = batch_loss3d
ext = str(epoch_counter) + "_" + str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 #0.08
s = 'VAL--> epoch %i | error %f, %f' % (val_counter, batch_loss3d,
n_test_batches)
u.log_write(s, params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
F_list_train,Y_train=du.shuffle_in_unison_inplace(F_list_train,Y_train)
params["len_train"]=len(F_list_train)
params["len_test"]=len(F_list_test)
u.start_log(params)
batch_size=params['batch_size']
n_train_batches = len(F_list_train)
n_train_batches /= batch_size
n_test_batches = len(F_list_test)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
is_train=1
for minibatch_index in range(n_train_batches):
if(minibatch_index==0):
x,y=du.prepare_cnn_batch(minibatch_index, batch_size, F_list_train, Y_train)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.train, (x, y,is_train))
async_b = pool.apply_async(du.prepare_cnn_batch, (minibatch_index, batch_size, F_list_train, Y_train))
pool.close()
pool.join()
loss = async_t.get() # get the return value from your function.
x=[]
y=[]
(x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
loss= model.train(x, y,is_train)
batch_loss += loss
if params['shufle_data']==1:
F_list_train,Y_train=du.shuffle_in_unison_inplace(F_list_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%1==0):
print("Model testing")
batch_loss3d = []
is_train=0
x=[]
y=[]
for minibatch_index in range(n_test_batches):
if(minibatch_index==0):
x,y=du.prepare_cnn_batch(minibatch_index, batch_size, F_list_test, Y_test)
pool = ThreadPool(processes=2)
async_t = pool.apply_async(model.predictions, (x,is_train))
async_b = pool.apply_async(du.prepare_cnn_batch, (minibatch_index, batch_size, F_list_test, Y_test))
pool.close()
pool.join()
pred = async_t.get() # get the return value from your function.
loss3d =np.mean(np.linalg.norm((np.asarray(pred) - y)))
x=[]
y=[]
batch_loss3d.append(loss3d)
(x,y) = async_b.get() # get the return value from your function.
if(minibatch_index==n_train_batches-1):
pred= model.predictions(x,is_train)
loss3d =np.mean(np.linalg.norm((np.asarray(pred) - y)))
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train_rnn(params):
rng = RandomStreams(seed=1234)
(X_train,Y_train,S_Train_list,F_list_train,G_list_train,X_test,Y_test,S_Test_list,F_list_test,G_list_test)=du.load_pose(params)
params["len_train"]=Y_train.shape[0]*Y_train.shape[1]
params["len_test"]=Y_test.shape[0]*Y_test.shape[1]
u.start_log(params)
index_train_list,S_Train_list=du.get_seq_indexes(params,S_Train_list)
index_test_list,S_Test_list=du.get_seq_indexes(params,S_Test_list)
batch_size=params['batch_size']
n_train_batches = len(index_train_list)
n_train_batches /= batch_size
n_test_batches = len(index_test_list)
n_test_batches /= batch_size
nb_epochs=params['n_epochs']
print("Batch size: %i, train batch size: %i, test batch size: %i"%(batch_size,n_train_batches,n_test_batches))
u.log_write("Model build started",params)
if params['run_mode']==1:
model= model_provider.get_model_pretrained(params,rng)
u.log_write("Pretrained loaded: %s"%(params['mfile']),params)
else:
model= model_provider.get_model(params,rng)
u.log_write("Number of parameters: %s"%(model.n_param),params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started",params)
val_counter=0
best_loss=1000
for epoch_counter in range(nb_epochs):
batch_loss = 0.
LStateList_t=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden state
LStateList_pre=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden state
state_reset_counter_lst=[0 for i in range(batch_size)]
is_train=1
for minibatch_index in range(n_train_batches):
state_reset_counter_lst=[s+1 for s in state_reset_counter_lst]
(LStateList_b,x,y,state_reset_counter_lst)=du.prepare_lstm_batch(index_train_list, minibatch_index, batch_size, S_Train_list,LStateList_t,LStateList_pre, F_list_train, params, Y_train, X_train,state_reset_counter_lst)
LStateList_pre=LStateList_b
args=(x, y,is_train)+tuple(LStateList_b)
result= model.train(*args)
loss=result[0]
LStateList_t=result[1:len(result)]
batch_loss += loss
if params['shufle_data']==1:
X_train,Y_train=du.shuffle_in_unison_inplace(X_train,Y_train)
train_errors[epoch_counter] = batch_loss
batch_loss/=n_train_batches
s='TRAIN--> epoch %i | error %f'%(epoch_counter, batch_loss)
u.log_write(s,params)
if(epoch_counter%1==0):
is_train=0
print("Model testing")
state_reset_counter=0
batch_loss3d = []
LStateList_t=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden state
LStateList_pre=[np.zeros(shape=(batch_size,params['n_hidden']), dtype=dtype) for i in range(params['nlayer']*2)] # initial hidden sta
state_reset_counter_lst=[0 for i in range(batch_size)]
for minibatch_index in range(n_test_batches):
state_reset_counter_lst=[s+1 for s in state_reset_counter_lst]
(LStateList_b,x,y,state_reset_counter_lst)=du.prepare_lstm_batch(index_test_list, minibatch_index, batch_size, S_Test_list, LStateList_t,LStateList_pre, F_list_test, params, Y_test, X_test,state_reset_counter_lst)
LStateList_pre=LStateList_b
args=(x,is_train)+tuple(LStateList_b)
result = model.predictions(*args)
pred=result[0]
LStateList_t=result[1:len(result)]
loss3d =u.get_loss(params,y,pred)
batch_loss3d.append(loss3d)
batch_loss3d=np.nanmean(batch_loss3d)
if(batch_loss3d<best_loss):
best_loss=batch_loss3d
ext=str(epoch_counter)+"_"+str(batch_loss3d)+"_best.p"
u.write_params(model.params,params,ext)
else:
ext=str(val_counter%2)+".p"
u.write_params(model.params,params,ext)
val_counter+=1#0.08
s ='VAL--> epoch %i | error %f, %f'%(val_counter,batch_loss3d,n_test_batches)
u.log_write(s,params)
def train_rnn(params):
data = []
for sindex in range(0, params["seq_length"], 5):
(X_train, Y_train, X_test, Y_test) = du.load_pose(params, sindex=sindex)
data.append((X_train, Y_train, X_test, Y_test))
(X_train, Y_train, X_test, Y_test) = data[0]
params["len_train"] = X_train.shape[0] * X_train.shape[1]
params["len_test"] = X_test.shape[0] * X_test.shape[1]
u.start_log(params)
batch_size = params["batch_size"]
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
nb_epochs = params["n_epochs"]
print("Batch size: %i, train batch size: %i, test batch size: %i" % (batch_size, n_train_batches, n_test_batches))
u.log_write("Model build started", params)
if params["resume"] == 1:
model = model_provider.get_model_pretrained(params)
else:
model = model_provider.get_model(params)
u.log_write("Number of parameters: %s" % (model.n_param), params)
train_errors = np.ndarray(nb_epochs)
u.log_write("Training started", params)
val_counter = 0
best_loss = 10000
for epoch_counter in range(nb_epochs):
(X_train, Y_train, X_test, Y_test) = data[np.mod(epoch_counter, len(data))]
if params["shufle_data"] == 1:
X_train, Y_train = du.shuffle_in_unison_inplace(X_train, Y_train)
n_train_batches = len(X_train)
n_train_batches /= batch_size
n_test_batches = len(X_test)
n_test_batches /= batch_size
batch_loss = 0.0
for minibatch_index in range(n_train_batches):
x = X_train[minibatch_index * batch_size : (minibatch_index + 1) * batch_size] # 60*20*1024
y = Y_train[minibatch_index * batch_size : (minibatch_index + 1) * batch_size] # 60*20*54
if params["model"] == "blstmnp":
x_b = np.asarray(map(np.flipud, x))
loss = model.train(x, x_b, y)
else:
loss = model.train(x, y)
batch_loss += loss
train_errors[epoch_counter] = batch_loss
batch_loss /= n_train_batches
s = "TRAIN--> epoch %i | error %f" % (epoch_counter, batch_loss)
u.log_write(s, params)
if epoch_counter % 10 == 0:
print("Model testing")
batch_loss = 0.0
batch_loss3d = 0.0
for minibatch_index in range(n_test_batches):
x = X_test[minibatch_index * batch_size : (minibatch_index + 1) * batch_size]
y = Y_test[minibatch_index * batch_size : (minibatch_index + 1) * batch_size]
if params["model"] == "blstmnp":
x_b = np.asarray(map(np.flipud, x))
pred = model.predictions(x, x_b)
else:
pred = model.predictions(x)
loss = np.nanmean(np.abs(pred - y) ** 2)
loss3d = u.get_loss(y, pred)
batch_loss += loss
batch_loss3d += loss3d
batch_loss /= n_test_batches
batch_loss3d /= n_test_batches
if batch_loss3d < best_loss:
best_loss = batch_loss3d
ext = str(batch_loss3d) + "_best.p"
u.write_params(model.params, params, ext)
else:
ext = str(val_counter % 2) + ".p"
u.write_params(model.params, params, ext)
val_counter += 1 # 0.08
s = "VAL--> epoch %i | error %f, %f %f" % (val_counter, batch_loss, batch_loss3d, n_test_batches)
u.log_write(s, params)