def MotionNet(order,
epoch,
batch_size,
save_period,
optimizer='sgd',
learning_rate=0.1,
lr_step=1000,
lr_factor=0.9,
stop_factor_lr=1e-08,
use_gpu=True,
use_cudnn=True,
test=False,
predict_size=5,
time_step=150,
seed_timestep=30,
batch_Frame=5,
frame_time=30):
print("-------------------Motion Net-------------------")
'''1. Data_Loading - bvh_reader'''
Normalization_factor, train_motion, train_label_motion, seed_timestep, pre_timestep, column, file_directory = br.Motion_Data_Preprocessing(
time_step, seed_timestep, batch_Frame)
if test == True:
data = OrderedDict()
data['seed_motion'] = train_motion
label = {'label_motion': train_label_motion}
test_iter = mx.io.NDArrayIter(data=data, label=label)
train_iter = mx.io.NDArrayIter(
data=data, label=label, shuffle=False, last_batch_handle='pad'
) # Motion data is complex and requires sequential learning to learn from easy examples. So shuffle = False
else:
train_motion = train_motion[:predict_size]
train_label_motion = train_label_motion[:predict_size]
data = OrderedDict()
data['seed_motion'] = train_motion
label = {'label_motion': train_label_motion}
train_iter = mx.io.NDArrayIter(
data=data,
label=label,
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad'
) # Motion data is complex and requires sequential learning to learn from easy examples. So shuffle = False
'''2. hyperparameter'''
rnn_layer_number = 1
rnn_hidden_number = 200
fc_number = 200
Dropout_rate = 0.0
if use_cudnn == True and use_gpu == True:
ctx = mx.gpu(0)
elif use_cudnn == False and use_gpu == True:
ctx = mx.gpu(0)
else:
#why? mx.rnn.FusedRNNCell only works on use_cudnn == True and use_gpu == True
ctx = mx.cpu(0)
'''3. Network'''
all_motion = mx.sym.Variable('seed_motion')
seed_motion = mx.sym.transpose(mx.sym.slice_axis(data=all_motion,
axis=1,
begin=0,
end=seed_timestep),
axes=(1, 0, 2)) # (time,batch,column)
label_motion = mx.sym.Variable('label_motion')
e_cell = encoder(use_cudnn=use_cudnn,
layer_number=rnn_layer_number,
hidden_number=rnn_hidden_number,
Dropout_rate=Dropout_rate)
d_cell = decoder(use_cudnn=use_cudnn,
layer_number=rnn_layer_number,
hidden_number=rnn_hidden_number,
Dropout_rate=Dropout_rate)
e_output, e_state = e_cell.unroll(length=seed_timestep,
inputs=seed_motion,
merge_outputs=False,
layout='TNC') #(batch,hidden)
print(e_state)
#seq2seq in test
if test == True:
e_output_end = e_output[-1] # Shape: (1, N, C)
e_output_end = mx.sym.reshape(data=e_output_end, shape=(1, 1, -1))
d_input = mx.sym.broadcast_to(e_output_end, shape=(pre_timestep, 0, 0))
d_output, d_state = d_cell.unroll(length=pre_timestep,
begin_state=e_state,
inputs=d_input,
merge_outputs=False,
layout='TNC')
#seq2seq in training
else:
e_output_end = e_output[-1] # Shape: (1, N, C)
e_output_end = mx.sym.reshape(data=e_output_end,
shape=(1, batch_size, -1))
d_input = mx.sym.broadcast_to(e_output_end, shape=(pre_timestep, 0, 0))
d_output, d_state = d_cell.unroll(length=pre_timestep,
begin_state=e_state,
inputs=d_input,
merge_outputs=False,
layout='TNC')
# if use_cudnn=False , begin_state = e_state do not working.
if use_cudnn:
rnn_output = mx.sym.Reshape(d_output[-1],
shape=(-1, 2 * rnn_hidden_number))
else:
rnn_output = d_output[-1]
# if you use dropout
# rnn_output = mx.sym.Dropout(data=rnn_output,p=0.3)
affine1 = mx.sym.FullyConnected(
data=rnn_output, num_hidden=fc_number,
name='affine1') # if use_cudnn=False , data=state[-1] i
act1 = mx.sym.Activation(data=affine1, act_type='tanh', name='tanh1')
drop1 = mx.sym.Dropout(act1, p=Dropout_rate)
affine2 = mx.sym.FullyConnected(data=drop1,
num_hidden=fc_number,
name='affine2')
act2 = mx.sym.Activation(data=affine2, act_type='tanh', name='tanh2')
drop2 = mx.sym.Dropout(act2, p=Dropout_rate)
output = mx.sym.FullyConnected(data=drop2,
num_hidden=pre_timestep * column,
name='affine3')
output = mx.sym.LinearRegressionOutput(data=output, label=label_motion)
# We visualize the network structure with output size (the batch_size is ignored.) - In pycharm or vs code, not working
if use_cudnn:
#shape = {'seed_motion': (seed_timestep, batch_size, column)
mx.viz.plot_network(
symbol=output) # The diagram can be found on the Jupiter notebook.
else:
#shape = {'seed_motion': (batch_size, seed_timestep , column)}
mx.viz.plot_network(
symbol=output) # The diagram can be found on the Jupiter notebook.
print(output.list_arguments())
# training mod
mod = mx.module.Module(symbol=output,
data_names=['seed_motion'],
label_names=['label_motion'],
context=ctx)
# Network information print
print(mod.data_names)
print(mod.label_names)
if test == False:
print(train_iter.provide_data)
print(train_iter.provide_label)
else:
print(test_iter.provide_data)
print(test_iter.provide_label)
'''
grad_req (str, list of str, dict of str to str) –
Requirement for gradient accumulation. Can be ‘write’, ‘add’, or ‘null’
(default to ‘write’). Can be specified globally (str) or for each argument (list, dict).
'''
mod.bind(data_shapes=train_iter.provide_data,
label_shapes=train_iter.provide_label,
for_training=True,
shared_module=None,
inputs_need_grad=False,
force_rebind=False,
grad_req='write')
# weights load
cudnn_weights_path = 'weights/Cudnn_MotionNet-{}th-{}.params'.format(
order, save_period)
weights_path = 'weights/MotionNet-{}th-{}.params'.format(
order, save_period)
if os.path.exists(
cudnn_weights_path) and use_cudnn == True and use_gpu == True:
mod.load_params(cudnn_weights_path)
elif (os.path.exists(weights_path) and use_cudnn == False and use_gpu
== True) and (os.path.exists(weights_path) and use_cudnn == True
and use_gpu == False) and (os.path.exists(weights_path)
and use_cudnn == False
and use_gpu == False):
mod.load_params(weights_path)
else:
mod.init_params(initializer=mx.initializer.Xavier(
rnd_type='gaussian', factor_type='avg', magnitude=1))
if optimizer == 'sgd':
lr_sch = mx.lr_scheduler.FactorScheduler(step=lr_step,
factor=lr_factor,
stop_factor_lr=stop_factor_lr)
mod.init_optimizer(optimizer=optimizer,
optimizer_params={
'learning_rate': learning_rate,
'lr_scheduler': lr_sch
})
else:
mod.init_optimizer(optimizer=optimizer,
optimizer_params={'learning_rate': learning_rate})
metric = mx.metric.create(['mse'])
start_time = time.time()
if test == False:
for epoch in range(1, epoch + 1, 1):
train_iter.reset()
for batch in train_iter:
mod.forward(batch, is_train=True)
mod.update_metric(metric, batch.label)
mod.backward()
mod.update()
print('Epoch : {}'.format(epoch, metric.get()))
if epoch % 100 == 0:
end_time = time.time()
print(
"-------------------------------------------------------")
print("{}_learning time : {}".format(epoch,
end_time - start_time))
print(
"-------------------------------------------------------")
cal = mod.predict(eval_data=train_iter,
merge_batches=True,
reset=True,
always_output_list=False).asnumpy()
cost = cal - train_label_motion
cost = (cost**2) / 2
cost = np.mean(cost)
print("joint angle Square Error : {}".format(cost))
if cost < 0.01:
if not os.path.exists("weights"):
os.makedirs("weights")
if use_cudnn:
print('Saving weights')
mod.save_params(
"weights/Cudnn_MotionNet-{}th-{}.params".format(
order, epoch))
else:
print('Saving weights')
mod.save_params("weights/MotionNet-{}th-{}.params".format(
order, epoch))
print(
"############################################################################################"
)
print("End the learning.")
print(
"############################################################################################"
)
#training-data_test
seed = train_iter.data[0][1].asnumpy()
prediction_motion = mod.predict(
eval_data=train_iter,
merge_batches=True,
reset=True,
always_output_list=False).asnumpy() / Normalization_factor
'''Creating a bvh file with predicted values -bvh_writer'''
bw.Motion_Data_Making(seed[:, :seed_timestep] /
Normalization_factor,
prediction_motion,
seed_timestep,
pre_timestep,
batch_Frame,
frame_time,
file_directory,
test=False)
return "completed", epoch
# Network information print
print(mod.data_shapes)
print(mod.label_shapes)
print(mod.output_shapes)
print(mod.get_params())
print(mod.get_outputs())
print("Optimization complete")
#tall data test
if test == True:
# test mod
test_mod = mx.mod.Module(symbol=output,
data_names=['seed_motion'],
label_names=['label_motion'],
context=ctx)
test_mod.bind(data_shapes=test_iter.provide_data,
label_shapes=test_iter.provide_label,
for_training=False,
shared_module=mod)
if os.path.exists(cudnn_weights_path) or os.path.exists(
weights_path): #FusedRNN
seed = test_iter.data[0][1].asnumpy()
prediction_motion = test_mod.predict(
eval_data=test_iter,
merge_batches=True,
always_output_list=False).asnumpy() / Normalization_factor
'''Creating a bvh file with predicted values -bvh_writer'''
bw.Motion_Data_Making(seed[:, :seed_timestep],
prediction_motion,
seed_timestep,
pre_timestep,
batch_Frame,
frame_time,
file_directory,
test=True)
else:
print("Can not test")
print("finished")
def MotionNet(order=None,
epoch=None,
batch_size=None,
save_period=None,
cost_limit=None,
optimizer=None,
learning_rate=None,
lr_step=None,
lr_factor=None,
stop_factor_lr=None,
use_gpu=True,
TEST=None,
num_layer=None,
cell=None,
hidden_unit=None,
time_step=None,
seed_timestep=None,
batch_Frame=None,
frame_time=None,
graphviz=None):
print("-------------------Motion Net-------------------")
'''1. Data_Loading - bvh_reader'''
Normalization_factor, train_motion, train_label_motion, seed_timestep, pre_timestep, column, file_directory = br.Motion_Data_Preprocessing(
time_step, seed_timestep, batch_Frame, TEST)
if TEST == True:
print("<TEST>")
data = OrderedDict()
data['seed_motion'] = train_motion
label = {'label_motion': train_label_motion}
test_iter = mx.io.NDArrayIter(data=data, label=label)
else:
print("<Training>")
train_motion = train_motion[:batch_size]
train_label_motion = train_label_motion[:batch_size]
data = OrderedDict()
data['seed_motion'] = train_motion
label = {'label_motion': train_label_motion}
train_iter = mx.io.NDArrayIter(
data=data,
label=label,
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad'
) # Motion data is complex and requires sequential learning to learn from easy examples. So shuffle = False
if use_gpu:
ctx = mx.gpu(0)
else:
ctx = mx.cpu(0)
'''2. Network'''
all_motion = mx.sym.Variable('seed_motion')
label_motion = mx.sym.Variable('label_motion')
seed_motion = mx.sym.slice_axis(data=all_motion,
axis=1,
begin=0,
end=seed_timestep) # (batch,time,column)
if TEST == True:
pre_motion = mx.sym.reshape(
mx.sym.slice_axis(data=all_motion,
axis=1,
begin=seed_timestep,
end=seed_timestep + 1),
shape=(1, -1)) # (batch=1,column) - first frame
else:
pre_motion = mx.sym.slice_axis(data=all_motion,
axis=1,
begin=seed_timestep - 1,
end=-1)
print("-------------------Network Shape--------------------")
e_cell = encoder(layer_number=num_layer,
hidden_number=hidden_unit,
Dropout_rate=0.2,
cell=cell)
d_cell = decoder(layer_number=num_layer,
hidden_number=hidden_unit,
output_number=column,
Dropout_rate=0.2,
cell=cell)
print("\n")
_, e_state = e_cell.unroll(length=seed_timestep,
inputs=seed_motion,
merge_outputs=True,
layout='NTC') #(batch,hidden)
#seq2seq in test
if TEST == True:
# customized by JG
if num_layer == 1:
d_output, _ = d_cell.SingleLayer_feed_previous_unroll(
length=pre_timestep,
begin_state=e_state,
inputs=pre_motion,
merge_outputs=True,
layout='NTC'
) # MultiLayer_feed_previous_unroll is also possible.
else:
d_output, _ = d_cell.MultiLayer_feed_previous_unroll(
length=pre_timestep,
begin_state=e_state,
inputs=pre_motion,
merge_outputs=True,
layout='NTC')
#seq2seq in training
else:
d_output, _ = d_cell.unroll(length=pre_timestep,
begin_state=e_state,
inputs=pre_motion,
merge_outputs=True,
layout='NTC')
output = mx.sym.LinearRegressionOutput(data=d_output, label=label_motion)
digraph = mx.viz.plot_network(symbol=output, hide_weights=True)
#why? batch_Frame>=10 ? -> graph 'plot' size too small for label
if graphviz == True and TEST == True and batch_Frame >= 10:
digraph.view(
"{}_batch_Frame_TEST_Seq2Seq".format(batch_Frame)) #show graph
elif graphviz == True and TEST == False and order == 1 and batch_Frame >= 10:
digraph.view(
"{}_batch_Frame_Training_Seq2Seq".format(batch_Frame)) #show graph
print("-------------------Network Learning Parameter--------------------")
print(output.list_arguments())
print("\n")
if TEST == False:
mod = mx.module.Module(symbol=output,
data_names=['seed_motion'],
label_names=['label_motion'],
context=ctx)
print("-------------------Network Data Name--------------------")
print(mod.data_names)
print(mod.label_names)
print("\n")
else:
# test mod
test_mod = mx.mod.Module(symbol=output,
data_names=['seed_motion'],
label_names=['label_motion'],
context=ctx)
print("-------------------Network Data Name--------------------")
print(test_mod.data_names)
print(test_mod.label_names)
print("\n")
print("-------------------Network Data Shape--------------------")
if TEST == False:
print(train_iter.provide_data)
print(train_iter.provide_label)
else:
print(test_iter.provide_data)
print(test_iter.provide_label)
print("\n")
'''
grad_req (str, list of str, dict of str to str) –
Requirement for gradient accumulation. Can be ‘write’, ‘add’, or ‘null’
(default to ‘write’). Can be specified globally (str) or for each argument (list, dict).
'''
if TEST == False:
mod.bind(data_shapes=train_iter.provide_data,
label_shapes=train_iter.provide_label,
for_training=True,
shared_module=None,
inputs_need_grad=False,
force_rebind=False,
grad_req='write')
# weights load
weights_path = 'weights/MotionNet-{}th-{}.params'.format(
order, save_period)
if os.path.exists(weights_path):
mod.load_params(weights_path)
else:
mod.init_params(initializer=mx.initializer.Xavier(
rnd_type='gaussian', factor_type='avg', magnitude=1))
start_time = time.time()
print("-------------------Learning Start--------------------")
if optimizer == 'sgd':
lr_sch = mx.lr_scheduler.FactorScheduler(
step=lr_step, factor=lr_factor, stop_factor_lr=stop_factor_lr)
mod.init_optimizer(optimizer=optimizer,
optimizer_params={
'learning_rate': learning_rate,
'lr_scheduler': lr_sch
})
else:
mod.init_optimizer(
optimizer=optimizer,
optimizer_params={'learning_rate': learning_rate})
metric = mx.metric.create(['mse'])
for epoch in range(1, epoch + 1, 1):
train_iter.reset()
metric.reset()
for batch in train_iter:
mod.forward(batch, is_train=True)
#Data Order Transform (N,T,C)
mod.update_metric(metric, batch.label)
mod.backward()
mod.update()
#print('Epoch : {} , MSE : {}'.format(epoch,metric.get()))
if epoch % 100 == 0:
end_time = time.time()
print(
"-------------------------------------------------------")
print("{}_learning time : {}".format(epoch,
end_time - start_time))
print(
"-------------------------------------------------------")
if epoch % 10000 == 0:
if not os.path.exists("weights"):
os.makedirs("weights")
print('Saving weights')
mod.save_params("weights/MotionNet-{}.params".format(epoch))
cal = mod.predict(eval_data=train_iter,
merge_batches=True,
reset=True,
always_output_list=False).asnumpy()
cost = cal - train_label_motion
cost = (cost**2) / 2
cost = np.mean(cost)
print('{} epoch '.format(epoch), end='')
print("Joint Angle Square Error : {}".format(cost))
if cost < cost_limit:
if not os.path.exists("weights"):
os.makedirs("weights")
print('Saving weights')
mod.save_params("weights/MotionNet-{}th-{}.params".format(
order, epoch))
print(
"############################################################################################"
)
print("End the learning.")
print(
"############################################################################################"
)
#order : (N , T(all_time) , C)
seed = train_iter.data[0][1].asnumpy()
#order : (N , T(predict time) , C)
prediction_motion = mod.predict(
eval_data=train_iter,
merge_batches=True,
reset=True,
always_output_list=False).asnumpy() / Normalization_factor
'''Creating a bvh file with predicted values -bvh_writer'''
bw.Motion_Data_Making(
seed[:, :seed_timestep] / Normalization_factor,
prediction_motion, seed_timestep, pre_timestep,
batch_Frame, frame_time, file_directory, TEST)
return "completed", epoch
print("\n")
print("-------------------Network Information--------------------")
print(mod.data_shapes)
print(mod.label_shapes)
print(mod.output_shapes)
print(mod.get_params())
print(mod.get_outputs())
print("Optimization complete")
print("\n")
if TEST == True:
test_mod.bind(data_shapes=test_iter.provide_data,
label_shapes=test_iter.provide_label,
for_training=False)
# weights load
weights_path = 'weights/MotionNet-{}th-{}.params'.format(
order, save_period)
if os.path.exists(weights_path):
test_mod.load_params(weights_path)
#order : (N , T(all time) , C)
seed = test_iter.data[0][1].asnumpy()
#order : (N , T(predict time) , C)
prediction_motion = test_mod.predict(
eval_data=test_iter,
merge_batches=True,
always_output_list=False).asnumpy() / Normalization_factor
print("Test Prediction motion shape : {}".format(
np.shape(prediction_motion)))
#test cost
cost = prediction_motion - train_label_motion
cost = (cost**2) / 2
cost = np.mean(cost)
print("prediction error : {}".format(cost))
'''Creating a bvh file with predicted values -bvh_writer'''
bw.Motion_Data_Making(
seed[:, :seed_timestep] / Normalization_factor,
prediction_motion, seed_timestep, pre_timestep, batch_Frame,
frame_time, file_directory, TEST)
else:
print("Can not test")
def MotionNet(epoch=None , batch_size=None , save_period=None , cost_limit=None ,
optimizer=None, learning_rate=None , lr_step=None , lr_factor=None , stop_factor_lr=None , use_gpu=True ,
TEST=None , num_layer=None , cell=None, hidden_unit=None ,time_step = None , seed_timestep = None , batch_Frame= None , frame_time=None, graphviz=None , parameter_shared=True , Model = None):
print("-------------------Motion Net-------------------")
'''1. Data_Loading - bvh_reader'''
Normalization_factor, train_motion, train_label_motion , seed_timestep, pre_timestep, column, file_directory = br.Motion_Data_Preprocessing(time_step , seed_timestep , batch_Frame , TEST , Model)
if TEST==True:
print("<TEST>")
data = OrderedDict()
data['seed_motion'] = train_motion
label = {'label_motion': train_label_motion}
test_iter = mx.io.NDArrayIter(data=data, label=label)
else:
print("<Training>")
data = OrderedDict()
data['seed_motion'] = train_motion
label = {'label_motion': train_label_motion}
train_iter = mx.io.NDArrayIter(data=data, label=label, batch_size=batch_size, shuffle=False, last_batch_handle='pad') # Motion data is complex and requires sequential learning to learn from easy examples. So shuffle = False ->In here, not using sequential learning
if use_gpu:
ctx = mx.gpu(0)
else:
ctx = mx.cpu(0)
'''2. Network'''
all_motion = mx.sym.Variable('seed_motion')
label_motion = mx.sym.Variable('label_motion')
seed_motion = mx.sym.slice_axis(data=all_motion , axis=1 , begin = 0 , end = seed_timestep) # (batch , time , column)
if TEST == True:
pre_motion = mx.sym.reshape(mx.sym.slice_axis(data=all_motion, axis=1, begin=seed_timestep-1, end=seed_timestep),
shape=(1, -1)) # (batch=1,column) - first frame
else:
pre_motion = mx.sym.reshape(mx.sym.slice_axis(data=all_motion, axis=1, begin=seed_timestep-1, end=seed_timestep),
shape=(batch_size, -1)) # (batch,column) - first frame
print("-------------------Network Shape--------------------")
'''
only if encoder's parameter_shared=True , parameter Encoder and decoder parameters are shared.
'''
e_cell , encoder_parameter = encoder(layer_number= num_layer , hidden_number=hidden_unit , Dropout_rate=0.0 , Zoneout_rate=0.0 , cell=cell , parameter_shared = parameter_shared ) # if parameter_shared = True, paramter = encoder's weights , else False = []
if num_layer==1 and parameter_shared==True: #only if num_layer=1 , Both Residual = True and Residual = False are possible.
d_cell = decoder(layer_number= num_layer , hidden_number=hidden_unit , output_number=column , Dropout_rate=0.0 , Zoneout_rate=0.0 , Residual = True , cell=cell , param=encoder_parameter)
elif num_layer>1 and parameter_shared==True:# There is no way to share parameters with the encoder.
d_cell = decoder(layer_number=num_layer, hidden_number=hidden_unit, output_number=column, Dropout_rate=0.0,Zoneout_rate=0.0, Residual=False, cell=cell, param=encoder_parameter)
else: # parameter_shared=False , Both Residual = True and Residual = False are possible.
d_cell = decoder(layer_number=num_layer, hidden_number=hidden_unit, output_number=column, Dropout_rate=0.0,Zoneout_rate=0.0, Residual=True, cell=cell, param=encoder_parameter)
print("\n")
_ , e_state = e_cell.unroll(length=seed_timestep , inputs=seed_motion , merge_outputs=True , layout='NTC')
# customized by JG
if num_layer==1:
d_output, _ = d_cell.SingleLayer_feed_previous_unroll(length=pre_timestep, begin_state=e_state,inputs=pre_motion, merge_outputs=True, layout='NTC') # MultiLayer_feed_previous_unroll is also possible.
else:
d_output, _ = d_cell.MultiLayer_feed_previous_unroll(length=pre_timestep, begin_state=e_state, inputs=pre_motion, merge_outputs=True, layout='NTC') # MultiLayer_feed_previous_unroll is also possible.
#output = mx.sym.LinearRegressionOutput(data = d_output , label=label_motion , grad_scale = 1)
output = mx.sym.Custom(data=d_output, label=label_motion, grad_scale=1, name="LinearRegression", op_type='LinearRegression')
digraph=mx.viz.plot_network(symbol=output,hide_weights=True)
#why? batch_Frame>=10 ? -> graph 'plot' size too small for label
if graphviz==True and TEST == True and batch_Frame>=10:
digraph.view("{}_batch_Frame_TEST_Seq2Seq".format(batch_Frame)) #show graph
elif graphviz==True and TEST == False and batch_Frame>=10:
digraph.view("{}_batch_Frame_Training_Seq2Seq".format(batch_Frame)) #show graph
print("-------------------Network Learning Parameter--------------------")
print(output.list_arguments())
print("\n")
if TEST == False:
mod = mx.module.Module(symbol=output, data_names=['seed_motion'], label_names=['label_motion'], context=ctx)
print("-------------------Network Data Name--------------------")
print(mod.data_names)
print(mod.label_names)
print("\n")
else:
# test mod
test_mod = mx.mod.Module(symbol=output , data_names=['seed_motion'] , label_names=['label_motion'] , context=ctx)
print("-------------------Network Data Name--------------------")
print(test_mod.data_names)
print(test_mod.label_names)
print("\n")
print("-------------------Network Data Shape--------------------")
if TEST==False:
print(train_iter.provide_data)
print(train_iter.provide_label)
else:
print(test_iter.provide_data)
print(test_iter.provide_label)
print("\n")
'''
grad_req (str, list of str, dict of str to str) –
Requirement for gradient accumulation. Can be ‘write’, ‘add’, or ‘null’
(default to ‘write’). Can be specified globally (str) or for each argument (list, dict).
'''
if TEST == False:
mod.bind(data_shapes=train_iter.provide_data , label_shapes=train_iter.provide_label , for_training=True , shared_module=None , inputs_need_grad=False , force_rebind=False , grad_req='write')
# weights load
weights_path = 'weights/MotionNet-{}.params'.format(save_period)
if os.path.exists(weights_path):
print("load weights")
mod.load_params(weights_path)
else:
print("init weights")
mod.init_params(initializer=mx.initializer.Normal(sigma=0.01)) # very important
start_time=time.time()
print("-------------------Learning Start--------------------")
if optimizer=='sgd':
lr_sch = mx.lr_scheduler.FactorScheduler(step = lr_step, factor = lr_factor , stop_factor_lr = stop_factor_lr)
mod.init_optimizer(optimizer=optimizer, optimizer_params={'learning_rate': learning_rate , 'lr_scheduler': lr_sch})
else:
mod.init_optimizer(optimizer=optimizer, optimizer_params={'learning_rate': learning_rate})
metric = mx.metric.create(['mse'])
for epoch in range(1, epoch + 1, 1):
train_iter.reset()
metric.reset()
for batch in train_iter:
if epoch % 2 == 0: # Only noise is added when it is an even number
'''1. Add noise to input - Data Augmentation'''
#random_normal
noise = mx.nd.random_normal(loc=0 , scale=5 , shape=(batch_size , seed_timestep+pre_timestep , column) , ctx=ctx) # random_normal
#random_uniform
#noise = mx.nd.random_uniform(low=-1 , high=1 , shape=(batch_size , seed_timestep+pre_timestep , column) ,ctx=ctx) # random_uniform
mod.forward(data_batch=mx.io.DataBatch(data = list([mx.nd.add(batch.data[0].as_in_context(ctx), noise)]), label= list(batch.label)), is_train=True)
#Data Order Transform (N,T,C)
else:
mod.forward(data_batch=batch , is_train=True)
mod.update_metric(metric,batch.label)
mod.backward()
mod.update()
#print('epoch : {} , MSE : {}'.format(epoch,metric.get()))
if epoch % 100 == 0:
end_time=time.time()
print("-------------------------------------------------------")
print("{}_learning time : {}".format(epoch,end_time-start_time))
print("-------------------------------------------------------")
if epoch % 10000 == 0:
if not os.path.exists("weights"):
os.makedirs("weights")
print('Saving weights')
mod.save_params("weights/MotionNet-{}.params".format(epoch))
cal = mod.predict(eval_data=train_iter , merge_batches=True , reset=True, always_output_list=False).asnumpy() / Normalization_factor
cost = cal - train_label_motion
cost=(cost**2)/2
cost=np.mean(cost)
print('{} epoch '.format(epoch), end='')
print("Joint Angle Square Error : {}".format(cost))
if cost < cost_limit:
if not os.path.exists("weights"):
os.makedirs("weights")
print('Saving weights')
mod.save_params("weights/MotionNet-{}.params".format(epoch))
print("############################################################################################")
print("End the learning.")
print("############################################################################################")
return "optimization completed"
print("\n")
print("-------------------Network Information--------------------")
print(mod.data_shapes)
print(mod.label_shapes)
print(mod.output_shapes)
print(mod.get_params())
print(mod.get_outputs())
print("{} learning optimization completed".format(epoch))
print("\n")
if TEST==True:
test_mod.bind(data_shapes=test_iter.provide_data , label_shapes=test_iter.provide_label , for_training=False)
# weights load
weights_path = 'weights/MotionNet-{}.params'.format(save_period)
if os.path.exists(weights_path):
test_mod.load_params(weights_path)
#order : (N , T(all time) , C)
seed = test_iter.data[0][1].asnumpy()
#order : (N , T(predict time) , C)
prediction_motion = test_mod.predict(eval_data=test_iter , merge_batches=True , always_output_list=False).asnumpy()/Normalization_factor
print("Test Prediction motion shape : {}".format(np.shape(prediction_motion)))
#test cost
cost = prediction_motion - train_label_motion
cost=(cost**2)/2
print(style.available)
TimeStepError_Array=np.mean(cost,axis=(0,2)) # y-axis
print(TimeStepError_Array)
TimeStep = np.arange(1,pre_timestep+1,1) # x-axis
#Dram Error graph
style.use('seaborn')
plt.figure(figsize=(9,4))
bbox = dict(boxstyle = 'round' , fc = 'w' , ec = 'b' , lw = 2)
#plt.plot(TimeStep , TimeStepError_Array , "r." , lw=3 ,label = "Error")
plt.bar(TimeStep , TimeStepError_Array , width=0.7 ,label ='error', color = 'r')
plt.annotate("Error Prevention" ,fontsize=14, xy = (60,1000) , xytext=(0,1000), textcoords='data' ,arrowprops={'color' : 'blue' , 'alpha' : 0.3 , 'arrowstyle' : "simple"}, bbox = bbox)
plt.grid()
plt.xlabel("Time", fontsize=14)
plt.ylabel("Joint Angle Error" , fontsize=14)
plt.ylim(0,4400)
plt.legend(fontsize=15,loc='upper left')
plt.title("Prediction Error Graph", fontdict={'fontsize': 15 , 'fontweight' : 5})
print("cost graph saved")
plt.savefig("Cost Graph.jpg")
cost=np.mean(cost)
print("prediction error : {}".format(cost))
'''Creating a bvh file with predicted values -bvh_writer'''
bw.Motion_Data_Making(seed[:,:seed_timestep] / Normalization_factor , prediction_motion , seed_timestep , pre_timestep , batch_Frame , frame_time , file_directory , Model)
plt.show()
return "Test completed"
else:
print("Can not test")
def MotionNet(epoch=None,
batch_size=None,
save_period=None,
optimizer=None,
learning_rate=None,
Dropout=0.0,
use_gpu=True,
use_cudnn=True,
TEST=None,
num_layer=None,
hidden_unit=None,
time_step=None,
batch_Frame=None,
graphviz=None):
print("-------------------Motion Net-------------------")
if use_gpu:
ctx = mx.gpu(0)
else:
ctx = mx.cpu(0)
'''1. Data_Loading - bvh_reader'''
Normalization_factor, motion_data, motion_label, time_step, class_number, column = br.Motion_Data_Preprocessing(
time_step, batch_Frame, TEST)
motion_one_hot_label = mx.nd.one_hot(mx.nd.array(motion_label, ctx),
class_number)
if TEST == True:
print("<Test>")
data = OrderedDict()
data['motion'] = motion_data
label = {'label': motion_label}
test_iter = mx.io.NDArrayIter(data=data, label=label)
else:
print("<Training>")
data = OrderedDict()
data['motion'] = motion_data
label = {'label': motion_label}
train_iter = mx.io.NDArrayIter(
data=data,
label=label,
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad'
) # Motion data is complex and requires sequential learning to learn from easy examples. So shuffle = False ->In here, not using sequential learning
'''2. Network'''
motion = mx.sym.Variable('motion') #(batch , time ,column)
label = mx.sym.Variable('label')
motion = mx.sym.transpose(motion,
axes=(1, 0, 2)) # (time , batch , column)
cell = mx.rnn.SequentialRNNCell()
if use_cudnn: # even faster - real!!!
for i in range(num_layer):
#When dropout is set to non-zero in FusedRNNCell, the dropout is applied to the output of all layers except the last layer. If there is only one layer in the FusedRNNCell, the dropout rate is ignored.
cell.add(
mx.rnn.FusedRNNCell(num_hidden=hidden_unit,
num_layers=1,
bidirectional=False,
mode="lstm",
prefix="lstm_{}".format(i),
params=None,
forget_bias=1.0,
get_next_state=False))
cell.add(
mx.rnn.DropoutCell(Dropout,
prefix="lstm_dropout{}_".format(i)))
else: # general
for i in range(num_layer):
cell.add(
mx.rnn.LSTMCell(num_hidden=hidden_unit,
prefix="lstm{}_".format(i)))
cell.add(
mx.rnn.DropoutCell(Dropout,
prefix="lstm_dropout{}_".format(i)))
print("-------------------Network Shape--------------------")
output, _ = cell.unroll(length=time_step,
inputs=motion,
merge_outputs=False,
layout='TNC')
'''FullyConnected Layer'''
#if use_cudnn:
# output = mx.sym.Reshape(output[-1], shape=(-1, hidden_unit))
#else:
# output = output[-1]
affine1 = mx.sym.FullyConnected(
data=output[-1], num_hidden=hidden_unit,
name='affine1') # if use_cudnn=False , data=state[-1] i
act1 = mx.sym.Activation(data=affine1, act_type='relu', name='relu')
dropout1 = mx.sym.Dropout(act1, p=Dropout)
affine2 = mx.sym.FullyConnected(data=dropout1,
num_hidden=class_number,
name='affine2')
output = mx.sym.SoftmaxOutput(data=affine2,
label=label,
grad_scale=1,
name="softmaxoutput")
digraph = mx.viz.plot_network(symbol=output, hide_weights=True)
if graphviz == True:
digraph.view("Network Structure") #show graph
print("-------------------Network Learning Parameter--------------------")
print(output.list_arguments())
print("\n")
if TEST == False:
mod = mx.module.Module(symbol=output,
data_names=['motion'],
label_names=['label'],
context=ctx)
print("-------------------Network Data Name--------------------")
print(mod.data_names)
print(mod.label_names)
print("\n")
else:
# test mod
test_mod = mx.mod.Module(symbol=output,
data_names=['motion'],
label_names=['label'],
context=ctx)
print("-------------------Network Data Name--------------------")
print(test_mod.data_names)
print(test_mod.label_names)
print("\n")
print("-------------------Network Data Shape--------------------")
if TEST == False:
print(train_iter.provide_data)
print(train_iter.provide_label)
else:
print(test_iter.provide_data)
print(test_iter.provide_label)
print("\n")
if TEST == False:
mod.bind(data_shapes=train_iter.provide_data,
label_shapes=train_iter.provide_label,
for_training=True,
shared_module=None,
inputs_need_grad=False,
force_rebind=False,
grad_req='write')
# weights load
if use_cudnn:
weights_path = 'weights/cudnn_MotionNet-{}.params'.format(
save_period)
else:
weights_path = 'weights/MotionNet-{}.params'.format(save_period)
if os.path.exists(weights_path):
print("Load weights")
mod.load_params(weights_path)
else:
print("Init weights")
mod.init_params(initializer=mx.initializer.Normal(
sigma=0.01)) # very important
start_time = time.time()
print("-------------------Learning Start--------------------")
mod.init_optimizer(optimizer=optimizer,
optimizer_params={'learning_rate': learning_rate})
for epoch in range(1, epoch + 1, 1):
train_iter.reset()
for batch in train_iter:
if epoch % 2 == 0: # Only noise is added when it is an even number
'''1. Add noise to input - Data Augmentation'''
#random_normal
#noise = mx.nd.random_normal(loc=0 , scale=20 , shape=(batch_size , time_step , column) , ctx=ctx) # random_normal
#random_uniform
noise = mx.nd.random_uniform(low=-10,
high=10,
shape=(batch_size, time_step,
column),
ctx=ctx) # random_uniform
mod.forward(data_batch=mx.io.DataBatch(data=list(
[mx.nd.add(batch.data[0].as_in_context(ctx), noise)]),
label=list(
batch.label)),
is_train=True)
else:
mod.forward(data_batch=batch, is_train=True)
mod.backward()
mod.update()
if not os.path.exists("weights"):
os.makedirs("weights")
if epoch % 10000 == 0:
end_time = time.time()
print(
"-------------------------------------------------------")
print("{}_learning time : {}".format(epoch,
end_time - start_time))
print(
"-------------------------------------------------------")
if epoch % 10000 == 0 and use_cudnn == True:
print('Saving weights')
mod.save_params(
"weights/cudnn_MotionNet-{}.params".format(epoch))
elif epoch % 10000 == 0 and use_cudnn == False:
print('Saving weights')
mod.save_params("weights/MotionNet-{}.params".format(epoch))
motion_class = mod.predict(
eval_data=train_iter,
merge_batches=True,
reset=True,
always_output_list=False) / Normalization_factor
cost = motion_class - motion_one_hot_label
cost = mx.nd.divide(mx.nd.square(cost), 2)
cost = mx.nd.mean(cost)
print("epoch : {} , ".format(epoch), end="")
print("cost : {}".format(cost.asscalar()))
print('Training Accuracy : {0:0.4f}%'.format(
float(
sum(motion_class.asnumpy().argmax(axis=1) == motion_label))
/ len(motion_label) * 100.0))
if use_cudnn == True:
print('Saving Final weights')
mod.save_params("weights/cudnn_MotionNet-{}.params".format(epoch))
else:
print('Saving Final weights')
mod.save_params("weights/MotionNet-{}.params".format(epoch))
print("\n")
print("-------------------Network Information--------------------")
print(mod.data_shapes)
print(mod.label_shapes)
print(mod.output_shapes)
#print(mod.get_params())
#print(mod.get_outputs())
print("{} learning optimization completed".format(epoch))
print("\n")
if TEST == True:
test_mod.bind(data_shapes=test_iter.provide_data,
label_shapes=test_iter.provide_label,
for_training=False)
# weights load
if use_cudnn:
weights_path = 'weights/cudnn_MotionNet-{}.params'.format(
save_period)
else:
weights_path = 'weights/MotionNet-{}.params'.format(save_period)
if os.path.exists(weights_path):
test_mod.load_params(weights_path)
motion_class = test_mod.predict(
eval_data=test_iter,
merge_batches=True,
always_output_list=False) / Normalization_factor
print('Test Accuracy : {}%'.format(
float(
sum(motion_class.asnumpy().argmax(axis=1) == motion_label))
/ len(motion_label) * 100.0))
print("Test completed")
else:
print("Can not test")
def MotionNet(epoch,
batch_size,
save_period,
optimizer='sgd',
learning_rate=0.001,
use_cudnn=True):
rnn_hidden_number = 500
fc_number = 200
optimizer = optimizer
learning_rate = learning_rate
use_cudnn = use_cudnn
ctx = mx.gpu(0)
motion_data, class_number, time_step, xyz_rotation = br.Motion_Data_Preprocessing(
)
motion, motion_label = zip(*motion_data)
'''data loading referenced by Data Loading API '''
train_iter = mx.io.NDArrayIter(data={'data': motion},
label={'label': motion_label},
batch_size=batch_size,
shuffle=False) #training data
####################################################-Network-################################################################
data = mx.sym.Variable('data')
label = mx.sym.Variable('label')
data = mx.sym.transpose(data, axes=(1, 0, 2)) # (time,batch,column)
#Fused RNN cell results in better performance on GPU(FusedRNNCell supports GPU-only) -> can't be called or stepped!!
#Fused RNN cell supports Bidirectional , Dropout , multilayer
if use_cudnn: #faster
cell = mx.rnn.SequentialRNNCell()
cell.add(
mx.rnn.FusedRNNCell(num_hidden=rnn_hidden_number,
num_layers=1,
bidirectional=False,
mode="lstm",
prefix="lstm1_",
params=None,
forget_bias=1.0,
get_next_state=True))
cell.add(mx.rnn.DropoutCell(0.3, prefix="lstm_dropout1_"))
cell.add(
mx.rnn.FusedRNNCell(num_hidden=rnn_hidden_number,
num_layers=1,
bidirectional=False,
mode="lstm",
prefix="lstm2_",
params=None,
forget_bias=1.0,
get_next_state=True))
#Note : When dropout is set to non-zero in FusedRNNCell, the dropout is applied to the output of all layers except the last layer. If there is only one layer in the FusedRNNCell, the dropout rate is ignored.
else: #general
cell = mx.rnn.SequentialRNNCell()
cell.add(mx.rnn.LSTMCell(num_hidden=rnn_hidden_number,
prefix="lstm1_"))
cell.add(mx.rnn.DropoutCell(0.3, prefix="lstm_dropout1_"))
cell.add(mx.rnn.LSTMCell(num_hidden=rnn_hidden_number,
prefix="lstm2_"))
#if you see the unroll function
# The hidden state and cell state from the final time step is returned: - state[-1]
layer, state = cell.unroll(length=time_step,
inputs=data,
merge_outputs=True,
layout='TNC')
'''FullyConnected Layer'''
if use_cudnn:
rnn_output = mx.sym.Reshape(state[-1], shape=(-1, rnn_hidden_number))
#or
#rnn_output = mx.sym.Reshape(d_state[-1], shape=(batch_size, rnn_hidden_number)) #using the test_iter, this code don't work, because batch_size is different from train_iter
else:
rnn_output = state[-1]
#if you use dropout
#rnn_output = mx.sym.Dropout(data=rnn_output,p=0.3)
affine1 = mx.sym.FullyConnected(
data=rnn_output, num_hidden=fc_number,
name='affine1') # if use_cudnn=False , data=state[-1] i
act1 = mx.sym.Activation(data=affine1, act_type='sigmoid', name='sigmoid1')
drop1 = mx.sym.Dropout(act1, p=0.3)
affine2 = mx.sym.FullyConnected(data=drop1,
num_hidden=class_number,
name='affine2')
output = mx.sym.SoftmaxOutput(data=affine2, label=label, name='softmax')
# We visualize the network structure with output size (the batch_size is ignored.)
if use_cudnn:
shape = {"data": (time_step, batch_size, xyz_rotation)}
mx.viz.plot_network(
symbol=output,
shape=shape) #The diagram can be found on the Jupiter notebook.
else:
shape = {"data": (batch_size, time_step, xyz_rotation)}
mx.viz.plot_network(
symbol=output,
shape=shape) #The diagram can be found on the Jupiter notebook.
print(output.list_arguments())
# training mod
mod = mx.module.Module(symbol=output,
data_names=['data'],
label_names=['label'],
context=ctx)
# Network information print
print(mod.data_names)
print(mod.label_names)
print(train_iter.provide_data)
print(train_iter.provide_label)
'''if the below code already is declared by mod.fit function, thus we don't have to write it.
but, when you load the saved weights, you must write the below code.'''
mod.bind(data_shapes=train_iter.provide_data,
label_shapes=train_iter.provide_label)
# weights save
if not os.path.exists("weights"):
os.makedirs("weights")
if use_cudnn:
model_name = 'weights/Cudnn_MotionNet'
checkpoint = mx.callback.do_checkpoint(model_name, period=save_period)
else:
model_name = 'weights/MotionNet'
checkpoint = mx.callback.do_checkpoint(model_name, period=save_period)
#weights load
if os.path.exists(
'weights/Cudnn_MotionNet-1000.params') and use_cudnn == True:
symbol, arg_params, aux_params = mx.model.load_checkpoint(
model_name, 1000)
mod.set_params(arg_params, aux_params)
elif os.path.exists(
'weights/MotionNet-1000.params') and use_cudnn == False:
symbol, arg_params, aux_params = mx.model.load_checkpoint(
model_name, 1000)
mod.set_params(arg_params, aux_params)
mod.fit(
train_iter,
initializer=mx.initializer.Xavier(rnd_type='gaussian',
factor_type="avg",
magnitude=1),
optimizer=optimizer,
optimizer_params={'learning_rate': learning_rate},
eval_metric=mx.metric.MSE(),
# Once the loaded parameters are declared here,You do not need to declare mod.set_params,mod.bind
num_epoch=epoch,
arg_params=None,
aux_params=None,
epoch_end_callback=checkpoint)
# Network information print
print(mod.data_shapes)
print(mod.label_shapes)
print(mod.output_shapes)
print(mod.get_params())
print(mod.get_outputs())
print("training_data : {}".format(mod.score(train_iter, ['mse', 'acc'])))
print("Optimization complete.")
'''test'''
result = mod.predict(train_iter).asnumpy().argmax(axis=1)
motion_result = np.argmax(motion_label, axis=1)
print(result)
print(motion_result)
print('Final accuracy : {}%'.format(
float(sum(result == motion_result)) / len(result) * 100.0))
import bvh_reader as br
import bvh_writer as bw
motion, time_step, batch_Frame, frame_Time, file_directory = br.Motion_Data_Preprocessing(
time_step=90, batch_Frame=1)
bw.Motion_Data_Making(motion, time_step, batch_Frame, frame_Time,
file_directory)