#Mean_skel, Std_skel, Mean_CNN, Std_CNN = net_convnet3d_grbm_early_fusion.load_normalisation_constant(load_path)
f = open('/home/zhiquan/fancy/meterials/temp/SK_normalization.pkl', 'rb')
SK_normalization = cPickle.load(f)
Mean_skel = SK_normalization['Mean1']
Std_skel = SK_normalization['Std1']
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(
name='x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here
load_path = '/idiap/user/dwu/chalearn/result/try/36.7% 2015.07.09.17.53.10'
dbn.load_params_DBN(os.path.join(load_path, 'paramsbest.zip'))
test_model = function([],
dbn.logLayer.p_y_given_x,
givens={x_skeleton: x_skeleton_},
on_unused_input='ignore')
for file_count, file in enumerate(samples):
condition = (file_count > -1)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
save_path = os.path.join(data, file)
print file
time_start = time()
def __init__(self, res_dir, load_path):
self.layers = [] # only contain the layers from fusion
self.insp_mean = [] # inspection for each layer mean activation
self.insp_std = [] # inspection for each layer std activation
self.params = [] # parameter list
self.idx_mini = T.lscalar(name="idx_mini") # minibatch index
self.idx_micro = T.lscalar(name="idx_micro") # microbatch index
# symbolic variables
self.x = ndtensor(len(tr.in_shape))(name='x') # video input
self.y = T.ivector(name='y') # labels
# symbolic variables
self.x_skeleton = ndtensor(len(tr._skeleon_in_shape))(
name='x_skeleton') # video input
if use.drop:
drop.p_vid = shared(float32(drop.p_vid_val))
drop.p_hidden = shared(float32(drop.p_hidden_val))
video_cnn = conv3d_chalearn(self.x, use, lr, batch, net, reg, drop, mom, \
tr, res_dir, load_path)
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=self.x_skeleton, label=self.y )
# we load the pretrained DBN skeleton parameteres here
if use.load == True:
dbn.load(os.path.join(load_path, 'dbn_2015-06-19-11-34-24.npy'))
#####################################################################
# fuse the ConvNet output with skeleton output -- need to change here
######################################################################
out = T.concatenate([video_cnn.out, dbn.sigmoid_layers[-1].output],
axis=1)
#####################################################################
# wudi add the mean and standard deviation of the activation values to exam the neural net
# Reference: Understanding the difficulty of training deep feedforward neural networks, Xavier Glorot, Yoshua Bengio
#####################################################################
insp_mean_list = []
insp_std_list = []
insp_mean_list.extend(dbn.out_mean)
insp_mean_list.extend(video_cnn.insp_mean)
insp_std_list.extend(dbn.out_std)
insp_std_list.extend(video_cnn.insp_std)
######################################################################
#MLP layer
self.layers.append(
HiddenLayer(out,
n_in=net.hidden,
n_out=net.hidden,
rng=tr.rng,
W_scale=net.W_scale[-1],
b_scale=net.b_scale[-1],
activation=net.activation))
out = self.layers[-1].output
if tr.inspect:
insp_mean_list.extend([T.mean(out)])
insp_std_list.extend([T.std(out)])
self.insp_mean = T.stacklists(insp_mean_list)
self.insp_std = T.stacklists(insp_std_list)
if use.drop:
out = DropoutLayer(out, rng=tr.rng, p=drop.p_hidden).output
######################################################################
# softmax layer
self.layers.append(
LogRegr(out,
rng=tr.rng,
n_in=net.hidden,
W_scale=net.W_scale[-1],
b_scale=net.b_scale[-1],
n_out=net.n_class))
self.p_y_given_x = self.layers[-1].p_y_given_x
######################################################################
# cost function
self.cost = self.layers[-1].negative_log_likelihood(self.y)
# function computing the number of errors
self.errors = self.layers[-1].errors(self.y)
# parameter list
for layer in video_cnn.layers:
self.params.extend(layer.params)
# pre-trained dbn parameter last layer (W, b) doesn't need to incorporate into the params
# for calculating the gradient
self.params.extend(dbn.params[:-2])
# MLP hidden layer params
self.params.extend(self.layers[-2].params)
# softmax layer params
self.params.extend(self.layers[-1].params)
# number of inputs for MLP = (# maps last stage)*(# convnets)*(resulting video shape) + trajectory size
print 'MLP:', video_cnn.n_in_MLP, "->", net.hidden_penultimate, "+", net.hidden_traj, '->', \
net.hidden, '->', net.hidden, '->', net.n_class, ""
return
def __init__(self, res_dir, load_path):
self.layers = [] # only contain the layers from fusion
self.insp_mean = [] # inspection for each layer mean activation
self.insp_std = [] # inspection for each layer std activation
self.params = [] # parameter list
self.idx_mini = T.lscalar(name="idx_mini") # minibatch index
self.idx_micro = T.lscalar(name="idx_micro") # microbatch index
# symbolic variables
self.x = ndtensor(len(tr.in_shape))(name = 'x') # video input
self.y = T.ivector(name = 'y') # labels
# symbolic variables
self.x_skeleton = ndtensor(len(tr._skeleon_in_shape))(name = 'x_skeleton') # video input
if use.drop:
drop.p_vid = shared(float32(drop.p_vid_val) )
drop.p_hidden = shared(float32(drop.p_hidden_val))
video_cnn = conv3d_chalearn(self.x, use, lr, batch, net, reg, drop, mom, \
tr, res_dir, load_path)
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=self.x_skeleton, label=self.y )
# we load the pretrained DBN skeleton parameteres here
if use.load == True: dbn.load(os.path.join(load_path,'dbn_2015-06-19-11-34-24.npy'))
#####################################################################
# fuse the ConvNet output with skeleton output -- need to change here
######################################################################
out = T.concatenate([video_cnn.out, dbn.sigmoid_layers[-1].output], axis=1)
#####################################################################
# wudi add the mean and standard deviation of the activation values to exam the neural net
# Reference: Understanding the difficulty of training deep feedforward neural networks, Xavier Glorot, Yoshua Bengio
#####################################################################
insp_mean_list = []
insp_std_list = []
insp_mean_list.extend(dbn.out_mean)
insp_mean_list.extend(video_cnn.insp_mean)
insp_std_list.extend(dbn.out_std)
insp_std_list.extend(video_cnn.insp_std)
######################################################################
#MLP layer
self.layers.append(HiddenLayer(out, n_in=net.hidden, n_out=net.hidden, rng=tr.rng,
W_scale=net.W_scale[-1], b_scale=net.b_scale[-1], activation=net.activation))
out = self.layers[-1].output
if tr.inspect:
insp_mean_list.extend([T.mean(out)])
insp_std_list.extend([T.std(out)])
self.insp_mean = T.stacklists(insp_mean_list)
self.insp_std = T.stacklists(insp_std_list)
if use.drop: out = DropoutLayer(out, rng=tr.rng, p=drop.p_hidden).output
######################################################################
# softmax layer
self.layers.append(LogRegr(out, rng=tr.rng, n_in=net.hidden,
W_scale=net.W_scale[-1], b_scale=net.b_scale[-1], n_out=net.n_class))
self.p_y_given_x = self.layers[-1].p_y_given_x
######################################################################
# cost function
self.cost = self.layers[-1].negative_log_likelihood(self.y)
# function computing the number of errors
self.errors = self.layers[-1].errors(self.y)
# parameter list
for layer in video_cnn.layers:
self.params.extend(layer.params)
# pre-trained dbn parameter last layer (W, b) doesn't need to incorporate into the params
# for calculating the gradient
self.params.extend(dbn.params[:-2])
# MLP hidden layer params
self.params.extend(self.layers[-2].params)
# softmax layer params
self.params.extend(self.layers[-1].params)
# number of inputs for MLP = (# maps last stage)*(# convnets)*(resulting video shape) + trajectory size
print 'MLP:', video_cnn.n_in_MLP, "->", net.hidden_penultimate, "+", net.hidden_traj, '->', \
net.hidden, '->', net.hidden, '->', net.n_class, ""
return
Mean1 = SK_normalization ['Mean1']
Std1 = SK_normalization['Std1']
# customized data loader for both video module and skeleton module
#loader = DataLoader_with_skeleton(src, tr.batch_size, Mean1, Std1) # Lio changed it to read from HDF5 files
loader = DataLoader_with_skeleton_normalisation(src, tr.batch_size, 0, 1, Mean1, Std1) # Lio changed it to read from HDF5 files
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(name = 'x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
########sample number:39,hidden_layer_size=[2000,1000]->error rate=77.1%;[2000,2000,1000],78.4%
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000,2000,1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here, currently pretraining is done
# unsupervisedly, we can load the supervised pretrainining parameters later
#when u pretain the network, comment the following line
dbn.load_params_DBN("/home/zhiquan/fancy/meterials/chalearn2014_fancy_data/result_temp/dbn/try/63.9% 2018.05.06.19.54.43/paramsbest.zip")
cost = dbn.finetune_cost
# function computing the number of errors
errors = dbn.errors
# wudi add the mean and standard deviation of the activation values to exam the neural net
# Reference: Understanding the difficulty of training deep feedforward neural networks, Xavier Glorot, Yoshua Bengio
# customized data loader for both video module and skeleton module
#loader = DataLoader_with_skeleton(src, tr.batch_size, Mean1, Std1) # Lio changed it to read from HDF5 files
loader = DataLoader_with_skeleton_normalisation(
src, tr.batch_size, 0, 1, Mean1,
Std1) # Lio changed it to read from HDF5 files
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(
name='x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here, currently pretraining is done
# unsupervisedly, we can load the supervised pretrainining parameters later
dbn.load_params_DBN(
"/idiap/user/dwu/chalearn/result/try/37.8% 2015.07.09.13.26.11/paramsbest.zip"
)
cost = dbn.finetune_cost
# function computing the number of errors
errors = dbn.errors
# wudi add the mean and standard deviation of the activation values to exam the neural net
# Reference: Understanding the difficulty of training deep feedforward neural networks, Xavier Glorot, Yoshua Bengio
out_mean = T.stack(dbn.out_mean)
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
p_y_given_x = net_convnet3d_grbm_early_fusion.prediction_function(x_, x_skeleton_)
#############################
# load normalisation constant given load_path
Mean_skel, Std_skel, Mean_CNN, Std_CNN = net_convnet3d_grbm_early_fusion.load_normalisation_constant(load_path)
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(name = 'x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here
load_path = '/idiap/user/dwu/chalearn/result/try/36.7% 2015.07.09.17.53.10'
dbn.load_params_DBN(os.path.join(load_path,'paramsbest.zip'))
test_model = function([], dbn.logLayer.p_y_given_x,
givens={x_skeleton: x_skeleton_},
on_unused_input='ignore')
for file_count, file in enumerate(samples):
condition = (file_count > -1)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
save_path= os.path.join(data, file)
print file
time_start = time()
Mean_CNN = CNN_normalization ['Mean_CNN']
Std_CNN = CNN_normalization['Std_CNN']
# customized data loader for both video module and skeleton module
loader = DataLoader_with_skeleton_normalisation(src, tr.batch_size, Mean_CNN, Std_CNN, Mean1, Std1) # Lio changed it to read from HDF5 files
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(name = 'x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here
dbn.load(os.path.join(load_path,'dbn_2015-06-19-11-34-24.npy'))
####################################################################
# 3DCNN for video module
####################################################################
# we load the CNN parameteres here
use.load = True
video_cnn = conv3d_chalearn(x, use, lr, batch, net, reg, drop, mom, tr, res_dir, load_path)
#####################################################################
# fuse the ConvNet output with skeleton output -- need to change here
######################################################################
Mean1 = SK_normalization ['Mean1']
Std1 = SK_normalization['Std1']
# customized data loader for both video module and skeleton module
#loader = DataLoader_with_skeleton(src, tr.batch_size, Mean1, Std1) # Lio changed it to read from HDF5 files
loader = DataLoader_with_skeleton_normalisation(src, tr.batch_size, 0, 1, Mean1, Std1) # Lio changed it to read from HDF5 files
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(name = 'x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here, currently pretraining is done
# unsupervisedly, we can load the supervised pretrainining parameters later
dbn.load_params_DBN("/idiap/user/dwu/chalearn/result/try/37.8% 2015.07.09.13.26.11/paramsbest.zip")
cost = dbn.finetune_cost
# function computing the number of errors
errors = dbn.errors
# wudi add the mean and standard deviation of the activation values to exam the neural net
# Reference: Understanding the difficulty of training deep feedforward neural networks, Xavier Glorot, Yoshua Bengio
out_mean = T.stack(dbn.out_mean)
# customized data loader for both video module and skeleton module
loader = DataLoader_with_skeleton_normalisation(
src, tr.batch_size, Mean_CNN, Std_CNN, Mean1,
Std1) # Lio changed it to read from HDF5 files
####################################################################
# DBN for skeleton modules
####################################################################
# ------------------------------------------------------------------------------
# symbolic variables
x_skeleton = ndtensor(len(tr._skeleon_in_shape))(
name='x_skeleton') # video input
x_skeleton_ = _shared(empty(tr._skeleon_in_shape))
dbn = GRBM_DBN(numpy_rng=random.RandomState(123), n_ins=891, \
hidden_layers_sizes=[2000, 2000, 1000], n_outs=101, input_x=x_skeleton, label=y )
# we load the pretrained DBN skeleton parameteres here
dbn.load(os.path.join(load_path, 'dbn_2015-06-19-11-34-24.npy'))
####################################################################
# 3DCNN for video module
####################################################################
# we load the CNN parameteres here
use.load = True
video_cnn = conv3d_chalearn(x, use, lr, batch, net, reg, drop, mom, tr,
res_dir, load_path)
#####################################################################
# fuse the ConvNet output with skeleton output -- need to change here
######################################################################
