def __init__(self,rng, params,cost_function='mse',optimizer = RMSprop):
batch_size=params['batch_size']
sequence_length=params["seq_length"]
lr=params['lr']
self.n_in = 1024
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
layer2=LSTMLayer(rng,1,self.n_lstm,self.n_lstm)
layer3=LSTMLayer(rng,2,self.n_lstm,self.n_lstm)
self.params = layer1.params+layer2.params+layer3.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,mask,h_tm1_1,c_tm1_1,h_tm1_2,c_tm1_2,h_tm1_3,c_tm1_3):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
dl1=DropoutLayer(rng,input=y_t_1,prob=0.5,is_train=is_train,mask=mask)
[h_t_2,c_t_2,y_t_2]=layer2.run(dl1.output,h_tm1_2,c_tm1_2)
[h_t_3,c_t_3,y_t_3]=layer3.run(y_t_2,h_tm1_3,c_tm1_3)
y = T.dot(y_t_3, self.W_hy) + self.b_y
return [h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
h0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
h0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
h0_3 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_3 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
mask_shape=(sequence_length,batch_size,self.n_lstm)
p_1=0.5
mask= rng.binomial(size=mask_shape, p=p_1, dtype=X.dtype)
[h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[X.dimshuffle(1,0,2),mask],
outputs_info=[h0_1, c0_1,h0_2, c0_2, h0_3, c0_3, None])
self.output = y_vals.dimshuffle(1,0,2)
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
self.train = theano.function(inputs=[X, Y,is_train],outputs=cost,updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train], outputs = y_vals.dimshuffle(1,0,2),allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng, params,cost_function='mse',optimizer = RMSprop):
batch_size=params['batch_size']
sequence_length=params["seq_length"]
lr=params['lr']
self.n_in = params['n_output']
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
layer2=LSTMLayer(rng,1,self.n_lstm,self.n_lstm)
layer3=LSTMLayer(rng,2,self.n_lstm,self.n_lstm)
self.params = layer1.params+layer2.params+layer3.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,mask,h_tm1_1,c_tm1_1,h_tm1_2,c_tm1_2,h_tm1_3,c_tm1_3):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
dl1=DropoutLayer(rng,input=y_t_1,prob=0.5,is_train=is_train,mask=mask)
[h_t_2,c_t_2,y_t_2]=layer2.run(dl1.output,h_tm1_2,c_tm1_2)
[h_t_3,c_t_3,y_t_3]=layer3.run(y_t_2,h_tm1_3,c_tm1_3)
y = T.dot(y_t_3, self.W_hy) + self.b_y
return [h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
h0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
h0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
h0_3 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_3 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
mask_shape=(sequence_length,batch_size,self.n_lstm)
p_1=0.5
mask= rng.binomial(size=mask_shape, p=p_1, dtype=X.dtype)
[h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[X.dimshuffle(1,0,2),mask],
outputs_info=[h0_1, c0_1,h0_2, c0_2, h0_3, c0_3, None])
self.output = y_vals.dimshuffle(1,0,2)
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
self.train = theano.function(inputs=[X, Y,is_train],outputs=cost,updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train], outputs = y_vals.dimshuffle(1,0,2),allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
batch_size=params["batch_size"]
n_output=params['n_output']
corruption_level=params["corruption_level"]
X = T.matrix(name="input",dtype=dtype) # batch of sequence of vector
Y = T.matrix(name="output",dtype=dtype) # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
bin_noise=rng.binomial(size=(batch_size,n_output/3,1), n=1,p=1 - corruption_level,dtype=theano.config.floatX)
#bin_noise_3d= T.reshape(T.concatenate((bin_noise, bin_noise,bin_noise),axis=1),(batch_size,n_output/3,3))
bin_noise_3d= T.concatenate((bin_noise, bin_noise,bin_noise),axis=2)
noise= rng.normal(size=(batch_size,n_output), std=0.03, avg=0.0,dtype=theano.config.floatX)
noise_bin=T.reshape(noise,(batch_size,n_output/3,3))*bin_noise_3d
X_train=T.reshape(noise_bin,(batch_size,n_output))+X
X_tilde= T.switch(T.neq(is_train, 0), X_train, X)
W_1_e =u.init_weight(shape=(n_output,1024),rng=rng,name="w_hid",sample="glorot")
b_1_e=u.init_bias(1024,rng)
W_2_e =u.init_weight(shape=(1024,2048),rng=rng,name="w_hid",sample="glorot")
b_2_e=u.init_bias(2048,rng)
W_2_d = W_2_e.T
b_2_d=u.init_bias(1024,rng)
W_1_d = W_1_e.T
b_1_d=u.init_bias(n_output,rng)
h_1_e=HiddenLayer(rng,X_tilde,0,0, W=W_1_e,b=b_1_e,activation=nn.relu)
h_2_e=HiddenLayer(rng,h_1_e.output,0,0, W=W_2_e,b=b_2_e,activation=nn.relu)
h_2_d=HiddenLayer(rng,h_2_e.output,0,0, W=W_2_d,b=b_2_d,activation=u.do_nothing)
h_1_d=LogisticRegression(rng,h_2_d.output,0,0, W=W_1_d,b=b_1_d)
self.output = h_1_d.y_pred
self.params =h_1_e.params+h_2_e.params
self.params.append(b_2_d)
self.params.append(b_1_d)
cost=get_err_fn(self,cost_function,Y)
L2_reg=0.0001
L2_sqr = theano.shared(0.)
for param in self.params:
L2_sqr += (T.sum(param[0] ** 2)+T.sum(param[1] ** 2))
cost += L2_reg*L2_sqr
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X,Y,is_train],outputs=cost,updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train], outputs = self.output,allow_input_downcast=True)
self.mid_layer = theano.function(inputs = [X,is_train], outputs = h_2_e.output,allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng, params,cost_function='mse',optimizer = RMSprop):
batch_size=params['batch_size']
sequence_length=params["seq_length"]
lr=params['lr']
self.n_in = 48
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
self.params = layer1.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,h_tm1_1,c_tm1_1):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
y = T.dot(y_t_1, self.W_hy) + self.b_y
return [h_t_1,c_t_1,y]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
H = T.matrix(name="H",dtype=dtype) # initial hidden state
C = T.matrix(name="C",dtype=dtype) # initial hidden state
noise= rng.normal(size=(batch_size,sequence_length,self.n_in), std=0.0002, avg=0.0,dtype=theano.config.floatX)
X_train=noise+X
X_tilde= T.switch(T.neq(is_train, 0), X_train, X)
# h0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
# c0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
[h_t_1,c_t_1,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[X_tilde.dimshuffle(1,0,2)],
outputs_info=[H, C, None])
self.output = y_vals.dimshuffle(1,0,2)
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t_1[-1],c_t_1[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t_1[-1],c_t_1[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self, rng, params, cost_function="mse", optimizer=RMSprop):
lr = params["lr"]
n_lstm = params["n_hidden"]
n_out = params["n_output"]
batch_size = params["batch_size"]
sequence_length = params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar("is_train") # pseudo boolean for switching between training and prediction
# CNN global parameters.
subsample = (1, 1)
p_1 = 0.5
border_mode = "same"
cnn_batch_size = batch_size * sequence_length
pool_size = (2, 2)
n_lstm_layer = 2
f_dict = dict()
f_dict["filter_shape_" + str(0)] = (64, 1, 9, 9)
f_dict["s_filter_shape_" + str(0)] = (64, f_dict["filter_shape_" + str(0)][0], 9, 9)
f_dict["filter_shape_" + str(1)] = (128, f_dict["filter_shape_" + str(0)][0], 3, 3)
f_dict["s_filter_shape_" + str(1)] = (128, f_dict["filter_shape_" + str(1)][0], 3, 3)
f_dict["filter_shape_" + str(2)] = (128, f_dict["filter_shape_" + str(1)][0], 3, 3)
f_dict["s_filter_shape_" + str(2)] = (128, f_dict["filter_shape_" + str(2)][0], 3, 3)
input_shape = (batch_size, sequence_length, 1, 120, 60) # input_shape= (samples, channels, rows, cols)
input = X.reshape(input_shape).dimshuffle(1, 0, 2, 3, 4)
input_shape = (batch_size, 1, 120, 60) # input_shape= (samples, channels, rows, cols)
s_dict = dict()
layer_list = ["p", "l", "p", "d", "l", "p", "l", "p"]
rows = input_shape[2]
cols = input_shape[3]
counter = 0
outputs_info = []
for layer in layer_list:
if layer == "l":
s_index = str(counter)
if counter == 0:
pre_nfilter = input_shape[1]
else:
pre_nfilter = f_dict["filter_shape_" + str(counter - 1)][0]
i_shape = (batch_size, pre_nfilter, rows, cols) # input_shape= (samples, channels, rows, cols)
s_shape = (
batch_size,
f_dict["s_filter_shape_" + s_index][0],
rows,
cols,
) # input_shape= (samples, channels, rows, cols)
h = shared(np.zeros(shape=s_shape, dtype=dtype)) # initial hidden state
c = shared(np.zeros(shape=s_shape, dtype=dtype)) # initial hidden state
s_dict["i_shape_" + s_index] = i_shape
s_dict["s_shape_" + s_index] = s_shape
outputs_info.append(h)
outputs_info.append(c)
counter += 1
if layer == "p":
rows = rows / 2
cols = cols / 2
s_dict["final_shape"] = (batch_size, sequence_length, pre_nfilter, rows, cols)
outputs_info.append(None)
outputs_info = tuple(outputs_info)
p_dict = dict()
for index in range(counter):
s_index = str(index)
p_dict["W_xi_" + s_index] = u.init_weight(
f_dict["filter_shape_" + s_index], rng=rng, name="W_xi_" + s_index, sample="glorot"
)
p_dict["W_hi_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_hi_" + s_index, sample="glorot"
)
p_dict["W_ci_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_ci_" + s_index, sample="glorot"
)
p_dict["b_i_" + s_index] = u.init_bias(f_dict["filter_shape_" + s_index][0], rng=rng, name="b_i_" + s_index)
p_dict["W_xf_" + s_index] = u.init_weight(
f_dict["filter_shape_" + s_index], rng=rng, name="W_xf_" + s_index, sample="glorot"
)
p_dict["W_hf_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_hf_" + s_index, sample="glorot"
)
p_dict["W_cf_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_cf_" + s_index, sample="glorot"
)
p_dict["b_f_" + s_index] = u.init_bias(
f_dict["filter_shape_" + s_index][0], rng=rng, name="b_f_" + s_index, sample="one"
)
p_dict["W_xc_" + s_index] = u.init_weight(
f_dict["filter_shape_" + s_index], rng=rng, name="W_xc_" + s_index, sample="glorot"
)
p_dict["W_hc_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_hc_" + s_index, sample="glorot"
)
p_dict["b_c_" + s_index] = u.init_bias(f_dict["filter_shape_" + s_index][0], rng=rng, name="b_c_" + s_index)
p_dict["W_xo_" + s_index] = u.init_weight(
f_dict["filter_shape_" + s_index], rng=rng, name="W_xo_" + s_index, sample="glorot"
)
p_dict["W_ho_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_ho_" + s_index, sample="glorot"
)
p_dict["W_co_" + s_index] = u.init_weight(
f_dict["s_filter_shape_" + s_index], rng=rng, name="W_co_" + s_index, sample="glorot"
)
p_dict["b_o_" + s_index] = u.init_bias(f_dict["filter_shape_" + s_index][0], rng=rng, name="b_o_" + s_index)
def step_lstm(x_t, mask, h_tm1_1, c_tm1_1, h_tm1_2, c_tm1_2, h_tm1_3, c_tm1_3):
p1 = PoolLayer(x_t, pool_size=pool_size, input_shape=s_dict["i_shape_0"])
layer_1 = CLSTMLayer(rng, 0, p_dict, f_dict, s_dict, p1.output, h_tm1_1, c_tm1_1, border_mode, subsample)
[h_t_1, c_t_1, y_t_1] = layer_1.output
p2 = PoolLayer(y_t_1, pool_size=pool_size, input_shape=layer_1.yt_shape)
dl1 = DropoutLayer(rng, input=p2.output, prob=p_1, is_train=is_train, mask=mask)
layer_2 = CLSTMLayer(rng, 1, p_dict, f_dict, s_dict, dl1.output, h_tm1_2, c_tm1_2, border_mode, subsample)
[h_t_2, c_t_2, y_t_2] = layer_2.output
p2 = PoolLayer(y_t_2, pool_size=pool_size, input_shape=layer_1.yt_shape)
layer_3 = CLSTMLayer(rng, 2, p_dict, f_dict, s_dict, p2.output, h_tm1_3, c_tm1_3, border_mode, subsample)
[h_t_3, c_t_3, y_t_3] = layer_3.output
p3 = PoolLayer(y_t_3, pool_size=pool_size, input_shape=layer_3.yt_shape)
return [h_t_1, c_t_1, h_t_2, c_t_2, h_t_3, c_t_3, p3.output]
# (1, 0, 2) -> AxBxC to BxAxC
# (batch_size,sequence_length, n_in) >> (sequence_length, batch_size ,n_in)
# T.dot(x_t, self.W_xi_1)x_t=(sequence_length, batch_size ,n_in), W_xi_1= [self.n_in, self.n_lstm]
# 5.293.568
# 185.983.658
# 19.100.202
# 8.090.154
s_shape = list(s_dict["i_shape_1"]) # after pooling filter sha[e
s_shape.insert(0, sequence_length)
mask_shape_1 = tuple(s_shape)
mask = rng.binomial(size=mask_shape_1, p=p_1, dtype=input.dtype)
[h_t_1, c_t_1, h_t_2, c_t_2, h_t_3, c_t_3, y_vals], _ = theano.scan(
fn=step_lstm, outputs_info=outputs_info, sequences=[input, mask]
)
s_dict["final_shape"] = (batch_size, sequence_length, pre_nfilter, rows, cols)
hidden_input = y_vals.dimshuffle(1, 0, 2, 3, 4)
n_in = reduce(lambda x, y: x * y, s_dict["final_shape"][2:])
x_flat = hidden_input.flatten(3)
h1 = HiddenLayer(rng, x_flat, n_in, 1024, activation=nn.relu)
n_in = 1024
lreg = LogisticRegression(rng, h1.output, n_in, 42)
self.output = lreg.y_pred
self.params = p_dict.values()
self.params.append(h1.params[0])
self.params.append(h1.params[1])
self.params.append(lreg.params[0])
self.params.append(lreg.params[1])
#
# tmp = theano.tensor.switch(theano.tensor.isnan(Y),0,Y)
cost = get_err_fn(self, cost_function, Y)
L2_reg = 0.0001
L2_sqr = theano.shared(0.0)
for param in self.params:
L2_sqr += T.sum(param ** 2)
cost += L2_reg * L2_sqr
_optimizer = optimizer(cost, self.params, lr=lr)
# self.train = theano.function(inputs=[X,Y,is_train],outputs=cost,allow_input_downcast=True)
#
# _optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(
inputs=[X, Y, is_train], outputs=cost, updates=_optimizer.getUpdates(), allow_input_downcast=True
)
self.predictions = theano.function(inputs=[X, is_train], outputs=self.output, allow_input_downcast=True)
self.n_param = count_params(self.params)
def __init__(self, rng, params, cost_function='mse', optimizer=RMSprop):
batch_size = params['batch_size']
sequence_length = params["seq_length"]
lr = params['lr']
self.n_in = 2048
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
self.W_hy = init_weight((self.n_lstm, self.n_out),
rng=rng,
name='W_hy',
sample='glorot')
self.b_y = init_bias(self.n_out, rng=rng, sample='zero')
layer1 = LSTMLayer(rng, 0, self.n_in, self.n_lstm)
self.params = layer1.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t, h_tm1_1, c_tm1_1):
[h_t_1, c_t_1, y_t_1] = layer1.run(x_t, h_tm1_1, c_tm1_1)
y = T.dot(y_t_1, self.W_hy) + self.b_y
return [h_t_1, c_t_1, y]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar(
'is_train'
) # pseudo boolean for switching between training and prediction
H = T.matrix(name="H", dtype=dtype) # initial hidden state
C = T.matrix(name="C", dtype=dtype) # initial hidden state
noise = rng.normal(size=(batch_size, sequence_length, self.n_in),
std=0.0002,
avg=0.0,
dtype=theano.config.floatX)
X_train = noise + X
X_tilde = T.switch(T.neq(is_train, 0), X_train, X)
# h0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
# c0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
[h_t_1, c_t_1,
y_vals], _ = theano.scan(fn=step_lstm,
sequences=[X_tilde.dimshuffle(1, 0, 2)],
outputs_info=[H, C, None])
self.output = y_vals.dimshuffle(1, 0, 2)
cost = get_err_fn(self, cost_function, Y)
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X, Y, is_train, H, C],
outputs=[cost, h_t_1[-1], c_t_1[-1]],
updates=_optimizer.getUpdates(),
allow_input_downcast=True)
self.predictions = theano.function(
inputs=[X, is_train, H, C],
outputs=[self.output, h_t_1[-1], c_t_1[-1]],
allow_input_downcast=True)
self.n_param = count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
n_lstm=params['n_hidden']
n_out=params['n_output']
batch_size=params["batch_size"]
sequence_length=params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample=(1,1)
p_1=0.5
border_mode="valid"
cnn_batch_size=batch_size*sequence_length
pool_size=(2,2)
#Layer1: conv2+pool+drop
filter_shape=(64,1,9,9)
input_shape=(cnn_batch_size,1,120,60) #input_shape= (samples, channels, rows, cols)
input= X.reshape(input_shape)
c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu)
p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape)
dl1=DropoutLayer(rng,input=p1.output,prob=p_1,is_train=is_train)
retain_prob = 1. - p_1
test_output = p1.output*retain_prob
d1_output = T.switch(T.neq(is_train, 0), dl1.output, test_output)
#Layer2: conv2+pool
filter_shape=(128,p1.output_shape[1],3,3)
c2=ConvLayer(rng, d1_output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu)
p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape=(128,p2.output_shape[1],3,3)
c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu)
p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape)
#Layer4: hidden
n_in= reduce(lambda x, y: x*y, p3.output_shape[1:])
x_flat = p3.output.flatten(2)
h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu)
n_in=1024
rnn_input = h1.output.reshape((batch_size,sequence_length, n_in))
#Layer5: gru
self.n_in = n_in
self.n_lstm = n_lstm
self.n_out = n_out
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
layer2=LSTMLayer(rng,1,self.n_lstm,self.n_lstm)
layer3=LSTMLayer(rng,2,self.n_lstm,self.n_lstm)
self.params = layer1.params+layer2.params+layer3.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,mask,h_tm1_1,c_tm1_1,h_tm1_2,c_tm1_2,h_tm1_3,c_tm1_3):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
dl1=DropoutLayer(rng,input=y_t_1,prob=0.5,is_train=is_train,mask=mask)
[h_t_2,c_t_2,y_t_2]=layer2.run(dl1.output,h_tm1_2,c_tm1_2)
[h_t_3,c_t_3,y_t_3]=layer3.run(y_t_2,h_tm1_3,c_tm1_3)
y = T.dot(y_t_3, self.W_hy) + self.b_y
return [h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y]
h0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
h0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
h0_3 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_3 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial cell state
mask_shape=(sequence_length,batch_size,self.n_lstm)
p_1=0.5
mask= rng.binomial(size=mask_shape, p=p_1, dtype=X.dtype)
#(1, 0, 2) -> AxBxC to BxAxC
#(batch_size,sequence_length, n_in) >> (sequence_length, batch_size ,n_in)
#T.dot(x_t, self.W_xi)x_t=(sequence_length, batch_size ,n_in), W_xi= [self.n_in, self.n_lstm]
[h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[rnn_input.dimshuffle(1,0,2),mask],
outputs_info=[h0_1, c0_1,h0_2, c0_2, h0_3, c0_3, None])
self.output = y_vals.dimshuffle(1,0,2)
self.params =c1.params+c2.params+c3.params+h1.params+self.params
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X,Y,is_train],outputs=cost,updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train], outputs = self.output,allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
n_lstm=params['n_hidden']
n_out=params['n_output']
batch_size=params["batch_size"]
sequence_length=params["seq_length"]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample=(1,1)
p_1=0.5
border_mode="valid"
cnn_batch_size=batch_size*sequence_length
pool_size=(2,2)
#Layer1: conv2+pool+drop
filter_shape=(64,1,9,9)
input_shape=(cnn_batch_size,1,120,60) #input_shape= (samples, channels, rows, cols)
input= X.reshape(input_shape)
c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu)
p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape)
dl1=DropoutLayer(rng,input=p1.output,prob=p_1,is_train=is_train)
#Layer2: conv2+pool
filter_shape=(128,p1.output_shape[1],3,3)
c2=ConvLayer(rng, dl1.output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu)
p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape=(128,p2.output_shape[1],3,3)
c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu)
p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape)
#Layer4: hidden
n_in= reduce(lambda x, y: x*y, p3.output_shape[1:])
x_flat = p3.output.flatten(2)
h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu)
n_in=1024
rnn_input = h1.output.reshape((batch_size,sequence_length, n_in))
#Layer5: LSTM
self.n_in = n_in
self.n_lstm = n_lstm
self.n_out = n_out
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
self.params = layer1.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,h_tm1,c_tm1):
[h_t,c_t,y_t]=layer1.run(x_t,h_tm1,c_tm1)
y = T.dot(y_t, self.W_hy) + self.b_y
return [h_t,c_t,y]
H = T.matrix(name="H",dtype=dtype) # initial hidden state
C = T.matrix(name="C",dtype=dtype) # initial hidden state
[h_t,c_t,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[rnn_input.dimshuffle(1,0,2)],
outputs_info=[H, C, None])
self.output = y_vals.dimshuffle(1,0,2)
self.params =c1.params+c2.params+c3.params+h1.params+self.params
cost=get_err_fn(self,cost_function,Y)
L2_reg=0.0001
L2_sqr = theano.shared(0.)
for param in self.params:
L2_sqr += (T.sum(param ** 2))
cost += L2_reg*L2_sqr
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t[-1],c_t[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t[-1],c_t[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self, rng, params, cost_function='mse', optimizer=RMSprop):
lr = params["lr"]
n_lstm = params['n_hidden']
n_out = params['n_output']
batch_size = params["batch_size"]
sequence_length = params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar(
'is_train'
) # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample = (1, 1)
p_1 = 0.5
border_mode = "valid"
cnn_batch_size = batch_size * sequence_length
pool_size = (2, 2)
#Layer1: conv2+pool+drop
filter_shape = (64, 1, 9, 9)
input_shape = (cnn_batch_size, 1, 120, 60
) #input_shape= (samples, channels, rows, cols)
input = X.reshape(input_shape)
c1 = ConvLayer(rng,
input,
filter_shape,
input_shape,
border_mode,
subsample,
activation=nn.relu)
p1 = PoolLayer(c1.output,
pool_size=pool_size,
input_shape=c1.output_shape)
dl1 = DropoutLayer(rng, input=p1.output, prob=p_1, is_train=is_train)
retain_prob = 1. - p_1
test_output = p1.output * retain_prob
d1_output = T.switch(T.neq(is_train, 0), dl1.output, test_output)
#Layer2: conv2+pool
filter_shape = (128, p1.output_shape[1], 3, 3)
c2 = ConvLayer(rng,
d1_output,
filter_shape,
p1.output_shape,
border_mode,
subsample,
activation=nn.relu)
p2 = PoolLayer(c2.output,
pool_size=pool_size,
input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape = (128, p2.output_shape[1], 3, 3)
c3 = ConvLayer(rng,
p2.output,
filter_shape,
p2.output_shape,
border_mode,
subsample,
activation=nn.relu)
p3 = PoolLayer(c3.output,
pool_size=pool_size,
input_shape=c3.output_shape)
#Layer4: hidden
n_in = reduce(lambda x, y: x * y, p3.output_shape[1:])
x_flat = p3.output.flatten(2)
h1 = HiddenLayer(rng, x_flat, n_in, 1024, activation=nn.relu)
n_in = 1024
rnn_input = h1.output.reshape((batch_size, sequence_length, n_in))
#Layer5: gru
self.n_in = n_in
self.n_lstm = n_lstm
self.n_out = n_out
self.W_hy = init_weight((self.n_lstm, self.n_out),
rng=rng,
name='W_hy',
sample='glorot')
self.b_y = init_bias(self.n_out, rng=rng, sample='zero')
layer1 = LSTMLayer(rng, 0, self.n_in, self.n_lstm)
layer2 = LSTMLayer(rng, 1, self.n_lstm, self.n_lstm)
layer3 = LSTMLayer(rng, 2, self.n_lstm, self.n_lstm)
self.params = layer1.params + layer2.params + layer3.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t, mask, h_tm1_1, c_tm1_1, h_tm1_2, c_tm1_2, h_tm1_3,
c_tm1_3):
[h_t_1, c_t_1, y_t_1] = layer1.run(x_t, h_tm1_1, c_tm1_1)
dl1 = DropoutLayer(rng,
input=y_t_1,
prob=0.5,
is_train=is_train,
mask=mask)
[h_t_2, c_t_2, y_t_2] = layer2.run(dl1.output, h_tm1_2, c_tm1_2)
[h_t_3, c_t_3, y_t_3] = layer3.run(y_t_2, h_tm1_3, c_tm1_3)
y = T.dot(y_t_3, self.W_hy) + self.b_y
return [h_t_1, c_t_1, h_t_2, c_t_2, h_t_3, c_t_3, y]
h0_1 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial hidden state
c0_1 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial cell state
h0_2 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial hidden state
c0_2 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial cell state
h0_3 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial hidden state
c0_3 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial cell state
mask_shape = (sequence_length, batch_size, self.n_lstm)
p_1 = 0.5
mask = rng.binomial(size=mask_shape, p=p_1, dtype=X.dtype)
#(1, 0, 2) -> AxBxC to BxAxC
#(batch_size,sequence_length, n_in) >> (sequence_length, batch_size ,n_in)
#T.dot(x_t, self.W_xi)x_t=(sequence_length, batch_size ,n_in), W_xi= [self.n_in, self.n_lstm]
[h_t_1, c_t_1, h_t_2, c_t_2, h_t_3, c_t_3, y_vals], _ = theano.scan(
fn=step_lstm,
sequences=[rnn_input.dimshuffle(1, 0, 2), mask],
outputs_info=[h0_1, c0_1, h0_2, c0_2, h0_3, c0_3, None])
self.output = y_vals.dimshuffle(1, 0, 2)
self.params = c1.params + c2.params + c3.params + h1.params + self.params
cost = get_err_fn(self, cost_function, Y)
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X, Y, is_train],
outputs=cost,
updates=_optimizer.getUpdates(),
allow_input_downcast=True)
self.predictions = theano.function(inputs=[X, is_train],
outputs=self.output,
allow_input_downcast=True)
self.n_param = count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
n_lstm=params['n_hidden']
n_out=params['n_output']
batch_size=params["batch_size"]
sequence_length=params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample=(1,1)
p_1=0.5
border_mode="valid"
cnn_batch_size=batch_size*sequence_length
pool_size=(2,2)
#Layer1: conv2+pool+drop
filter_shape=(64,1,9,9)
input_shape=(cnn_batch_size,1,120,60) #input_shape= (samples, channels, rows, cols)
input= X.reshape(input_shape)
c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu)
p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape)
dl1=DropoutLayer(rng,input=p1.output,prob=p_1,is_train=is_train)
#Layer2: conv2+pool
filter_shape=(128,p1.output_shape[1],3,3)
c2=ConvLayer(rng, dl1.output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu)
p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape=(128,p2.output_shape[1],3,3)
c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu)
p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape)
#Layer4: hidden
n_in= reduce(lambda x, y: x*y, p3.output_shape[1:])
x_flat = p3.output.flatten(2)
h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu)
n_in=1024
rnn_input = h1.output.reshape((batch_size,sequence_length, n_in))
#Layer5: gru
self.n_in = n_in
self.n_lstm = n_lstm
self.n_out = n_out
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
self.params = layer1.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,h_tm1,c_tm1):
[h_t,c_t,y_t]=layer1.run(x_t,h_tm1,c_tm1)
y = T.dot(y_t, self.W_hy) + self.b_y
return [h_t,c_t,y]
H = T.matrix(name="H",dtype=dtype) # initial hidden state
C = T.matrix(name="C",dtype=dtype) # initial hidden state
#(1, 0, 2) -> AxBxC to BxAxC
#(batch_size,sequence_length, n_in) >> (sequence_length, batch_size ,n_in)
#T.dot(x_t, self.W_xi)x_t=(sequence_length, batch_size ,n_in), W_xi= [self.n_in, self.n_lstm]
[h_t,c_t,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[rnn_input.dimshuffle(1,0,2)],
outputs_info=[H, C, None])
self.output = y_vals.dimshuffle(1,0,2)
self.params =c1.params+c2.params+c3.params+h1.params+self.params
cost=get_err_fn(self,cost_function,Y)
L2_reg=0.0001
L2_sqr = theano.shared(0.)
for param in self.params:
L2_sqr += (T.sum(param ** 2))
cost += L2_reg*L2_sqr
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t[-1],c_t[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t[-1],c_t[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng, params,cost_function='mse',optimizer = RMSprop):
batch_size=params['batch_size']
sequence_length=params["seq_length"]
lr=params['lr']
self.n_in = 48
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
n_fc=512
self.W_hy = init_weight((self.n_lstm, n_fc), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(n_fc,rng=rng, sample='zero')
self.numOfLayers=1
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
self.params = layer1.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,h_tm1_1,c_tm1_1):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
y = T.dot(y_t_1, self.W_hy) + self.b_y
return [h_t_1,c_t_1,y]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
H = T.matrix(name="H",dtype=dtype) # initial hidden state
C = T.matrix(name="C",dtype=dtype) # initial hidden state
noise= rng.normal(size=(batch_size,sequence_length,self.n_in), std=0.008, avg=0.0,dtype=theano.config.floatX)
X_train=noise+X
X_tilde= T.switch(T.neq(is_train, 0), X_train, X)
[h_t_1,c_t_1,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[X_tilde.dimshuffle(1,0,2)],
outputs_info=[H, C, None])
# self.output = y_vals.dimshuffle(1,0,2)
mdn_input=y_vals.dimshuffle(1,0,2)
mdn_input=T.reshape(mdn_input,(batch_size*sequence_length,n_fc))
Y_ll=T.reshape(Y,(batch_size*sequence_length,params['n_output']))
mdn = MDNoutputLayer(rng=rng,
input=mdn_input,
n_in=n_fc,
n_out=params['n_output'],
mu_activation=do_nothing,
n_components=240)
self.params=self.params+mdn.params
# self.params.append(mdn.W_mixing)
# self.params.append(mdn.W_mu)
# self.params.append(mdn.W_sigma)
cost = nll(mu = mdn.mu,
sigma = mdn.sigma,
mixing = mdn.mixing,
y = Y_ll) #+ L2_reg * self.frame_pred.L2_sqr
L2_reg=0.0001
L2_sqr = theano.shared(0.)
for param in self.params:
L2_sqr += (T.sum(param[0] ** 2)+T.sum(param[1] ** 2))
cost += L2_reg*L2_sqr
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
#Sampling from the GMM
component = rng.multinomial(pvals=mdn.mixing)
component_mean = T.sum(mdn.mu * component.dimshuffle(0,'x',1),axis=2)
component_std = T.sum(mdn.sigma * component, axis=1, keepdims=True)
samples=rng.normal(size=(batch_size*sequence_length,params['n_output']),avg = component_mean, std=component_std)
self.output = T.reshape(samples,(batch_size,sequence_length,params['n_output']))
self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t_1[-1],c_t_1[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t_1[-1],c_t_1[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
# n_lstm=params['n_hidden']
# n_out=params['n_output']
batch_size=params["batch_size"]
sequence_length=params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample=(1,1)
p_1=0.5
border_mode="same"
cnn_batch_size=batch_size*sequence_length
pool_size=(2,2)
#Layer1: conv2+pool+drop
filter_shape=(64,3,9,9)
input_shape=(cnn_batch_size,3,112,112) #input_shape= (samples, channels, rows, cols)
input=X.reshape(input_shape)
# input= X_r.dimshuffle(0,3,1,2)
c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu)
p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape)
dl1=DropoutLayer(rng,input=p1.output,prob=p_1,is_train=is_train)
#Layer2: conv2+pool
filter_shape=(128,p1.output_shape[1],3,3)
c2=ConvLayer(rng, dl1.output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu)
p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape=(128,p2.output_shape[1],3,3)
c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu)
p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape)
#Layer4: conv2+pool
filter_shape=(64,p3.output_shape[1],3,3)
c4=ConvLayer(rng, p3.output,filter_shape,p3.output_shape,border_mode,subsample, activation=nn.relu)
p4=PoolLayer(c4.output,pool_size=pool_size,input_shape=c4.output_shape)
#Layer5: hidden
n_in= reduce(lambda x, y: x*y, p4.output_shape[1:])
x_flat = p4.output.flatten(2)
h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu)
#Layer6: Regressin layer
lreg=LogisticRegression(rng,h1.output,1024,2048)
#LSTM paramaters
self.n_in = 2048
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
self.params =c1.params+c2.params+c3.params+c4.params+h1.params+lreg.params+layer1.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,h_tm1_1,c_tm1_1):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
y = T.dot(y_t_1, self.W_hy) + self.b_y
return [h_t_1,c_t_1,y]
H = T.matrix(name="H",dtype=dtype) # initial hidden state
C = T.matrix(name="C",dtype=dtype) # initial hidden state
rnn_input = lreg.y_pred.reshape((batch_size,sequence_length, self.n_in))
[h_t_1,c_t_1,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[rnn_input.dimshuffle(1,0,2)],
outputs_info=[H, C, None])
self.output = y_vals.dimshuffle(1,0,2)
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t_1[-1],c_t_1[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t_1[-1],c_t_1[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng, params,cost_function='mse',optimizer = RMSprop):
batch_size=params['batch_size']
lr=params['lr']
self.n_in = 48
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
# self.n_lstm = n_lstm
# self.n_out = n_out
self.n_fc1=256
self.n_fc2=256
self.n_fc3=256
self.W_fc1 = init_weight((self.n_fc1, self.n_fc2),'W_fc1', 'glorot')
self.b_fc1 = init_bias(self.n_fc2, sample='zero')
self.W_fc2 = init_weight((self.n_fc2, self.n_fc3),'W_fc2', 'glorot')
self.b_fc2 =init_bias(self.n_fc3, sample='zero')
self.W_fc3 = init_weight((self.n_fc3, self.n_out),'w_fc3', 'glorot')
self.b_fc3 =init_bias(self.n_out, sample='zero')
#1th layer
self.W_xi_1 = init_weight((self.n_in, self.n_lstm), 'W_xi_1', 'glorot')
self.W_hi_1 = init_weight((self.n_lstm, self.n_lstm), 'W_hi_1', 'glorot')
self.W_ci_1 = init_weight((self.n_lstm, self.n_lstm), 'W_ci_1', 'glorot')
self.b_i_1 = init_bias(self.n_lstm, sample='zero')
self.W_xf_1 = init_weight((self.n_in, self.n_lstm), 'W_xf_1', 'glorot')
self.W_hf_1 = init_weight((self.n_lstm, self.n_lstm), 'W_hf_1', 'glorot')
self.W_cf_1 = init_weight((self.n_lstm, self.n_lstm), 'W_cf_1', 'glorot')
self.b_f_1 = init_bias(self.n_lstm, sample='one')
self.W_xc_1 = init_weight((self.n_in, self.n_lstm), 'W_xc_1', 'glorot')
self.W_hc_1 = init_weight((self.n_lstm, self.n_lstm), 'W_hc_1', 'glorot')
self.b_c_1 = init_bias(self.n_lstm, sample='zero')
self.W_xo_1 = init_weight((self.n_in, self.n_lstm), 'W_xo_1', 'glorot')
self.W_ho_1 = init_weight((self.n_lstm, self.n_lstm), 'W_ho_1', 'glorot')
self.W_co_1 = init_weight((self.n_lstm, self.n_lstm), 'W_co_1', 'glorot')
self.b_o_1 = init_bias(self.n_lstm, sample='zero')
#self.W_hy_1 = init_weight((self.n_lstm, self.n_out), 'W_hy_1')
#self.b_y_1 = init_bias(self.n_lstm, sample='zero')
#2th layer
self.W_xi_2 = init_weight((self.n_lstm, self.n_lstm), 'W_xi_2', 'glorot')
self.W_hi_2 = init_weight((self.n_lstm, self.n_lstm), 'W_hi_2', 'glorot')
self.W_ci_2 = init_weight((self.n_lstm, self.n_lstm), 'W_ci_2', 'glorot')
self.b_i_2 = init_bias(self.n_lstm, sample='zero')
self.W_xf_2 = init_weight((self.n_lstm, self.n_lstm), 'W_xf_2', 'glorot')
self.W_hf_2 = init_weight((self.n_lstm, self.n_lstm), 'W_hf_2', 'glorot')
self.W_cf_2 = init_weight((self.n_lstm, self.n_lstm), 'W_cf_2', 'glorot')
self.b_f_2 = init_bias(self.n_lstm, sample='one')
self.W_xc_2 = init_weight((self.n_lstm, self.n_lstm), 'W_xc_2', 'glorot')
self.W_hc_2 = init_weight((self.n_lstm, self.n_lstm), 'W_hc_2', 'glorot')
self.b_c_2 = init_bias(self.n_lstm, sample='zero')
self.W_xo_2 = init_weight((self.n_lstm, self.n_lstm), 'W_xo_2', 'glorot')
self.W_ho_2 = init_weight((self.n_lstm, self.n_lstm), 'W_ho_2', 'glorot')
self.W_co_2 = init_weight((self.n_lstm, self.n_lstm), 'W_co_2', 'glorot')
self.b_o_2 = init_bias(self.n_lstm, sample='zero')
self.W_hy_2 = init_weight((self.n_lstm, self.n_out), 'W_hy_2', 'glorot')
self.b_y_2 = init_bias(self.n_out, sample='zero')
self.params = [
self.W_xi_1, self.W_hi_1, self.W_ci_1, self.b_i_1,
self.W_xf_1, self.W_hf_1, self.W_cf_1, self.b_f_1,
self.W_xc_1, self.W_hc_1, self.b_c_1, self.W_xo_1, self.W_ho_1,
self.W_co_1, self.b_o_1, # self.W_hy_1, self.b_y_1,
self.W_xi_2, self.W_hi_2, self.W_ci_2, self.b_i_2,
self.W_xf_2, self.W_hf_2, self.W_cf_2, self.b_f_2,
self.W_xc_2, self.W_hc_2, self.b_c_2, self.W_xo_2, self.W_ho_2,
self.W_co_2, self.b_o_2, self.W_hy_2, self.b_y_2,
self.W_fc1, self.b_fc1,self.W_fc2, self.b_fc2,self.W_fc3, self.b_fc3
]
def step_lstm(x_t, h_tm1, c_tm1,h_tm2,c_tm2):
i_t_1 = T.nnet.sigmoid(T.dot(x_t, self.W_xi_1) + T.dot(h_tm1, self.W_hi_1) + T.dot(c_tm1, self.W_ci_1) + self.b_i_1)
f_t_1 = T.nnet.sigmoid(T.dot(x_t, self.W_xf_1) + T.dot(h_tm1, self.W_hf_1) + T.dot(c_tm1, self.W_cf_1) + self.b_f_1)
c_t_1 = f_t_1 * c_tm1 + i_t_1 * T.tanh(T.dot(x_t, self.W_xc_1) + T.dot(h_tm1, self.W_hc_1) + self.b_c_1)
o_t_1 = T.nnet.sigmoid(T.dot(x_t, self.W_xo_1) + T.dot(h_tm1, self.W_ho_1) + T.dot(c_t_1, self.W_co_1) + self.b_o_1)
h_t_1 = o_t_1 * T.tanh(c_t_1)
#y_t_1 = output_activation(T.dot(h_t_1, self.W_hy_1) + self.b_y_1)
i_t_2 = T.nnet.sigmoid(T.dot(h_t_1, self.W_xi_2) + T.dot(h_tm2, self.W_hi_2) + T.dot(c_tm2, self.W_ci_2) + self.b_i_2)
f_t_2 = T.nnet.sigmoid(T.dot(h_t_1, self.W_xf_2) + T.dot(h_tm2, self.W_hf_2) + T.dot(c_tm2, self.W_cf_2) + self.b_f_2)
c_t_2 = f_t_2 * c_tm2 + i_t_2 * T.tanh(T.dot(h_t_1, self.W_xc_2) + T.dot(h_tm2, self.W_hc_2) + self.b_c_2)
o_t_2 = T.nnet.sigmoid(T.dot(h_t_1, self.W_xo_2) + T.dot(h_tm2, self.W_ho_2) + T.dot(c_t_2, self.W_co_2) + self.b_o_2)
h_t_2 = o_t_2 * T.tanh(c_t_2)
y_t_2 = T.tanh(T.dot(h_t_2, self.W_hy_2) + self.b_y_2)
return [h_t_1,c_t_1,h_t_2,c_t_2, y_t_2]
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector (should be 0 when X is not null)
h0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_1 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
h0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
c0_2 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
[h_vals_1, c_vals_1,h_vals_2, c_vals_2, y_vals], _ = theano.scan(fn=step_lstm,
sequences=X.dimshuffle(1,0,2),
outputs_info=[h0_1, c0_1,h0_2, c0_2, None])
#Hidden layers
fc1_out = T.tanh(T.dot(y_vals, self.W_fc1) + self.b_fc1)
fc2_out = T.tanh(T.dot(fc1_out, self.W_fc2) + self.b_fc2)
fc3_out = T.tanh(T.dot(fc2_out, self.W_fc3) + self.b_fc3)
self.output=fc3_out.dimshuffle(1,0,2)
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t_1[-1],c_t_1[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t_1[-1],c_t_1[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self, rng, params, cost_function='mse', optimizer=RMSprop):
lr = params["lr"]
n_lstm = params['n_hidden']
n_out = params['n_output']
batch_size = params["batch_size"]
sequence_length = params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar(
'is_train'
) # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample = (1, 1)
p_1 = 0.5
border_mode = "valid"
cnn_batch_size = batch_size * sequence_length
pool_size = (2, 2)
#Layer1: conv2+pool+drop
filter_shape = (64, 1, 9, 9)
input_shape = (cnn_batch_size, 1, 120, 60
) #input_shape= (samples, channels, rows, cols)
input = X.reshape(input_shape)
c1 = ConvLayer(rng,
input,
filter_shape,
input_shape,
border_mode,
subsample,
activation=nn.relu)
p1 = PoolLayer(c1.output,
pool_size=pool_size,
input_shape=c1.output_shape)
dl1 = DropoutLayer(rng, input=p1.output, prob=p_1)
retain_prob = 1. - p_1
test_output = p1.output * retain_prob
d1_output = T.switch(T.neq(is_train, 0), dl1.output, test_output)
#Layer2: conv2+pool
filter_shape = (128, p1.output_shape[1], 3, 3)
c2 = ConvLayer(rng,
d1_output,
filter_shape,
p1.output_shape,
border_mode,
subsample,
activation=nn.relu)
p2 = PoolLayer(c2.output,
pool_size=pool_size,
input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape = (128, p2.output_shape[1], 3, 3)
c3 = ConvLayer(rng,
p2.output,
filter_shape,
p2.output_shape,
border_mode,
subsample,
activation=nn.relu)
p3 = PoolLayer(c3.output,
pool_size=pool_size,
input_shape=c3.output_shape)
#Layer4: hidden
n_in = reduce(lambda x, y: x * y, p3.output_shape[1:])
x_flat = p3.output.flatten(2)
h1 = HiddenLayer(rng, x_flat, n_in, 1024, activation=nn.relu)
n_in = 1024
rnn_input = h1.output.reshape((batch_size, sequence_length, n_in))
#Layer5: gru
self.n_in = n_in
self.n_lstm = n_lstm
self.n_out = n_out
self.W_xr = init_weight((self.n_in, self.n_lstm),
rng=rng,
name='W_xi',
sample='glorot')
self.W_hr = init_weight((self.n_lstm, self.n_lstm),
rng=rng,
name='W_hr',
sample='glorot')
self.b_r = init_bias(self.n_lstm, rng=rng, sample='zero')
self.W_xz = init_weight((self.n_in, self.n_lstm),
rng=rng,
name='W_xz',
sample='glorot')
self.W_hz = init_weight((self.n_lstm, self.n_lstm),
rng=rng,
name='W_hz',
sample='glorot')
self.b_z = init_bias(self.n_lstm, rng=rng, sample='zero')
self.W_xh = init_weight((self.n_in, self.n_lstm),
rng=rng,
name='W_xh',
sample='glorot')
self.W_hh = init_weight((self.n_lstm, self.n_lstm),
rng=rng,
name='W_hh',
sample='glorot')
self.b_h = init_bias(self.n_lstm, rng=rng, sample='zero')
self.W_hy = init_weight((self.n_lstm, self.n_out),
rng=rng,
name='W_hy',
sample='glorot')
self.b_y = init_bias(self.n_out, rng=rng, sample='zero')
self.params = [
self.W_xr, self.W_hr, self.b_r, self.W_xz, self.W_hz, self.b_z,
self.W_xh, self.W_hh, self.b_h, self.W_hy, self.b_y
]
def step_lstm(x_t, h_tm1):
r_t = T.nnet.sigmoid(
T.dot(x_t, self.W_xr) + T.dot(h_tm1, self.W_hr) + self.b_r)
z_t = T.nnet.sigmoid(
T.dot(x_t, self.W_xz) + T.dot(h_tm1, self.W_hz) + self.b_z)
h_t = T.tanh(
T.dot(x_t, self.W_xh) + T.dot((r_t * h_tm1), self.W_hh) +
self.b_h)
hh_t = z_t * h_t + (1 - z_t) * h_tm1
y_t = T.dot(hh_t, self.W_hy) + self.b_y
return [hh_t, y_t]
h0 = shared(np.zeros(shape=(batch_size, self.n_lstm),
dtype=dtype)) # initial hidden state
#(1, 0, 2) -> AxBxC to BxAxC
#(batch_size,sequence_length, n_in) >> (sequence_length, batch_size ,n_in)
#T.dot(x_t, self.W_xi)x_t=(sequence_length, batch_size ,n_in), W_xi= [self.n_in, self.n_lstm]
[h_vals,
y_vals], _ = theano.scan(fn=step_lstm,
sequences=rnn_input.dimshuffle(1, 0, 2),
outputs_info=[h0, None])
self.output = y_vals.dimshuffle(1, 0, 2)
self.params = c1.params + c2.params + c3.params + h1.params + self.params
cost = get_err_fn(self, cost_function, Y)
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X, Y, is_train],
outputs=cost,
updates=_optimizer.getUpdates(),
allow_input_downcast=True)
self.predictions = theano.function(inputs=[X, is_train],
outputs=self.output,
allow_input_downcast=True)
self.n_param = count_params(self.params)
def __init__(self, rng, params, cost_function='mse', optimizer=RMSprop):
lr = params["lr"]
batch_size = params["batch_size"]
sequence_length = params["seq_length"]
# minibatch)
X = T.matrix(name="input", dtype=dtype) # batch of sequence of vector
Y = T.matrix(name="output", dtype=dtype) # batch of sequence of vector
is_train = T.iscalar(
'is_train'
) # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample = (1, 1)
p_1 = 0.5
border_mode = "same"
cnn_batch_size = batch_size
pool_size = (2, 2)
#Layer1: conv2+pool+drop
filter_shape = (128, 1, 10, 10)
input_shape = (cnn_batch_size, 1, 144, 176
) #input_shape= (samples, channels, rows, cols)
input = X.reshape(input_shape)
c1 = ConvLayer(rng,
input,
filter_shape,
input_shape,
border_mode,
subsample,
activation=nn.relu)
p1 = PoolLayer(c1.output,
pool_size=pool_size,
input_shape=c1.output_shape)
dl1 = DropoutLayer(rng, input=p1.output, prob=p_1, is_train=is_train)
#Layer2: conv2+pool
subsample = (1, 1)
filter_shape = (256, p1.output_shape[1], 3, 3)
c2 = ConvLayer(rng,
dl1.output,
filter_shape,
p1.output_shape,
border_mode,
subsample,
activation=nn.relu)
p2 = PoolLayer(c2.output,
pool_size=pool_size,
input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape = (256, p2.output_shape[1], 3, 3)
c3 = ConvLayer(rng,
p2.output,
filter_shape,
p2.output_shape,
border_mode,
subsample,
activation=nn.relu)
p3 = PoolLayer(c3.output,
pool_size=pool_size,
input_shape=c3.output_shape)
#Layer4: conv2+pool
filter_shape = (128, p3.output_shape[1], 3, 3)
c4 = ConvLayer(rng,
p3.output,
filter_shape,
p3.output_shape,
border_mode,
subsample,
activation=nn.relu)
p4 = PoolLayer(c4.output,
pool_size=pool_size,
input_shape=c4.output_shape)
#Layer5: hidden
n_in = reduce(lambda x, y: x * y, p4.output_shape[1:])
x_flat = p4.output.flatten(2)
h1 = HiddenLayer(rng, x_flat, n_in, 1024, activation=nn.relu)
#Layer6: hidden
lreg = LogisticRegression(rng, h1.output, 1024, params['n_output'])
self.output = lreg.y_pred
self.params = c1.params + c2.params + c3.params + c4.params + h1.params + lreg.params
cost = get_err_fn(self, cost_function, Y)
L2_reg = 0.0001
L2_sqr = theano.shared(0.)
for param in self.params:
L2_sqr += (T.sum(param[0]**2) + T.sum(param[1]**2))
cost += L2_reg * L2_sqr
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X, Y, is_train],
outputs=cost,
updates=_optimizer.getUpdates(),
allow_input_downcast=True)
self.predictions = theano.function(inputs=[X, is_train],
outputs=self.output,
allow_input_downcast=True)
self.n_param = count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
n_lstm=params['n_hidden']
n_out=params['n_output']
batch_size=params["batch_size"]
sequence_length=params["seq_length"]
# minibatch)
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample=(1,1)
p_1=0.5
border_mode="valid"
cnn_batch_size=batch_size*sequence_length
pool_size=(2,2)
#Layer1: conv2+pool+drop
filter_shape=(64,1,9,9)
input_shape=(cnn_batch_size,1,120,60) #input_shape= (samples, channels, rows, cols)
input= X.reshape(input_shape)
c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu)
p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape)
dl1=DropoutLayer(rng,input=p1.output,prob=p_1)
retain_prob = 1. - p_1
test_output = p1.output*retain_prob
d1_output = T.switch(T.neq(is_train, 0), dl1.output, test_output)
#Layer2: conv2+pool
filter_shape=(128,p1.output_shape[1],3,3)
c2=ConvLayer(rng, d1_output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu)
p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape=(128,p2.output_shape[1],3,3)
c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu)
p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape)
#Layer4: hidden
n_in= reduce(lambda x, y: x*y, p3.output_shape[1:])
x_flat = p3.output.flatten(2)
h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu)
n_in=1024
rnn_input = h1.output.reshape((batch_size,sequence_length, n_in))
#Layer5: gru
self.n_in = n_in
self.n_lstm = n_lstm
self.n_out = n_out
self.W_xr = init_weight((self.n_in, self.n_lstm),rng=rng,name='W_xi',sample= 'glorot')
self.W_hr = init_weight((self.n_lstm, self.n_lstm),rng=rng, name='W_hr', sample='glorot')
self.b_r = init_bias(self.n_lstm,rng=rng, sample='zero')
self.W_xz = init_weight((self.n_in, self.n_lstm), rng=rng, name='W_xz', sample='glorot')
self.W_hz = init_weight((self.n_lstm, self.n_lstm), rng=rng, name='W_hz', sample='glorot')
self.b_z = init_bias(self.n_lstm,rng=rng, sample='zero')
self.W_xh = init_weight((self.n_in, self.n_lstm), rng=rng, name='W_xh', sample='glorot')
self.W_hh = init_weight((self.n_lstm, self.n_lstm), rng=rng, name='W_hh',sample= 'glorot')
self.b_h = init_bias(self.n_lstm,rng=rng, sample='zero')
self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(self.n_out,rng=rng, sample='zero')
self.params = [self.W_xr, self.W_hr, self.b_r,
self.W_xz, self.W_hz, self.b_z,
self.W_xh, self.W_hh, self.b_h,
self.W_hy, self.b_y]
def step_lstm(x_t, h_tm1):
r_t = T.nnet.sigmoid(T.dot(x_t, self.W_xr) + T.dot(h_tm1, self.W_hr) + self.b_r)
z_t = T.nnet.sigmoid(T.dot(x_t, self.W_xz) + T.dot(h_tm1, self.W_hz) + self.b_z)
h_t = T.tanh(T.dot(x_t, self.W_xh) + T.dot((r_t*h_tm1),self.W_hh) + self.b_h)
hh_t = z_t * h_t + (1-z_t)*h_tm1
y_t = T.dot(hh_t, self.W_hy) + self.b_y
return [hh_t, y_t]
h0 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state
#(1, 0, 2) -> AxBxC to BxAxC
#(batch_size,sequence_length, n_in) >> (sequence_length, batch_size ,n_in)
#T.dot(x_t, self.W_xi)x_t=(sequence_length, batch_size ,n_in), W_xi= [self.n_in, self.n_lstm]
[h_vals, y_vals], _ = theano.scan(fn=step_lstm,
sequences=rnn_input.dimshuffle(1,0,2),
outputs_info=[h0, None])
self.output = y_vals.dimshuffle(1,0,2)
self.params =c1.params+c2.params+c3.params+h1.params+self.params
cost=get_err_fn(self,cost_function,Y)
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X,Y,is_train],outputs=cost,updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train], outputs = self.output,allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop):
lr=params["lr"]
batch_size=params["batch_size"]
# minibatch)
X = T.tensor4(name="input",dtype=dtype) # batch of sequence of vector
Y = T.matrix(name="output",dtype=dtype) # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
#CNN global parameters.
subsample=(1,1)
p_1=0.5
border_mode="same"
cnn_batch_size=batch_size
pool_size=(2,2)
#Layer1: conv2+pool+drop
filter_shape=(64,3,9,9)
input_shape=(cnn_batch_size,3,112,112) #input_shape= (samples, channels, rows, cols)
input= X.dimshuffle(0,3,1,2)
c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu)
p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape)
dl1=DropoutLayer(rng,input=p1.output,prob=p_1,is_train=is_train)
#Layer2: conv2+pool
subsample=(1,1)
filter_shape=(128,p1.output_shape[1],3,3)
c2=ConvLayer(rng, dl1.output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu)
p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape)
#Layer3: conv2+pool
filter_shape=(128,p2.output_shape[1],3,3)
c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu)
p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape)
#Layer4: conv2+pool
filter_shape=(64,p3.output_shape[1],3,3)
c4=ConvLayer(rng, p3.output,filter_shape,p3.output_shape,border_mode,subsample, activation=nn.relu)
p4=PoolLayer(c4.output,pool_size=pool_size,input_shape=c4.output_shape)
#Layer5: hidden
n_in= reduce(lambda x, y: x*y, p4.output_shape[1:])
x_flat = p4.output.flatten(2)
h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu)
#Layer6: hidden
lreg=LogisticRegression(rng,h1.output,1024,params['n_output'])
self.output = lreg.y_pred
self.params =c1.params+c2.params+c3.params+c4.params+h1.params+lreg.params
cost=get_err_fn(self,cost_function,Y)
L2_reg=0.0001
L2_sqr = theano.shared(0.)
for param in self.params:
L2_sqr += (T.sum(param[0] ** 2)+T.sum(param[1] ** 2))
cost += L2_reg*L2_sqr
_optimizer = optimizer(cost, self.params, lr=lr)
self.train = theano.function(inputs=[X,Y,is_train],outputs=cost,updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train], outputs = self.output,allow_input_downcast=True)
self.n_param=count_params(self.params)
def __init__(self,rng, params,cost_function='mse',optimizer = RMSprop):
batch_size=params['batch_size']
sequence_length=params["seq_length"]
lr=params['lr']
self.n_in = 48
self.n_lstm = params['n_hidden']
self.n_out = params['n_output']
n_fc=512
X = T.tensor3() # batch of sequence of vector
Y = T.tensor3() # batch of sequence of vector
is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction
self.W_hy = init_weight((self.n_lstm, n_fc), rng=rng,name='W_hy', sample= 'glorot')
self.b_y = init_bias(n_fc,rng=rng, sample='zero')
self.numOfLayers=3
layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm)
layer2=LSTMLayer(rng,1,self.n_lstm,self.n_lstm)
layer3=LSTMLayer(rng,2,self.n_lstm,self.n_lstm)
self.params = layer1.params+layer2.params+layer3.params
self.params.append(self.W_hy)
self.params.append(self.b_y)
def step_lstm(x_t,mask,h_tm1_1,c_tm1_1,h_tm1_2,c_tm1_2,h_tm1_3,c_tm1_3):
[h_t_1,c_t_1,y_t_1]=layer1.run(x_t,h_tm1_1,c_tm1_1)
dl1=DropoutLayer(rng,input=y_t_1,prob=0.5,is_train=is_train,mask=mask)
[h_t_2,c_t_2,y_t_2]=layer2.run(dl1.output,h_tm1_2,c_tm1_2)
[h_t_3,c_t_3,y_t_3]=layer3.run(y_t_2,h_tm1_3,c_tm1_3)
y = T.dot(y_t_3, self.W_hy) + self.b_y
return [h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y]
h0_1 = T.matrix(name="h0_1",dtype=dtype) # initial hidden state
c0_1 = T.matrix(name="c0_1",dtype=dtype) # initial hidden state
h0_2 = T.matrix(name="h0_2",dtype=dtype) # initial hidden state
c0_2 = T.matrix(name="c0_2",dtype=dtype) # initial hidden state
h0_3 = T.matrix(name="h0_3",dtype=dtype) # initial hidden state
c0_3 = T.matrix(name="c0_3",dtype=dtype) # initial hidden state
mask_shape=(sequence_length,batch_size,self.n_lstm)
p_1=0.5
mask= rng.binomial(size=mask_shape, p=p_1, dtype=X.dtype)
noise= rng.normal(size=(batch_size,sequence_length,self.n_in), std=0.008, avg=0.0,dtype=theano.config.floatX)
X_train=noise+X
X_tilde= T.switch(T.neq(is_train, 0), X_train, X)
[h_t_1,c_t_1,h_t_2,c_t_2,h_t_3,c_t_3,y_vals], _ = theano.scan(fn=step_lstm,
sequences=[X_tilde.dimshuffle(1,0,2),mask],
outputs_info=[h0_1, c0_1,h0_2, c0_2, h0_3, c0_3, None])
# self.output = y_vals.dimshuffle(1,0,2)
mdn_input=y_vals.dimshuffle(1,0,2)
mdn_input=T.reshape(mdn_input,(batch_size*sequence_length,n_fc))
Y_ll=T.reshape(Y,(batch_size*sequence_length,params['n_output']))
mdn = MDNoutputLayer(rng=rng,
input=mdn_input,
n_in=n_fc,
n_out=params['n_output'],
mu_activation=do_nothing,
n_components=5)
self.params=self.params+mdn.params
# self.params.append(mdn.W_mixing)
# self.params.append(mdn.W_mu)
# self.params.append(mdn.W_sigma)
cost = nll(mu = mdn.mu,
sigma = mdn.sigma,
mixing = mdn.mixing,
y = Y_ll) #+ L2_reg * self.frame_pred.L2_sqr
_optimizer = optimizer(
cost,
self.params,
lr=lr
)
#Sampling from the GMM
component = rng.multinomial(pvals=mdn.mixing)
component_mean = T.sum(mdn.mu * component.dimshuffle(0,'x',1),axis=2)
component_std = T.sum(mdn.sigma * component, axis=1, keepdims=True)
samples=rng.normal(size=(batch_size*sequence_length,params['n_output']),avg = component_mean, std=component_std)
self.output = T.reshape(samples,(batch_size,sequence_length,params['n_output']))
self.train = theano.function(inputs=[X,Y,is_train,h0_1, c0_1,h0_2, c0_2, h0_3, c0_3],outputs=[cost,h_t_1[-1],c_t_1[-1],h_t_2[-1],c_t_2[-1],h_t_3[-1],c_t_3[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True)
self.predictions = theano.function(inputs = [X,is_train,h0_1, c0_1,h0_2, c0_2, h0_3, c0_3], outputs = [self.output,h_t_1[-1],c_t_1[-1],h_t_2[-1],c_t_2[-1],h_t_3[-1],c_t_3[-1]],allow_input_downcast=True)
self.n_param=count_params(self.params)
