def build_b(self): m = np.float32( self._m_val ) X=self.X y=self.y lambda_mult=self.lambda_mult sigma=self._sigma_val def rbf_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi): # non-sequences needed at each iteration; they don't change b_i = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + b_i b_00 = theano.shared(np.float32(0.)) output,update=theano.reduce(fn=rbf_step, sequences=[X,y,lambda_mult],outputs_info=[b_00], non_sequences=[X[0]]) updates=[update,] for i in range(1,m): b_i0= theano.shared(np.float32(0.)) outputi,updatei=theano.reduce(fn=rbf_step, sequences=[X,y,lambda_mult],outputs_info=[b_i0], non_sequences=[X[i]]) output +=outputi updates.append(updatei) # output = np.float32(1.)/m * ( y.sum() - np.float32(-1.) * output ) output = np.float32(1./m) * ( y.sum() - output ) self.b = output return output, updates
def make_predict(self,X_test_val): m = self._m_val X_test_val = X_test_val.astype(theano.config.floatX) X_test = theano.shared( X_test_val ) X=self.X y=self.y lambda_mult=self.lambda_mult sigma=self._sigma_val b = self.b def rbf_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi ): # non-sequences needed at each iteration; they don't change y_pred = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + y_pred y_pred_var = theano.shared(np.float32(0.)) output,update=theano.reduce(fn=rbf_step, sequences=[X,y,lambda_mult],outputs_info=[y_pred_var], non_sequences=[X_test]) output += b y_pred_function = theano.function(inputs=[], outputs = output) y_pred_val = y_pred_function() return y_pred_val, y_pred_function
def pv_function(self, tensor_input):
indexf_matrix = theano.shared(np.zeros(
[self.max_length, self.max_length], dtype=np.int32),
name='indexf_matrix',
borrow=True)
pf_matrix = theano.shared(np.zeros([self.max_length, self.max_length],
dtype=theano.config.floatX),
name='pf_matrix',
borrow=True)
pf_matrix = T.set_subtensor(pf_matrix[0, 0:tensor_input.shape[0]], 1.0)
vf_matrix = theano.shared(np.zeros(
(self.max_length, self.max_length, self.size),
dtype=theano.config.floatX),
name='vf_matrix',
borrow=True)
results, updates = theano.map(
fn=lambda i, L, t_tensor_input: L[t_tensor_input[i]],
sequences=[T.arange(tensor_input.shape[0])],
non_sequences=[self.L, tensor_input],
name='vf_matrix prepare')
vf_matrix = T.set_subtensor(vf_matrix[0, 0:tensor_input.shape[0]],
results)
[indexf_matrix, pf_matrix, vf_matrix], updates = theano.reduce(
fn=self._pv_row,
sequences=[T.arange(1, self.max_length)],
outputs_info=[indexf_matrix, pf_matrix, vf_matrix],
#name = 'pv function'
)
return indexf_matrix, pf_matrix, vf_matrix
def ncdf(t, A, b, v, s):
"""Probability of no response from a set of accumulators."""
ncdf_all, updates = theano.reduce(fn=lambda v_i, tot, t, A, b, s:
(1 - tcdf(t, A, b, v_i, s)) * tot,
sequences=v,
outputs_info=tt.ones_like(t),
non_sequences=[t, A, b, s])
return ncdf_all
def pv_function(self, tensor_input): indexf_matrix = theano.shared( np.ones( (self.max_length, self.max_length), dtype=theano.config.floatX ), name = 'indexf_matrix', borrow=True ) pf_matrix = theano.shared( np.eye( self.max_length, dtype=theano.config.floatX ), name = 'pf_matrix', borrow=True ) vf_matrix = theano.shared( np.zeros( (self.max_length, self.max_length, self.size), dtype=theano.config.floatX ), name = 'vf_matrix', borrow=True ) results, updates = theano.reduce( fn = lambda i, t_vf_matrix, L, t_tensor_input: T.set_subtensor(t_vf_matrix[i, i], L[t_tensor_input[i]]), outputs_info = vf_matrix, sequences=[T.arange(tensor_input.shape[0])], non_sequences=[self.L, tensor_input], name = 'vf_matrix prepare' ) vf_matrix = results for i in range(1,self.max_length): ''' for j in range(self.max_length-i): new_index, new_pf, new_vf = self._pv_function1(j, j+i, pf_matrix, vf_matrix) indexf_matrix = T.set_subtensor(indexf_matrix[j, j+i], new_index) pf_matrix = T.set_subtensor(pf_matrix[j, j+i], new_pf) vf_matrix = T.set_subtensor(vf_matrix[j, j+i], new_vf) ''' results, updates = theano.map( fn = lambda j, pf_matrix, vf_matrix, i: self._pv_function1(j, j+i, pf_matrix, vf_matrix), sequences=[T.arange(self.max_length-i)], non_sequences = [pf_matrix, vf_matrix, i], #name = 'pv function' ) for j in range(self.max_length-i): indexf_matrix = T.set_subtensor(indexf_matrix[j, j+i], results[0][j]) pf_matrix = T.set_subtensor(pf_matrix[j, j+i], results[1][j]) vf_matrix = T.set_subtensor(vf_matrix[j, j+i], results[2][j]) return indexf_matrix, pf_matrix, vf_matrix
def R2_RNN_block(tparams, inputs, prefix=None, name='r2_rnn', std=True):
prefix = GetPrefix(prefix, name)
n_steps = inputs.shape[0]
n_samples = inputs.shape[1]
x_size = inputs.shape[2]
r_steps = T.ceil(T.log2(n_steps)).astype('uint32')
r_steps = T.arange(r_steps)
# r_steps=r_steps.reshape([r_steps.shape[0],1]);
def _step_inner(index, num, inps):
index = index * 2
index_ = T.minimum(index + 2, num)
h = RNN_layer(tparams,
inps[index:index_, :, :],
prefix=prefix,
name=None,
std=False)
return h[-1, :, :]
def _step(r_step, num, inps, std=True):
n = num
steps = T.arange((n + 1) / 2)
# steps=steps.reshape([steps.shape[0],1]);
out, updates = theano.scan(
lambda index, num, inps: _step_inner(index, num, inps),
sequences=[steps],
outputs_info=None,
non_sequences=[num, inps],
name=_p(prefix, 'inner_scan'),
n_steps=steps.shape[0],
profile=False)
# if std: out=standardize(out);
num = out.shape[0]
h = T.zeros_like(inps)
h = T.set_subtensor(h[:num], out)
return num, h
# return out;
if std: inputs = standardize(inputs)
out, updates = theano.reduce(
lambda r_step, num, inps: _step(r_step, num, inps),
sequences=r_steps,
outputs_info=[inputs.shape[0], inputs],
# non_sequences=inputs,
name=_p(prefix, 'scan'))
return out[1][:out[0]]
def build_W(self): m = self._m_val X=self.X y=self.y lambda_mult=self.lambda_mult sigma=self._sigma_val def rbf_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi,yi,lambda_multi): # non-sequences needed at each iteration; they don't change W_i = lambda_multi*yi*lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + W_i W_00 = theano.shared(np.float32(0.)) output,update=theano.reduce(fn=rbf_step, sequences=[X,y,lambda_mult],outputs_info=[W_00], non_sequences=[X[0],y[0],lambda_mult[0]]) updates=[update,] for i in range(1,m): W_i0= theano.shared(np.float32(0.)) outputi,updatei=theano.reduce(fn=rbf_step, sequences=[X,y,lambda_mult],outputs_info=[W_i0], non_sequences=[X[i],y[i],lambda_mult[i]]) output +=outputi updates.append(updatei) output *= np.float32(0.5) output -= lambda_mult.sum() self.W = sandbox.cuda.basic_ops.gpu_from_host( output ) return output, updates
def step(Xi, # sequence to iterate over X,y,lambda_mult,b): # non-recurrent non-sequences needed at each iteration; they don't change def j_step(Xj,yj,lambda_multj, # sequences to iterate over the sum from j=1...m cumulative_sum, # previous iteration Xi): # non-sequence needed at each iteration for the summation yhat_j = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + yhat_j yhat_0 = theano.shared(np.float32(0.)) output,update=theano.reduce(fn=j_step, sequences=[X,y,lambda_mult],outputs_info=[yhat_0], non_sequences=[Xi] ) #output now represents the summation in the formula for yhat, but without the intercept b output += b return output
def sens_map_fn(self, x_set, x_c_set, x_d_diff_set, x_r_diff_set): #large batch comes
pred_map_0 = T.zeros((x_set.shape[1], x_set.shape[2], 28, 28), 'float32')
sens_map_0 = T.zeros((x_set.shape[1], x_set.shape[2], 28, 28), 'float32')
tv_scan_0 = T.zeros(1, 'float32')
[pred_map, sens_map, tv_scan], _ = theano.reduce(
self._sens_map_fn_wrapper,
sequences=[x_set, x_c_set, x_d_diff_set, x_r_diff_set],
outputs_info=[pred_map_0, sens_map_0, tv_scan_0]
)
pred_map /= T.cast(x_set.shape[0], 'float32')
sens_map /= T.cast(x_set.shape[0], 'float32')
tv_scan /= T.cast(x_set.shape[0], 'float32')
return pred_map, sens_map, tv_scan
def make_predictions(self,X_pred_vals): # m = self._m_val m_pred = X_pred_vals.shape[0] # number of examples X to make predictions on X_pred_vals = X_pred_vals.astype(theano.config.floatX) X_pred = theano.shared( X_pred_vals ) X_pred_var = theano.shared( X_pred_vals[0] ) X=self.X y=self.y lambda_mult=self.lambda_mult sigma=self._sigma_val b = self.b def rbf_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi ): # non-sequences needed at each iteration; they don't change y_pred = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + y_pred predictions_lst = [] # define a general prediction formula, y_pred_var = theano.shared(np.float32(0.)) # accumulate predictions of y in this shared variable output,update=theano.reduce(fn=rbf_step, sequences=[X,y,lambda_mult],outputs_info=[y_pred_var], non_sequences=[X_pred_var]) output += b output = sandbox.cuda.basic_ops.gpu_from_host( output ) # output here corresponds to yhat(X), X\equiv Xi, and is ready for substituting in values through givens parameter for i in range(m_pred): y_pred_i = theano.shared(np.float32(0.)) # accumulate predictions of y in this shared variable X_pred_i = X_pred[i] # ith example of input data X to make predictions on y_pred_function = theano.function(inputs=[],outputs=output,givens=[(X_pred_var,X_pred_i),(y_pred_var,y_pred_i)]) y_pred_val = y_pred_function() predictions_lst.append( y_pred_val ) return predictions_lst
def R2_RNN_block(tparams,inputs,prefix=None,name='r2_rnn',std=True):
prefix=GetPrefix(prefix,name);
n_steps=inputs.shape[0];
n_samples=inputs.shape[1];
x_size=inputs.shape[2];
r_steps=T.ceil(T.log2(n_steps)).astype('uint32');
r_steps=T.arange(r_steps);
# r_steps=r_steps.reshape([r_steps.shape[0],1]);
def _step_inner(index,num,inps):
index=index*2;
index_=T.minimum(index+2,num);
h=RNN_layer(tparams,inps[index:index_,:,:],prefix=prefix,name=None,std=False);
return h[-1,:,:];
def _step(r_step,num,inps,std=True):
n=num;
steps=T.arange((n+1)/2);
# steps=steps.reshape([steps.shape[0],1]);
out,updates=theano.scan(lambda index,num,inps:_step_inner(index,num,inps),
sequences=[steps],
outputs_info=None,
non_sequences=[num,inps],
name=_p(prefix,'inner_scan'),
n_steps=steps.shape[0],
profile=False);
# if std: out=standardize(out);
num=out.shape[0];
h=T.zeros_like(inps);
h=T.set_subtensor(h[:num],out);
return num,h;
# return out;
if std: inputs=standardize(inputs);
out,updates=theano.reduce(lambda r_step,num,inps:_step(r_step,num,inps),
sequences=r_steps,
outputs_info=[inputs.shape[0],inputs],
# non_sequences=inputs,
name=_p(prefix,'scan')
);
return out[1][:out[0]];
def pv_function(self, tensor_input):
indexf_matrix = theano.shared(np.ones(
(self.max_length, self.max_length), dtype=theano.config.floatX),
name='indexf_matrix',
borrow=True)
pf_matrix = theano.shared(np.eye(self.max_length,
dtype=theano.config.floatX),
name='pf_matrix',
borrow=True)
vf_matrix = theano.shared(np.zeros(
(self.max_length, self.max_length, self.size),
dtype=theano.config.floatX),
name='vf_matrix',
borrow=True)
results, updates = theano.reduce(
fn=lambda i, t_vf_matrix, L, t_tensor_input: T.set_subtensor(
t_vf_matrix[i, i], L[t_tensor_input[i]]),
outputs_info=vf_matrix,
sequences=[T.arange(tensor_input.shape[0])],
non_sequences=[self.L, tensor_input],
name='vf_matrix prepare')
vf_matrix = results
for i in range(1, self.max_length):
'''
for j in range(self.max_length-i):
new_index, new_pf, new_vf = self._pv_function1(j, j+i, pf_matrix, vf_matrix)
indexf_matrix = T.set_subtensor(indexf_matrix[j, j+i], new_index)
pf_matrix = T.set_subtensor(pf_matrix[j, j+i], new_pf)
vf_matrix = T.set_subtensor(vf_matrix[j, j+i], new_vf)
'''
results, updates = theano.map(
fn=lambda j, pf_matrix, vf_matrix, i: self._pv_function1(
j, j + i, pf_matrix, vf_matrix),
sequences=[T.arange(self.max_length - i)],
non_sequences=[pf_matrix, vf_matrix, i],
#name = 'pv function'
)
for j in range(self.max_length - i):
indexf_matrix = T.set_subtensor(indexf_matrix[j, j + i],
results[0][j])
pf_matrix = T.set_subtensor(pf_matrix[j, j + i], results[1][j])
vf_matrix = T.set_subtensor(vf_matrix[j, j + i], results[2][j])
return indexf_matrix, pf_matrix, vf_matrix
def trial_resp_pdf(t, ind, A, b, v, s):
"""Probability density function for response i at time t."""
p_neg, updates = theano.reduce(
fn=lambda v_i, tot, s: normcdf(-v_i / s) * tot,
sequences=v,
outputs_info=tt.ones(1, dtype='float64'),
non_sequences=s)
# PDF for i and no finish yet for others
v_ind = tt.arange(v.shape[0])
i = tt.cast(ind, 'int64')
pdf = (tpdf(t, A, b, v[i], s) *
ncdf(t, A, b, v[tt.nonzero(tt.neq(v_ind, i))], s)) / (1 - p_neg)
# define probability of negative times to zero
pdf_cond = tt.switch(tt.gt(t, 0), pdf, 0)
return pdf_cond
def i_step(Xi, # sequences to iterate over cumulative_sum, # previous iteration X,y,lambda_mult): # non-sequences needed at each iteration; they don't change # j step, calculating W_i def j_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi): # non-sequence needed at each jth iteration, it doesn't change b_ij = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + b_ij b_i0=theano.shared(np.float32(0.)) output,update=theano.reduce(fn=j_step, sequences=[X,y,lambda_mult],outputs_info=[b_i0], non_sequences=[Xi] ) # output at this point, symbolically represents b_i return cumulative_sum + output
def pv_function(self, tensor_input): indexf_matrix = theano.shared( np.zeros( [self.max_length, self.max_length], dtype=np.int32 ), name = 'indexf_matrix', borrow=True ) pf_matrix = theano.shared( np.zeros( [self.max_length, self.max_length], dtype=theano.config.floatX ), name = 'pf_matrix', borrow=True ) pf_matrix = T.set_subtensor(pf_matrix[0, 0:tensor_input.shape[0]], 1.0) vf_matrix = theano.shared( np.zeros( (self.max_length, self.max_length, self.size), dtype=theano.config.floatX ), name = 'vf_matrix', borrow=True ) results, updates = theano.map( fn = lambda i, L, t_tensor_input: L[t_tensor_input[i]], sequences=[T.arange(tensor_input.shape[0])], non_sequences=[self.L, tensor_input], name = 'vf_matrix prepare' ) vf_matrix = T.set_subtensor(vf_matrix[0, 0:tensor_input.shape[0]], results) [indexf_matrix, pf_matrix, vf_matrix], updates = theano.reduce( fn = self._pv_row, sequences=[T.arange(1,self.max_length)], outputs_info=[indexf_matrix, pf_matrix, vf_matrix], #name = 'pv function' ) return indexf_matrix, pf_matrix, vf_matrix
def build_W(self): m = self._m_val X=self.X y=self.y lambda_mult=self.lambda_mult sigma=self._sigma_val W = theano.shared(np.float32(0.)) def i_step(Xi,yi,lambda_multi, # sequences to iterate over cumulative_sum, # previous iteration X,y,lambda_mult): # non-sequences needed at each iteration; they don't change # j step, calculating W_i def j_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi): # non-sequence needed at each jth iteration, it doesn't change W_ij = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + W_ij W_i0=theano.shared(np.float32(0.)) output,update=theano.reduce(fn=j_step, sequences=[X,y,lambda_mult],outputs_info=[W_i0], non_sequences=[Xi] ) # output at this point, symbolically represents W_i W_i = lambda_multi * yi * output return cumulative_sum + W_i output,update=theano.reduce(fn=i_step, sequences=[X,y,lambda_mult],outputs_info=[W], non_sequences=[X,y,lambda_mult]) output *= np.float32(0.5) output -= lambda_mult.sum() self.W = sandbox.cuda.basic_ops.gpu_from_host( output ) return output, update
def calc_cost(self, phi_x, phi_f1_1, phi_f1_2, phi_f2_1, phi_f2_2, phi_m0, phi_m1, phi_r, phi_rbars, tot_sr_cost):
score1_1 = self.calc_score_o(phi_x, phi_f1_1)
score1_2 = self.calc_score_o(phi_x, phi_f1_2)
score2_1 = self.calc_score_o(phi_x + phi_m0, phi_f2_1)
score2_2 = self.calc_score_o(phi_x + phi_m0, phi_f2_2)
s_o_cost = (
T.maximum(0, self.margin - score1_1) + T.maximum(0, self.margin + score1_2) +
T.maximum(0, self.margin - score2_1) + T.maximum(0, self.margin + score2_2)
)
def compute_sr_cost(phi_rbar, correct_score):
false_score = self.calc_score_r(phi_x + phi_m0 + phi_m1, phi_rbar)
return T.maximum(0, self.margin - correct_score + false_score)
correct_score3 = self.calc_score_r(phi_x + phi_m0 + phi_m1, phi_r)
sr_costs, sr_updates = theano.reduce(lambda phi_rbar, tot_sr_cost: tot_sr_cost + compute_sr_cost(phi_rbar, correct_score3),
sequences=phi_rbars, outputs_info=[{'initial': tot_sr_cost}])
cost = s_o_cost + sr_costs
return cost
def build_b(self): m = np.float32( self._m_val ) X=self.X y=self.y lambda_mult=self.lambda_mult sigma=self._sigma_val def i_step(Xi, # sequences to iterate over cumulative_sum, # previous iteration X,y,lambda_mult): # non-sequences needed at each iteration; they don't change # j step, calculating W_i def j_step(Xj,yj,lambda_multj, # sequences to iterate over cumulative_sum, # previous iteration Xi): # non-sequence needed at each jth iteration, it doesn't change b_ij = lambda_multj*yj*rbf(Xj,Xi,sigma) return cumulative_sum + b_ij b_i0=theano.shared(np.float32(0.)) output,update=theano.reduce(fn=j_step, sequences=[X,y,lambda_mult],outputs_info=[b_i0], non_sequences=[Xi] ) # output at this point, symbolically represents b_i return cumulative_sum + output b = theano.shared(np.float32(0.)) output,update=theano.reduce(fn=i_step, sequences=[X],outputs_info=[b], non_sequences=[X,y,lambda_mult]) output = np.float32(1./m) * ( y.sum() - output ) self.b = sandbox.cuda.basic_ops.gpu_from_host( output ) return output, update
def apply_gaussian(Uv, mk, a, th, current):
for i in range(ngauss):
arg = T.exp(Uv[i * nu:i * nu + nout])
current += T.sum(arg)
U = T.diag(arg)
U = T.set_subtensor(U[np.triu_indices(nout, 1)],
Uv[i * nu + nout:(i + 1) * nu])
r = th - mk[i * nout:(i + 1) * nout]
r2 = T.dot(r, T.dot(U.T, T.dot(U, r)))
current += T.log(a[i]) - 0.5 * r2
return current
outputs_info = T.as_tensor_variable(np.asarray(0.0, float))
lnprob, _ = theano.reduce(apply_gaussian, [Uvals, mkvals, alpha, theta],
outputs_info)
cost = -lnprob
params = [W, b, W_alpha, b_alpha, W_mk, b_mk, W_u, b_u]
# for p in params:
# cost += 0.1 * T.sum(p**2)
grads = T.grad(cost, params)
updates = get_adam_updates(cost, params)
strt = time.time()
update_step = theano.function([x, theta], outputs=cost, updates=updates)
cost_func = theano.function([x, theta], outputs=cost)
print(time.time() - strt)
# Compute the Gaussian cost using a reduce:
Uvals = T.dot(y, W_u) + b_u
mkvals = T.dot(y, W_mk) + b_mk
def apply_gaussian(Uv, mk, a, th, current):
for i in range(ngauss):
arg = T.exp(Uv[i*nu:i*nu+nout])
current += T.sum(arg)
U = T.diag(arg)
U = T.set_subtensor(U[np.triu_indices(nout, 1)],
Uv[i*nu+nout:(i+1)*nu])
r = th - mk[i*nout:(i+1)*nout]
r2 = T.dot(r, T.dot(U.T, T.dot(U, r)))
current += T.log(a[i]) - 0.5 * r2
return current
outputs_info = T.as_tensor_variable(np.asarray(0.0, float))
lnprob, _ = theano.reduce(apply_gaussian, [Uvals, mkvals, alpha, theta],
outputs_info)
cost = -lnprob
params = [W, b, W_alpha, b_alpha, W_mk, b_mk, W_u, b_u]
# for p in params:
# cost += 0.1 * T.sum(p**2)
grads = T.grad(cost, params)
updates = get_adam_updates(cost, params)
strt = time.time()
update_step = theano.function([x, theta], outputs=cost, updates=updates)
cost_func = theano.function([x, theta], outputs=cost)
print(time.time() - strt)
def __init__(self, nin, nout, nhidden, ngauss, nvar):
self.nin = nin
self.nout = nout
self.nhidden = nhidden
self.ngauss = ngauss
self.nu = nu = (self.nout * (self.nout + 1)) // 2
# In/out:
x = T.dmatrix("x")
theta = T.dmatrix("theta")
# Parameters:
W = theano.shared(np.random.rand(nin, nhidden), name="W")
b = theano.shared(np.random.rand(nhidden), name="b")
y = T.dot(x, W) + b
W_alpha = theano.shared(1e-8 * np.random.rand(nhidden, ngauss),
name="W_alpha")
b_alpha = theano.shared(np.zeros(ngauss), name="b_alpha")
alpha = T.nnet.softmax(T.dot(y, W_alpha) + b_alpha)
W_mk = theano.shared(1e-8 * np.random.rand(nhidden, ngauss * nout),
name="W_mk")
b_mk = theano.shared(np.zeros((ngauss * nout)), name="b_mk")
W_u = theano.shared(1e-8 * np.random.rand(nhidden, ngauss * nu),
name="W_u")
b_u = theano.shared(np.zeros((ngauss * nu)), name="b_u")
# Compute the Gaussian cost using a reduce:
Uvals = T.dot(y, W_u) + b_u
mkvals = T.dot(y, W_mk) + b_mk
def apply_gaussian(Uv, mk, a, th, current):
for i in range(ngauss):
arg = T.exp(Uv[i * nu:i * nu + nout])
current += T.sum(arg)
U = T.diag(arg)
U = T.set_subtensor(U[np.triu_indices(nout, 1)],
Uv[i * nu + nout:(i + 1) * nu])
r = th - mk[i * nout:(i + 1) * nout]
r2 = T.dot(r, T.dot(U.T, T.dot(U, r)))
current += T.log(a[i]) - 0.5 * r2
return current
outputs_info = T.as_tensor_variable(np.asarray(0.0, float))
lnprob, _ = theano.reduce(apply_gaussian,
[Uvals, mkvals, alpha, theta], outputs_info)
cost = -lnprob
self.params = [W, b, W_alpha, b_alpha, W_mk, b_mk, W_u, b_u]
self.grads = T.grad(cost, self.params)
updates = get_adam_updates(cost, self.params)
self.update_step = theano.function([x, theta],
outputs=cost,
updates=updates)
self.cost_func = theano.function([x, theta], outputs=cost)
# Stochastic objective:
ntot = np.sum([np.prod(np.shape(p.get_value())) for p in self.params])
rng = RandomStreams()
u = rng.normal((nvar, ntot))
phi_m = theano.shared(np.zeros(ntot), name="phi_m")
phi_s = theano.shared(np.zeros(ntot), name="phi_s")
phi = (phi_m + T.exp(0.5 * phi_s))[None, :] * u
print(theano.function([], outputs=phi)().shape)
