def forward_pass(W1, W2, W3, b1, b2, b3, x):
"""
forward-pass for an fully connected neural network with 2 hidden layers of M neurons
Inputs:
W1 : (M, 784) weights of first (hidden) layer
W2 : (M, M) weights of second (hidden) layer
W3 : (10, M) weights of third (output) layer
b1 : (M, 1) biases of first (hidden) layer
b2 : (M, 1) biases of second (hidden) layer
b3 : (10, 1) biases of third (output) layer
x : (N, 784) training inputs
Outputs:
Fhat : (N, 10) output of the neural network at training inputs
"""
H1 = np.maximum(0, np.dot(x, W1.T) + b1.T) # layer 1 neurons with ReLU activation, shape (N, M)
H2 = np.maximum(0, np.dot(H1, W2.T) + b2.T) # layer 2 neurons with ReLU activation, shape (N, M)
Fhat = np.dot(H2, W3.T) + b3.T # layer 3 (output) neurons with linear activation, shape (N, 10)
a = np.max(Fhat, axis=1)
# expand the value of a into a matrix to compute log-exp trick easy
A = -1*np.ones(np.shape(Fhat))*a[:, np.newaxis]
log_sum = np.log(np.sum(np.exp(np.add(Fhat, A)), axis=1))
# turn sums into a matrix as well
Log_Sum = -1*np.ones(np.shape(Fhat))*log_sum[:, np.newaxis]
# log-exp trick on each element of Fhat
Fhat = np.add(np.add(Fhat, A), Log_Sum)
return Fhat
def forward(self, X1, X2, **kwargs):
alpha, mean_lam, gamma, delta = self._get_params(X1, **kwargs)
cfg1, res1, kappa1, kr_pref1, _ = self._compute_terms(
X1, alpha, mean_lam, gamma, delta)
if X2 is not X1:
cfg2, res2, kappa2, kr_pref2, _ = self._compute_terms(
X2, alpha, mean_lam, gamma, delta)
else:
cfg2, res2, kappa2, kr_pref2 = cfg1, res1, kappa1, kr_pref1
res2 = anp.reshape(res2, (1, -1))
kappa2 = anp.reshape(kappa2, (1, -1))
kr_pref2 = anp.reshape(kr_pref2, (1, -1))
kappa12 = self._compute_kappa(
anp.add(res1, res2), alpha, mean_lam)
kmat_res = anp.subtract(kappa12, anp.multiply(kappa1, kappa2))
kmat_res = anp.multiply(kr_pref1, anp.multiply(
kr_pref2, kmat_res))
kmat_x = self.kernel_x(cfg1, cfg2)
if self.encoding_delta is None:
if delta > 0.0:
tmpmat = anp.add(kappa1, anp.subtract(
kappa2, kappa12 * delta))
tmpmat = tmpmat * (-delta) + 1.0
else:
tmpmat = 1.0
else:
tmpmat = anp.add(kappa1, anp.subtract(
kappa2, anp.multiply(kappa12, delta)))
tmpmat = anp.multiply(tmpmat, -delta) + 1.0
return kmat_x * tmpmat + kmat_res
def forward(params, inputs=None, channels_indexed=None, hps=None):
hidden_activation = hps['hidden_activation'](np.add(
np.matmul(
inputs,
params['input']['hidden']['weights'],
),
params['input']['hidden']['bias'],
))
channel_activations = np.array([
hps['channel_activation'](np.add(
np.matmul(
hidden_activation,
params['hidden']['channels']['weights'][c, :, :],
),
params['hidden']['channels']['bias'][c, :, :],
)) for c in channels_indexed
])
classifier_activation = hps['classifier_activation'](np.add(
np.matmul(
hidden_activation,
params['hidden']['classifier']['weights'],
),
params['hidden']['classifier']['bias'],
))
return [hidden_activation, channel_activations, classifier_activation]
def attn_fwd(q, k, v, wq, wk, wv, wo, nheads, rlink):
out = 0
# Threshold to detect a single spike softmax output. When
# the softmax output is close to 1.0 at one location and
# zeros or very small values elsewhere, in the backward pass
# the Jacobian of the softmax block is all zeros or very
# small values - basically noise. In such a case the multi-
# head attention dgrad/wgrad results will be inaccurate.
threshold = 1.0 / np.shape(k)[1]
for i in range(nheads):
q_bar = np.dot(wq[i, :, :], q)
k_bar = np.dot(wk[i, :, :], k)
v_bar = np.dot(wv[i, :, :], v)
beta = scaler * np.dot(k_bar.transpose(), q_bar)
alpha = softmax(beta)
chk = np.sum(alpha > threshold, axis=0)
chk = (chk == 1)
if np.all(chk):
print("WARNING: dgrad/wgrad may be inaccurate")
h_bar = np.dot(v_bar, alpha)
h = np.dot(wo[i, :, :], h_bar)
out = np.add(out, h)
# Residual connection from 'q' to 'out'.
if rlink != 0:
out = np.add(out, q)
return out
def forward(params, inputs = None, hps = None):
hidden_activations = np.array([
hps['hidden_activation'](
np.add(
np.matmul(
inputs,
params['input']['hidden']['weights'][c,:,:],
),
params['input']['hidden']['bias'][c,:,:],
)
)
for c in range(params['input']['hidden']['weights'].shape[0])
])
channel_activations = np.array([
hps['channel_activation'](
np.add(
np.matmul(
hidden_activations[c,:,:],
params['hidden']['output']['weights'][c,:,:],
),
params['hidden']['output']['bias'][c,:,:],
)
)
for c in range(params['input']['hidden']['weights'].shape[0])
])
return [hidden_activations, channel_activations]
def forward(params, inputs=None, channels=None, hps=None):
hidden_activation = hps['hidden_activation'](np.add(
np.matmul(
inputs,
params['input']['hidden']['weights'],
),
params['input']['hidden']['bias'],
))
channel_activations = np.array([
hps['channel_activation'](np.add(
np.add(
np.matmul(
hidden_activation,
params['hidden'][channel]['weights'],
),
params['hidden'][channel]['bias'],
),
np.add(np.matmul(
inputs,
params['input'][channel]['weights'],
), params['input'][channel]['bias']))) for channel in channels
])
return [hidden_activation, channel_activations]
def forward_pass(W1, W2, W3, b1, b2, b3, x):
"""
forward-pass for an fully connected neural network with 2 hidden layers of M neurons
Inputs:
W1 : (M, 784) weights of first (hidden) layer
W2 : (M, M) weights of second (hidden) layer
W3 : (10, M) weights of third (output) layer
b1 : (M, 1) biases of first (hidden) layer
b2 : (M, 1) biases of second (hidden) layer
b3 : (10, 1) biases of third (output) layer
x : (N, 784) training inputs
Outputs:
Fhat : (N, 10) output of the neural network at training inputs
"""
H1 = np.maximum(0,
np.dot(x, W1.T) +
b1.T) # layer 1 neurons with ReLU activation, shape (N, M)
H2 = np.maximum(0,
np.dot(H1, W2.T) +
b2.T) # layer 2 neurons with ReLU activation, shape (N, M)
Fhat = np.dot(
H2, W3.T
) + b3.T # layer 3 (output) neurons with linear activation, shape (N, 10)
# #######
# Note that the activation function at the output layer is linear!
# You must impliment a stable log-softmax activation function at the ouput layer
# #######
a_mat = -1 * np.ones(np.shape(Fhat)) * np.max(Fhat, axis=1)[:, np.newaxis]
log_sums = np.ones(np.shape(Fhat)) * -1 * np.log(
np.sum(np.exp(np.add(Fhat, a_mat)), axis=1))[:, np.newaxis]
Fhat = np.add(np.add(Fhat, a_mat), log_sums)
return Fhat
def d_ll(x, T, \
robot_mu_x, robot_mu_y, \
ped_mu_x, ped_mu_y, \
cov_robot_x, cov_robot_y, \
inv_cov_robot_x, inv_cov_robot_y, \
cov_ped_x, cov_ped_y, \
inv_cov_ped_x, inv_cov_ped_y, \
one_over_cov_sum_x, one_over_cov_sum_y, normalize):
d_alpha = [0. for _ in range(4 * T)]
d_beta = [0. for _ in range(4 * T)]
d_llambda = np.asarray([0. for _ in range(4 * T)])
n = 2
vel_x = x[:T] - x[n * T:(n + 1) * T]
vel_y = x[T:2 * T] - x[(n + 1) * T:(n + 2) * T]
one_over_var_sum_x = np.diag(one_over_cov_sum_x)
one_over_var_sum_y = np.diag(one_over_cov_sum_y)
# if normalize == True:
# normalize_x = np.multiply(np.power(2*np.pi, -0.5), \
# np.diag(one_over_std_sum_x))
# normalize_y = np.multiply(np.power(2*np.pi, -0.5), \
# np.diag(one_over_std_sum_y))
# else:
normalize_x = 1.
normalize_y = 1.
quad_x = np.multiply(one_over_var_sum_x, np.power(vel_x, 2))
quad_y = np.multiply(one_over_var_sum_y, np.power(vel_y, 2))
Z_x = np.multiply(normalize_x, np.exp(-0.5 * quad_x))
Z_y = np.multiply(normalize_y, np.exp(-0.5 * quad_y))
Z = np.multiply(Z_x, Z_y)
X = np.divide(Z, 1. - Z)
alpha_x = np.multiply(X, np.multiply(vel_x, one_over_var_sum_x))
alpha_y = np.multiply(X, np.multiply(vel_y, one_over_var_sum_y))
# X and Y COMPONENT OF R DERIVATIVE
d_alpha[:T] = np.add(d_alpha[:T], alpha_x)
d_alpha[T:2 * T] = np.add(d_alpha[T:2 * T], alpha_y)
d_alpha[n * T:(n + 1) * T] = -alpha_x
d_alpha[(n + 1) * T:(n + 2) * T] = -alpha_y
d_beta[n * T:(n + 1) *
T] = -np.dot(x[n * T:(n + 1) * T] - ped_mu_x, inv_cov_ped_x)
d_beta[(n + 1) * T:(n + 2) *
T] = -np.dot(x[(n + 1) * T:(n + 2) * T] - ped_mu_y, inv_cov_ped_y)
d_beta[:T] = -np.dot(x[:T] - robot_mu_x, inv_cov_robot_x)
d_beta[T:2 * T] = -np.dot(x[T:2 * T] - robot_mu_y, inv_cov_robot_y)
d_llambda[0:2 * T] = np.add(d_alpha[0:2 * T], d_beta[0:2 * T])
d_llambda[2 * T:] = np.add(d_alpha[2 * T:], d_beta[2 * T:])
return -1. * d_llambda
def forward(params, inputs = None, hps = None):
hidden1_activations = np.array([
hps['hidden1_activation'](
np.add(
np.matmul(
inputs,
params['input']['hidden1']['weights'][c,:,:],
),
params['input']['hidden1']['bias'][c,:,:],
)
)
for c in range(params['input']['hidden1']['weights'].shape[0])
])
hidden2_activations = np.array([
hps['hidden2_activation'](
np.add(
np.matmul(
hidden1_activations[c,:,:],
params['hidden1']['hidden2']['weights'][c,:,:],
),
params['hidden1']['hidden2']['bias'][c,:,:],
)
)
for c in range(params['hidden1']['hidden2']['weights'].shape[0])
])
channel_activations = np.array([
hps['channel_activation'](
np.add(
np.matmul(
hidden2_activations[c,:,:],
params['hidden2']['output']['weights'][c,:,:],
),
params['hidden2']['output']['bias'][c,:,:],
)
)
for c in range(params['hidden2']['output']['weights'].shape[0])
])
## reconstructive error
output_activation = np.sum(
np.square(
np.subtract(
inputs,
channel_activations,
)
),
axis = 2
).T
output_activation = 1 - hps['classifier_activation'](
output_activation / output_activation.sum(axis=1, keepdims = True)
)
return [hidden1_activations, hidden2_activations, channel_activations, output_activation]
def forward(self, params, X, Im, test=True):
"""
Feed-forward pass through the model
Args:
params: list of weight matrix
X: word indices,
Im: images,
test: flag, whether test or train
"""
batchsize = X.shape[0]
R = params[0] # word representations, variable 'R' in the paper
C = params[
1] # context parameter of text modality, variable 'C^i' in the paper
bw = params[2] # bias of text modality, variable 'R' in the paper
M = params[
3] # context parameter of image modality, 'C^m' in the paper
J = params[4] # weight of image modality, 'J' in the paper
bj = params[5] # bias of image modality, 'h' in the paper
########################################################################
# You should implement forward pass here!
# preds: softmax output
########################################################################
# words = segma Cr
Rtrans = np.transpose(R)
rw = np.array([[Rtrans[indice] for indice in example]
for example in X])
words = np.tensordot(rw, C, 2)
# find_words = np.reshape(np.arange(batchsize),(batchsize,1))
# print(find_words)
# print(words.shape)
# words = np.choose(find_words,words)
# image = M*g(Jx+h)
images = np.transpose(np.add(np.dot(Im, J), bj))
images = np.maximum(images, 0.0)
# images = np.array([[max(x,0.0) for x in y] for y in images])
images = np.dot(np.transpose(M), images)
images = np.transpose(images)
rhat = np.add(words, images)
total = np.reshape(np.sum(np.exp(np.add(np.dot(rhat, R), bw)), 1),
(batchsize, 1))
single = np.exp(np.add(np.dot(rhat, R), bw))
preds = np.divide(single, total)
########################################################################
return preds
def forward(params, inputs=None, hps=None):
hidden_activation = hps['hidden_activation'](np.add(
np.matmul(inputs, params['input']['hidden']['weights']),
params['input']['hidden']['bias']))
output_activation = hps['output_activation'](np.add(
np.matmul(
hidden_activation,
params['hidden']['output']['weights'],
),
params['hidden']['output']['bias'],
))
return [hidden_activation, output_activation]
def forward(model, inputs=None, hps=None):
conv_act = hps['c1_activation'](np.add(
signal.convolve(inputs, model['input']['c1']['weights']),
model['input']['c1']['bias'])).reshape(
inputs.shape[0], -1) # <-- flatten final activations
dense1_act = hps['d1_activation'](np.add(
np.matmul(conv_act, model['c1']['d1']['weights']),
model['c1']['d1']['bias']))
output_act = hps['output_activation'](np.add(
np.matmul(dense1_act, model['d1']['output']['weights']),
model['d1']['output']['bias']))
return output_act
def feed_forward(features, w1, b1, w2, b2, w3, b3):
HL1 = np.matmul(w1, features)
HL1_with_bias = np.add(HL1, b1)
HL1_with_bias_and_activation = np.maximum(HL1_with_bias, np.zeros((4, 1)))
HL2 = np.matmul(w2, HL1_with_bias_and_activation)
HL2_with_bias = np.add(HL2, b2)
HL2_with_bias_and_activation = np.maximum(HL2_with_bias, np.zeros((3, 1)))
targets_predicted = np.matmul(w3, HL2_with_bias_and_activation)
targets_predicted = np.add(targets_predicted, b3)
# Use sigmoid for the output activation
targets_predicted = sigmoid(targets_predicted)
return targets_predicted
def posteriorPriorNatParams( cls, x=None, constParams=None, prior_params=None, prior_nat_params=None, stats=None ):
assert ( prior_params is None ) ^ ( prior_nat_params is None )
assert ( x is None ) ^ ( stats is None )
prior_nat_params = prior_nat_params if prior_nat_params is not None else cls.priorClass.standardToNat( *prior_params )
if( x is not None ):
t1, t2, t3, t4, t5, t6, t7, t8 = cls.sufficientStats( x, constParams=constParams )
transFactor, emissionFactor, initFactor = cls.partitionFactors( x )
stats = [ t1, t2, t3, t4, t5, t6, t7, t8, transFactor, transFactor, emissionFactor, emissionFactor, initFactor, initFactor, initFactor ]
for s, p in zip( stats, prior_nat_params ):
if( isinstance( s, np.ndarray ) ):
assert isinstance( p, np.ndarray )
assert s.shape == p.shape
else:
assert not isinstance( s, np.ndarray ) and not isinstance( p, np.ndarray )
# for s in stats:
# print( s )
# print()
# print()
pnp = [ np.add( s, p ) for s, p in zip( stats, prior_nat_params ) ]
# pp = cls.priorClass.natToStandard( *pnp )
# for p in pp:
# print()
# print( p )
# assert 0
return pnp
def new_position_from_nth_solution_equation(points):
interior_point_trial_solution = points[0:len(self.q_list)]
q_n_plus_1_trial_solution = points[len(self.q_list):2 *
len(self.q_list)]
S_of_n = lambda q_n: self.action(
t, time_step, q_n, interior_point_trial_solution,
q_n_plus_1_trial_solution)
S_of_interior = lambda q_interior: self.action(
t, time_step, self.q_solutions[i], q_interior,
q_n_plus_1_trial_solution)
partial_differential_of_action_wrt_interior_point = egrad(
S_of_interior)
interior_equation = partial_differential_of_action_wrt_interior_point(
interior_point_trial_solution)
partial_differential_of_action_wrt_q_n = egrad(S_of_n)
conservation_equation = np.add(
self.p_solutions[i],
partial_differential_of_action_wrt_q_n(
self.q_solutions[i]))
return np.concatenate(
(interior_equation, conservation_equation))
def get_marginal(self, u, V, R, x_test): ''' current metric to test convergence-- log space predictive marginal likelihood ''' I = self.sigx*np.identity(self.dimx) mu = np.zeros(self.dimx,) n_samples = 200 ll = 0 test_size = x_test.shape[0] for i in xrange(test_size): x = x_test[i] mc = 0 for j in xrange(n_samples): w = self.sample_w(u, V) var = np.dot(w, np.transpose(w)) var = np.add(var, I) px = gaussian.Gaussian_full(mu, var) px = px.eval(x)#eval_log_properly(x) mc = mc + px mc = mc/float(n_samples) mc = np.log(mc) ll += mc return (ll/float(test_size))
def forward_step(params, X=None, cell_state_0=None, hid_state_0=None):
hid_state = np.repeat(hid_state_0,
X.shape[0] - hid_state_0.shape[0] + 1,
axis=0)
cell_state_1 = np.add(
np.multiply( # <-- forget old info
cell_state_0,
sigmoid(
c([X, hid_state]) @ params['forget']['w'] +
params['forget']['b']), # <-- forget gate
),
np.multiply( # <-- write new info
sigmoid(
c([X, hid_state]) @ params['ingate']['w'] +
params['ingate']['b']), # <-- input gate
np.tanh(
c([X, hid_state]) @ params['change']['w'] +
params['change']['b']), # <-- change gate
))
hid_state_1 = np.multiply(
sigmoid(c([X, hid_state]) @ params['outgate']['w']),
# 1,
np.tanh(cell_state_1))
return cell_state_1, hid_state_1
def forward(params, inputs=None, hps=None):
hidden_activation = np.array([
np.exp(-np.matmul(
np.subtract(inputs, params['input']['hidden']['bias'][:, h])**2,
params['input']['hidden']['weights'][:, h],
)) for h in range(params['input']['hidden']['weights'].shape[1])
]).T
channel_activations = hps['channel_activation'](np.add(
np.matmul(
hidden_activation,
params['hidden']['categories']['weights'],
),
params['hidden']['categories']['bias'],
))
## reconstructive error
output_activation = np.sum(np.square(
np.subtract(
inputs,
channel_activations,
)),
axis=2).T
output_activation = 1 - hps['output_activation'](
output_activation / output_activation.sum(axis=1, keepdims=True))
return [hidden_activation, channel_activations, output_activation]
def expectedEmissionStats( self, t, ys, alphas, betas, conditionOnY=True ):
# E[ x_t * x_t^T ], E[ y_t * x_t^T ] and E[ y_t * y_t^T ]
# Find the expected sufficient statistic for N( y_t, x_t | Y )
J_x, h_x, _ = np.add( alphas[ t ], betas[ t ] )
if( conditionOnY == False ):
J11 = self.J1Emiss
J12 = -self._hy
J22 = self.Jy + J_x
D = J11.shape[ 0 ]
J = np.block( [ [ J11, J12 ], [ J12.T, J22 ] ] )
h = np.hstack( ( np.zeros( D ), h_x ) )
# This is a block matrix with block E[ y_t * y_t^T ], E[ y_t * x_t^T ]
# and E[ x_t * x_t^T ]
E, _ = Normal.expectedSufficientStats( nat_params=( -0.5 * J, h ) )
Eyt_yt, Eyt_xt, Ext_xt = toBlocks( E, D )
else:
Ext_xt, E_xt = Normal.expectedSufficientStats( nat_params=( -0.5 * J_x, h_x ) )
Eyt_yt = np.einsum( 'mi,mj->ij', ys[ :, t ], ys[ :, t ] )
Eyt_xt = np.einsum( 'mi,j->ij', ys[ :, t ], E_xt )
return Eyt_yt, Eyt_xt, Ext_xt
def prediction_loss(x, y, W, V, b, c):
# Compute f(x) for all x.
Wx = np.matmul(W, x)
bPlusWx = np.add(b, Wx)
# fx = c + V.tanh(b + W.x)
fx = np.add(c, np.matmul(V, np.tanh(bPlusWx)))
expFx = 0
for fi in fx:
expFx = expFx + np.exp(fi)
# Loss = -f_y(x) + log(sum(exp(f_i(x))))
L = 0 - fx[y] + np.log(expFx)
return L
def evaluate_net(sample, params, max_depth=None):
"""Evalutes sample with network params"""
inputs = sample
for W,b in params[:max_depth]:
outputs = np.add(np.dot(inputs, W), b)
inputs = np.maximum(outputs, 0.0) # ReLU
return inputs # Do I need to do a different operation at the output layer for autoencoder? Need to enforce sparsity too
def forward(params, inputs=None, hps=None):
output_activation = np.add(
np.matmul(
inputs,
params['input']['output']['weights'],
),
params['input']['output']['bias'],
)
return [output_activation]
def f3(x):
a=np.array([1.0,0.0])
b=np.array([0.0,-1.0])
B=np.array([[3.0,-1.0],[-1.0,3.0]])
I=np.array([[1.0,0.0],[0.0,1.0]])
y1=np.exp(-(np.dot(np.subtract(x,a),np.transpose(np.subtract(x,a)))))
y2=np.exp(-(np.dot(np.dot(np.subtract(x,b),B),np.transpose(np.subtract(x,b)))))
y3=(np.log(np.linalg.det(np.add(0.01*I,np.dot(np.transpose(x),x)))))/10.0
#y3=np.log((x[0]*x[0]+0.01)*(x[1]*x[1]+0.01)-x[0]*x[0]*x[1]*x[1])/10.0
return 1.0-(y1+y2-y3)
def new_position_from_nth_solution_equation(
q_n_plus_1_trial_solutions):
S = lambda q_n: self.action(q_n, q_n_plus_1_trial_solutions, t,
time_step)
partial_differential_of_action_wrt_q_n = egrad(S)
equation = np.add(
self.p_solutions[i],
partial_differential_of_action_wrt_q_n(
self.q_solutions[i]))
return equation
def mean_function(self, X):
alpha, mean_lam, gamma, delta = self._get_params(X)
cfg, res, kappa, kr_pref, mean = self._compute_terms(X,
alpha,
mean_lam,
gamma,
delta,
ret_mean=True)
return anp.add(mean, anp.multiply(kappa, kr_pref))
def get_q_dist(self, u, V): ''' returns mean and covariance of q distribution of SEP ''' prec = self.n*V prec = np.add(prec, self.SEP_prior_prec) qsig = np.linalg.inv(prec) qmu = self.n*np.dot(V, u) #TODO: add in prior mean to make general- assumes zero mean prior npw qmu = np.dot(qsig, qmu) return qmu, qsig
def sufficientStats(cls, x, constParams=None):
# Compute T( x )
if (cls.dataN(x) > 1):
t = (0, 0, 0, 0, 0, 0)
for pi_0, pi, L in zip(*x):
t = np.add(t, cls.sufficientStats((pi_0, pi, L)))
return t
t1, t2, t3 = HMMState.standardToNat(*x)
t4, t5, t6 = HMMState.log_partition(params=x, split=True)
return t1, t2, t3, -t4, -t5, -t6
def sufficientStats(cls, x, constParams=None):
# Compute T( x )
if (cls.dataN(x) > 1):
t = (0, 0)
for _x in x:
t = np.add(t, cls.sufficientStats(_x))
return t
(t1, ) = Categorical.standardToNat(x)
(t2, ) = Categorical.log_partition(params=(x, ), split=True)
return t1, t2
def prediction_loss_full(X, Y, W, V, b, c, l):
WX = np.matmul(W, np.transpose(X))
b = np.array([
b,
] * X.shape[0]).transpose()
c = np.array([
c,
] * X.shape[0]).transpose()
b_plus_WX = np.add(b, WX)
sigma = np.tanh(b_plus_WX)
f = np.add(c, np.matmul(V, sigma))
L = 0
for i in range(Y.shape[0]):
softmax = np.log(np.sum(np.exp(f[:, i])))
y = Y[i]
L += -f[y][i] + softmax
L += l * (np.sum(np.square(V)) + np.sum(np.square(W)))
return L
def sufficientStats(cls, x, constParams=None):
# Compute T( x )
if (cls.dataN(x) > 1):
t = (0, 0, 0, 0, 0)
for mu, sigma in zip(*x):
t = np.add(t, cls.sufficientStats((mu, sigma)))
return t
t1, t2 = Normal.standardToNat(*x)
t3, t4, t5 = Normal.log_partition(params=x, split=True)
return t1, t2, -t3, -t4, -t5
def sufficientStats(cls, x, constParams=None):
# Compute T( x )
if (cls.dataN(x) > 1):
t = (0, 0)
for _x in x:
t = np.add(t, cls.sufficientStats(_x))
return t
t1, = Transition.standardToNat(x)
t2, = Transition.log_partition(params=(x, ), split=True)
return t1, -t2
def forward(params, inputs=None, hps=None):
hidden_activation = hps['hidden_activation'](np.add(
np.multiply(
inputs,
params['input']['hidden']['weights'],
),
params['input']['hidden']['bias'],
))
hidden_activation = np.exp(hidden_activation) / np.exp(
hidden_activation).sum()
channel_activations = hps['output_activation'](np.add(
np.matmul(
hidden_activation,
params['hidden']['output']['weights'],
),
params['hidden']['output']['bias'],
))
return [hidden_activation, channel_activations]
def true_marg(self, x_test): ''' returns true predictive marginal likelihood based on generative model params: W ''' I = self.sigx*np.identity(self.dimx) mu = np.zeros(self.dimx,) test_size = x_test.shape[0] test_size = x_test.shape[0] ll = 0 for i in range(test_size): x = x_test[i] var = np.dot(self.W, self.W.T) var = np.add(var, I) px = gaussian.Gaussian_full(mu, var) px = px.eval(x) ll += np.log(px) return (ll/float(test_size))
