def _loss_fn(self, matrix, rels_reversed):
"""Given a numpy array with vectors for u, v and negative samples, computes loss value.
Parameters
----------
matrix : numpy.array
Array containing vectors for u, v and negative samples, of shape (2 + negative_size, dim).
rels_reversed : bool
Returns
-------
float
Computed loss value.
Warnings
--------
Only used for autograd gradients, since autograd requires a specific function signature.
"""
vector_u = matrix[0]
vectors_v = matrix[1:]
if not rels_reversed:
entailment_penalty = grad_np.maximum(0, vector_u - vectors_v) # (1 + negative_size, dim).
else:
entailment_penalty = grad_np.maximum(0, - vector_u + vectors_v) # (1 + negative_size, dim).
energy_vec = grad_np.linalg.norm(entailment_penalty, axis=1) ** 2
positive_term = energy_vec[0]
negative_terms = energy_vec[1:]
return positive_term + grad_np.maximum(0, self.margin - negative_terms).sum()
def get_default_theta(self):
if self.k: # kernel2 MF
# sn2 + (output_scale + lengthscale) + (output_scale + lengthscales) * 2
theta = np.random.randn(3 + 2 * self.dim)
theta[2] = np.maximum(
-100,
np.log(0.5 * (self.train_x[self.dim - 1].max() -
self.train_x[self.dim - 1].min())))
for i in range(self.dim - 1):
tmp = np.maximum(
-100,
np.log(0.5 *
(self.train_x[i].max() - self.train_x[i].min())))
theta[4 + i] = tmp
theta[4 + self.dim + i] = tmp
else: # kernel1 RBF
# sn2 + output_scale + lengthscales
theta = np.random.randn(2 + self.dim)
for i in range(self.dim):
theta[2 + i] = np.maximum(
-100,
np.log(0.5 *
(self.train_x[i].max() - self.train_x[i].min())))
theta[0] = np.log(np.std(self.train_y) + 0.000001) # sn2
return theta
def forward(self, X, WEIGHTS, BIASES):
# Forward passs through the network... input = activation(previous_layer_output)*Weights + Biases
self.X = X
inp_next = X
for i in range(self.N):
output = anp.dot(anp.array(inp_next), WEIGHTS[i]) + anp.array(
[BIASES[i]] * inp_next.shape[0])
activation = self.a_hidden_layers[i]
if (activation == "relu"):
output = anp.maximum(output, 0.0)
elif (activation == "sigmoid"):
output = (1.0) / (1 + e**(-anp.array(output)))
else:
pass
inp_next = output
output = anp.dot(anp.array(inp_next), WEIGHTS[-1]) + anp.array(
[BIASES[-1]] * inp_next.shape[0])
if (self.fit_type == "classification"
): # LAst layer is softmax if classifiaction
output = self.softmax(output)
else:
output = anp.maximum(output, 0.0)
return output
def equations_in_RealPigments(
K,
S,
r,
h,
eps=1e-8,
model='normal'
): ## r is substrate reflectance, h is layer thicness, all of parameters can either be array or scalar values
K = np.maximum(K, eps)
S = np.maximum(S, eps)
a = 1 + K / S
# b=sqrt(a**2-1.0)
b = (a**2 - 1.0)**(1 / 2.0)
if model == 'normal':
d = mycoth(b * S * h)
numerator = 1 - r * (a - b * d)
denumerator = a - r + b * d
R = numerator / denumerator
elif model == 'infinite':
R = a - b
else:
print 'wrong option!'
return R
def logphi(x):
if x**2 < 0.0492:
lp0 = -x / np.sqrt(2 * np.pi)
c = np.array([
0.00048204, -0.00142906, 0.0013200243174, 0.0009461589032,
-0.0045563339802, 0.00556964649138, 0.00125993961762116,
-0.01621575378835404, 0.02629651521057465, -0.001829764677455021,
2 * (1 - np.pi / 3), (4 - np.pi) / 3, 1, 1
])
f = 0
for i in range(14):
f = lp0 * (c[i] + f)
return -2 * f - np.log(2)
elif x < -11.3137:
r = np.array([
1.2753666447299659525, 5.019049726784267463450,
6.1602098531096305441, 7.409740605964741794425,
2.9788656263939928886
])
q = np.array([
2.260528520767326969592, 9.3960340162350541504,
12.048951927855129036034, 17.081440747466004316,
9.608965327192787870698, 3.3690752069827527677
])
num = 0.5641895835477550741
for i in range(5):
num = -x * num / np.sqrt(2) + r[i]
den = 1.0
for i in range(6):
den = -x * den / np.sqrt(2) + q[i]
return np.log(0.5 * np.maximum(0.000001, num / den)) - 0.5 * (x**2)
else:
return np.log(0.5 * np.maximum(0.000001, (1.0 - erf(-x / np.sqrt(2)))))
def __init__(self, layer_sizes, **kwargs):
# set default values for layer sizes, activation, and scale
activation = 'relu'
# decide on these parameters via user input
if 'activation' in kwargs:
activation = kwargs['activation']
# switches
if activation == 'linear':
self.activation = lambda data: data
elif activation == 'tanh':
self.activation = lambda data: np.tanh(data)
elif activation == 'relu':
self.activation = lambda data: np.maximum(0, data)
elif activation == 'sinc':
self.activation = lambda data: np.sinc(data)
elif activation == 'sin':
self.activation = lambda data: np.sin(data)
elif activation == 'maxout':
self.activation = lambda data1, data2: np.maximum(data1, data2)
# select layer sizes and scale
self.layer_sizes = layer_sizes
self.scale = 0.1
if 'scale' in kwargs:
self.scale = kwargs['scale']
# assign initializer / feature transforms function
if activation == 'linear' or activation == 'tanh' or activation == 'relu' or activation == 'sinc' or activation == 'sin':
self.initializer = self.standard_initializer
self.feature_transforms = self.standard_feature_transforms
elif activation == 'maxout':
self.initializer = self.maxout_initializer
self.feature_transforms = self.maxout_feature_transforms
def __init__(self, **kwargs):
if 'layer_sizes' in kwargs:
self.layer_sizes = kwargs['layer_sizes'] # Set layer sizes
else: # Else create default setup
N = 1 # Input dimensions
M = 1 # Output dimensions
U = 10 # 10-unit hidden layer
self.layer_sizes = [
N, U, M
] # Build layer sizes to generate weight matrix
if 'scale' in kwargs:
self.scale = kwargs['scale'] # Set scale
else:
self.scale = 0.1
a = 'relu' # Set default activation to ReLU
if 'activation' in kwargs:
a = kwargs['activation'] # Manually set activation if present
self.activation_name = a
if a == 'relu':
self.activation = lambda data: np.maximum(0, data)
elif a == 'tanh':
self.activation = lambda data: np.tanh(data)
elif a == 'maxout':
self.activation = lambda data1, data2: np.maximum(data1, data2)
self.weight_matrix = self.maxout_init_weights
self.transforms = self.maxout_feature_transforms
else: # Manual activation
self.activation = kwargs['activation']
if a in ['relu', 'tanh']:
self.weight_matrix = self.init_weights
self.transforms = self.feature_transforms
def forward_pass(W1, W2, W3, b1, b2, b3, x):
"""
forward-pass for an fully connected neural network with 2 hidden layers of M neurons
Inputs:
W1 : (M, 784) weights of first (hidden) layer
W2 : (M, M) weights of second (hidden) layer
W3 : (10, M) weights of third (output) layer
b1 : (M, 1) biases of first (hidden) layer
b2 : (M, 1) biases of second (hidden) layer
b3 : (10, 1) biases of third (output) layer
x : (N, 784) training inputs
Outputs:
Fhat : (N, 10) output of the neural network at training inputs
"""
H1 = np.maximum(0, np.dot(x, W1.T) + b1.T) # layer 1 neurons with ReLU activation, shape (N, M)
H2 = np.maximum(0, np.dot(H1, W2.T) + b2.T) # layer 2 neurons with ReLU activation, shape (N, M)
Fhat = np.dot(H2, W3.T) + b3.T # layer 3 (output) neurons with linear activation, shape (N, 10)
a = np.max(Fhat, axis=1)
# expand the value of a into a matrix to compute log-exp trick easy
A = -1*np.ones(np.shape(Fhat))*a[:, np.newaxis]
log_sum = np.log(np.sum(np.exp(np.add(Fhat, A)), axis=1))
# turn sums into a matrix as well
Log_Sum = -1*np.ones(np.shape(Fhat))*log_sum[:, np.newaxis]
# log-exp trick on each element of Fhat
Fhat = np.add(np.add(Fhat, A), Log_Sum)
return Fhat
def rand_x(self, n=1):
tmp = np.random.uniform(0, 1, (n))
idx = (tmp < 0.4)
x = np.random.uniform(-0.5, 0.5, (self.dim, n))
x[:, idx] = (0.05 * np.random.uniform(-0.5, 0.5,
(self.dim, idx.sum())).T +
self.best_x[1]).T
x[:, idx] = np.maximum(-0.5, np.minimum(0.5, x[:, idx]))
idx = (tmp < 0.5) * (tmp > 0.4)
x[:, idx] = (0.05 * np.random.uniform(-0.5, 0.5,
(self.dim, idx.sum())).T +
self.best_x[0]).T
x[:, idx] = np.maximum(-0.5, np.minimum(0.5, x[:, idx]))
idx = (tmp > 0.5) * (tmp < 0.6)
num = idx.sum()
num_seed = np.minimum(self.dataset['high_y'].shape[1], 5)
idx = np.argsort(idx)[-num:]
idx1 = np.random.randint(0, self.dim, num)
idx2 = np.random.randint(0, num_seed, num)
idx3 = np.random.randint(0, num_seed, num)
for i in range(num):
while idx2[i] == idx3[i]:
idx3[i] = random.randint(0, num_seed)
x[:idx1[i], i] = self.dataset['high_x'][:idx1[i], -idx2[i]]
x[idx1[i]:, i] = self.dataset['high_x'][idx1[i]:, -idx3[i]]
return x
def rand_theta(self, scale):
if self.k: # kernel2
# sn2 + (output_scale + lengthscale) + (output_scale + lengthscales) * 2
# 1 + 2 + (1 + self.dim - 1)*2 = 3 + 2*self.dim
theta = scale * np.random.randn(3 + 2 * self.dim)
theta[2] = np.maximum(
-100,
np.log(0.5 * (self.train_x[self.dim - 1].max() -
self.train_x[self.dim - 1].min())))
for i in range(self.dim - 1):
tmp = np.maximum(
-100,
np.log(0.5 *
(self.train_x[i].max() - self.train_x[i].min())))
theta[4 + i] = tmp
theta[4 + self.dim + i] = tmp
else: # kernel1 RBF
# sn2 + output_scale + lengthscales, 1 + 1 + self.dim
theta = scale * np.random.randn(2 + self.dim)
for i in range(self.dim):
theta[2 + i] = np.maximum(
-100,
np.log(0.5 *
(self.train_x[i].max() - self.train_x[i].min())))
theta[0] = np.log(np.std(self.train_y)) # sn2
# theta[1] = np.log(np.std(self.train_y))
return theta
def forward_pass(W1, W2, W3, b1, b2, b3, x):
"""
forward-pass for an fully connected neural network with 2 hidden layers of M neurons
Inputs:
W1 : (M, 784) weights of first (hidden) layer
W2 : (M, M) weights of second (hidden) layer
W3 : (10, M) weights of third (output) layer
b1 : (M, 1) biases of first (hidden) layer
b2 : (M, 1) biases of second (hidden) layer
b3 : (10, 1) biases of third (output) layer
x : (N, 784) training inputs
Outputs:
Fhat : (N, 10) output of the neural network at training inputs
"""
H1 = np.maximum(0,
np.dot(x, W1.T) +
b1.T) # layer 1 neurons with ReLU activation, shape (N, M)
H2 = np.maximum(0,
np.dot(H1, W2.T) +
b2.T) # layer 2 neurons with ReLU activation, shape (N, M)
Fhat = np.dot(
H2, W3.T
) + b3.T # layer 3 (output) neurons with linear activation, shape (N, 10)
# #######
# Note that the activation function at the output layer is linear!
# You must impliment a stable log-softmax activation function at the ouput layer
# #######
return Fhat
def rand_theta(self, scale):
theta = scale * np.random.randn(self.num_param)
theta[0] = 1.0
theta[1] = np.log(np.std(self.low_y))
theta[2] = np.log(np.std(self.high_y))
for i in range(self.dim):
theta[4+i] = np.maximum(-100, np.log(0.5*(self.low_x[i].max() - self.low_x[i].min())))
theta[5+self.dim+i] = np.maximum(-100, np.log(0.5*(self.high_x[i].max() - self.high_x[i].min())))
return theta
def odeInt(f, x0, y0, xT):
# 1993 Solving Ordinary Differential Equations I, page 169
# initial step size
f0 = f(x0, y0)
d0 = rmsNorm(y0)
d1 = rmsNorm(f0)
if d0 < 1e-5 or d1 < 1e-5:
h0 = 1e-6
else:
h0 = 1e-2 * d0 / d1
y1 = y0 + f0 * h0
f1 = f(x0 + h0, y1)
d2 = rmsNorm(f1 - f0) / h0
maxD = np.maximum(d1, d2)
if maxD <= 1e-15:
h1 = np.maximum(1e-6, h0 * 1e-3)
else:
h1 = np.power(1e-2 / maxD, 1 / (P + 1))
step = np.minimum(1e2 * h0, h1)
# integrate
x = x0
y = y0
k1 = f0
while x < xT:
rejected = False
accepted = False
while not accepted:
ks, y1, y1h = odeIntStep(step, f, x, y, k1)
xNew = x + step
scale = ABS_TOL + np.maximum(np.abs(y1), np.abs(y1h)) * REL_TOL
errNorm = rmsNorm((y1 - y1h) / scale)
if errNorm < 1:
accepted = True
if errNorm == 0:
updateFactor = MAX_UPDATE_FACTOR
else:
updateFactor = np.minimum(
MAX_UPDATE_FACTOR,
SAFETY_FACTOR * np.power(errNorm, ERROR_EXP))
if rejected:
updateFactor = np.minimum(1, updateFactor)
step *= updateFactor
else:
rejected = True
updateFactor = np.maximum(
MIN_UPDATE_FACTOR,
SAFETY_FACTOR * np.power(errNorm, ERROR_EXP))
step *= updateFactor
# interpolate
# update
x = xNew
y = y1
k1 = ks[6]
step = xT - x
ks, y1, y1h = odeIntStep(step, f, x, y, k1)
return y1
def test_minmax(): grad_test(lambda x: ti.min(x, 0), lambda x: np.minimum(x, 0)) grad_test(lambda x: ti.min(x, 1), lambda x: np.minimum(x, 1)) grad_test(lambda x: ti.min(0, x), lambda x: np.minimum(0, x)) grad_test(lambda x: ti.min(1, x), lambda x: np.minimum(1, x)) grad_test(lambda x: ti.max(x, 0), lambda x: np.maximum(x, 0)) grad_test(lambda x: ti.max(x, 1), lambda x: np.maximum(x, 1)) grad_test(lambda x: ti.max(0, x), lambda x: np.maximum(0, x)) grad_test(lambda x: ti.max(1, x), lambda x: np.maximum(1, x))
def compute_xnew(inputs, lambda_):
x, dc, dv = unpack(inputs)
# avoid dividing by zero outside the design region
dv = np.where(np.ravel(args['mask']) > 0, dv, 1)
# square root is not defined for negative numbers, which can happen due to
# small numerical errors in the computed gradients.
xnew = x * np.maximum(-dc / (lambda_ * dv), 0) ** eta
lower = np.maximum(0.0, x - max_move)
upper = np.minimum(1.0, x + max_move)
# note: autograd does not define gradients for np.clip
return np.minimum(np.maximum(xnew, lower), upper)
def invert(self, data, input=None, mask=None, tag=None):
yhat = smooth(data,20)
xhat = self.link(np.clip(yhat, 0.01, np.inf))
xhat = self._invert(xhat, input=input, mask=mask, tag=tag)
for t in range(xhat.shape[0]):
if np.all(xhat[np.max([0,t-2]):t+3]>0.99) and t>2:
xhat[np.minimum(0,t-2):] = 1.01*np.ones(np.shape(xhat[np.minimum(0,t-2):]))
xhat[:np.maximum(0,t-2)] = np.clip(xhat[:np.maximum(0,t-2)], -0.5,0.95)
if np.abs(xhat[0])>1.0:
xhat[0] = 0.5 + 0.01*npr.randn(1,np.shape(xhat)[1])
return xhat
def _loss_fn(self, matrix, rels_reversed):
"""Given a numpy array with vectors for u, v and negative samples, computes loss value.
Parameters
----------
matrix : numpy.array
Array containing vectors for u, v and negative samples, of shape (2 + negative_size, dim).
rels_reversed : bool
Returns
-------
float
Computed loss value.
Warnings
--------
Only used for autograd gradients, since autograd requires a specific function signature.
"""
vector_u = matrix[0]
vectors_v = matrix[1:]
norm_u = grad_np.linalg.norm(vector_u)
norms_v = grad_np.linalg.norm(vectors_v, axis=1)
euclidean_dists = grad_np.linalg.norm(vector_u - vectors_v, axis=1)
dot_prod = (vector_u * vectors_v).sum(axis=1)
if not rels_reversed:
# u is x , v is y
cos_angle_child = (dot_prod * (1 + norm_u ** 2) - norm_u ** 2 * (1 + norms_v ** 2)) /\
(norm_u * euclidean_dists * grad_np.sqrt(1 + norms_v ** 2 * norm_u ** 2 - 2 * dot_prod))
angles_psi_parent = grad_np.arcsin(self.K * (1 - norm_u**2) /
norm_u) # scalar
else:
# v is x , u is y
cos_angle_child = (dot_prod * (1 + norms_v ** 2) - norms_v **2 * (1 + norm_u ** 2) ) /\
(norms_v * euclidean_dists * grad_np.sqrt(1 + norms_v**2 * norm_u**2 - 2 * dot_prod))
angles_psi_parent = grad_np.arcsin(self.K * (1 - norms_v**2) /
norms_v) # 1 + neg_size
# To avoid numerical errors
clipped_cos_angle_child = grad_np.maximum(cos_angle_child, -1 + EPS)
clipped_cos_angle_child = grad_np.minimum(clipped_cos_angle_child,
1 - EPS)
angles_child = grad_np.arccos(clipped_cos_angle_child) # 1 + neg_size
energy_vec = grad_np.maximum(0, angles_child - angles_psi_parent)
positive_term = energy_vec[0]
negative_terms = energy_vec[1:]
return positive_term + grad_np.maximum(
0, self.margin - negative_terms).sum()
def IoG(self, box):
inter_xmin = np.maximum(box[:, 0], self.IoG_gt[:, 0])
inter_ymin = np.maximum(box[:, 1], self.IoG_gt[:, 1])
inter_xmax = np.minimum(box[:, 2], self.IoG_gt[:, 2])
inter_ymax = np.minimum(box[:, 3], self.IoG_gt[:, 3])
Iw = np.clip(inter_xmax - inter_xmin, 0, 5)
Ih = np.clip(inter_ymax - inter_ymin, 0, 5)
I = Iw * Ih
G = (self.IoG_gt[:, 2] - self.IoG_gt[:, 0]) * (self.IoG_gt[:, 3] -
self.IoG_gt[:, 1])
iog = I / G
smln = smoothln(iog)
n_p = float(I.shape[0])
return smln.sum() / n_p
def feed_forward(features, w1, b1, w2, b2, w3, b3):
HL1 = np.matmul(w1, features)
HL1_with_bias = np.add(HL1, b1)
HL1_with_bias_and_activation = np.maximum(HL1_with_bias, np.zeros((4, 1)))
HL2 = np.matmul(w2, HL1_with_bias_and_activation)
HL2_with_bias = np.add(HL2, b2)
HL2_with_bias_and_activation = np.maximum(HL2_with_bias, np.zeros((3, 1)))
targets_predicted = np.matmul(w3, HL2_with_bias_and_activation)
targets_predicted = np.add(targets_predicted, b3)
# Use sigmoid for the output activation
targets_predicted = sigmoid(targets_predicted)
return targets_predicted
def to_safe_common_arr(topics_KV, min_eps=MIN_EPS):
''' Force provided topics_KV array to be numerically safe.
Returns
-------
topics_KV : 2D array, size K x V
minimum value of each row is min_eps
each row will sum to 1.0 (+/- min_eps)
'''
K, V = topics_KV.shape
topics_KV = topics_KV.copy()
for rep in range(2):
topics_KV /= topics_KV.sum(axis=1)[:, np.newaxis]
np.maximum(topics_KV, min_eps, out=topics_KV)
return topics_KV
def integrate(self, t, *args, **kwargs):
"""
This integral is a simple linear interpolant,
which I would like to do as a spline.
However, I need to do it manually, since
it needs to be autograd differentiable, which splines are not.
The method here is not especially efficent.
"""
tau = self.get_param('tau', **kwargs)
kappa = self.get_param('kappa', **kwargs)
mu = self.get_param('mu', 0.0, **kwargs)
f_kappa = self.f_kappa(kappa=kappa, mu=mu)
t = np.reshape(t, (-1, 1))
delta = np.diff(tau)
each = np.maximum(
0, (t - tau[:-1].reshape(1, -1))
)
each = np.minimum(
each,
delta.reshape(1, -1)
)
return np.sum(
each * np.reshape(f_kappa, (1, -1)),
1
) + (mu * t.ravel())
def IoU(self, box):
inter_xmin = np.maximum(box[:, 0], self.IoU_const[:, 0])
inter_ymin = np.maximum(box[:, 1], self.IoU_const[:, 1])
inter_xmax = np.minimum(box[:, 2], self.IoU_const[:, 2])
inter_ymax = np.minimum(box[:, 3], self.IoU_const[:, 3])
Iw = np.clip(inter_xmax - inter_xmin, 0, 5)
Ih = np.clip(inter_ymax - inter_ymin, 0, 5)
I = Iw * Ih
A1 = (self.box[:, 2] - self.box[:, 0]) * (self.box[:, 3] -
self.box[:, 1])
A2 = (self.IoU_const[:, 2] - self.IoU_const[:, 0]) * (
self.IoU_const[:, 3] - self.IoU_const[:, 1])
iou = I / (A1 + A2 - I)
smln = smoothln(iog)
n_p = float(I.shape[0])
return smln.sum()
def _composite_log_likelihood(data,
demo,
mut_rate=None,
truncate_probs=0.0,
vector=False,
p_missing=None,
use_pairwise_diffs=False,
**kwargs):
try:
sfs = data.sfs
except AttributeError:
sfs = data
sfs_probs = np.maximum(
expected_sfs(demo, sfs.configs, normalized=True, **kwargs),
truncate_probs)
log_lik = sfs._integrate_sfs(np.log(sfs_probs), vector=vector)
# add on log likelihood of poisson distribution for total number of SNPs
if mut_rate is not None:
log_lik = log_lik + \
_mut_factor(sfs, demo, mut_rate, vector,
p_missing, use_pairwise_diffs)
if not vector:
log_lik = np.squeeze(log_lik)
return log_lik
def __init__(self, **kwargs):
# set default values for layer sizes, activation, and scale
activation = 'relu'
# decide on these parameters via user input
if 'activation' in kwargs:
activation = kwargs['activation']
# switches
if activation == 'linear':
self.activation = lambda data: data
elif activation == 'tanh':
self.activation = lambda data: np.tanh(data)
elif activation == 'relu':
self.activation = lambda data: np.maximum(0, data)
elif activation == 'sinc':
self.activation = lambda data: np.sinc(data)
elif activation == 'sin':
self.activation = lambda data: np.sin(data)
else: # user-defined activation
self.activation = kwargs['activation']
# select layer sizes and scale
N = 1
M = 1
U = 10
self.layer_sizes = [N, U, M]
self.scale = 0.1
if 'layer_sizes' in kwargs:
self.layer_sizes = kwargs['layer_sizes']
if 'scale' in kwargs:
self.scale = kwargs['scale']
def log_logist(theta, Z, nu, mu, Sig):
dots = -np.dot(Z, theta.T)
log_lik = -np.sum(np.maximum(dots, 0) + np.log1p(np.exp(-np.abs(dots))),
axis=0)
log_pri = log_multivariate_t(theta[:, :-1], mu, Sig, nu) + log_cauchy(
theta[:, -1])
return log_pri + log_lik
def make_positive_definite(m, tol=None):
''' Computes a matrix close to the original matrix m that is positive definite.
This function is just a transcript of R' make.positive.definite function.
m (2d array): A matrix that is not necessary psd.
tol (int): A tolerence level controlling how "different" the psd matrice
can be from the original matrix
---------------------------------------------------------------
returns (2d array): A psd matrix
'''
d = m.shape[0]
if (m.shape[1] != d):
raise RuntimeError("Input matrix is not square!")
eigvalues, eigvect = eigh(m)
# Sort the eigen values
idx = eigvalues.argsort()[::-1]
eigvalues = eigvalues[idx]
eigvect = eigvect[:, idx]
if (tol == None):
tol = d * np.max(np.abs(eigvalues)) * sys.float_info.epsilon
delta = 2 * tol
tau = np.maximum(0, delta - eigvalues)
dm = multi_dot([eigvect, np.diag(tau), eigvect.T])
return (m + dm)
def log_likelihood(self, theta):
sn2, sp2, log_lscale, w = self.split_theta(theta)
scaled_x = scale_x(log_lscale, self.train_x)
Phi = self.calc_Phi(w, scaled_x)
Phi_y = np.dot(Phi, self.train_y.T)
sp2 = np.maximum(sp2, 0.000001)
A = np.dot(Phi, Phi.T) + self.m * sn2 / sp2 * np.eye(
self.m) # A.shape: self.m, self.m
LA = np.linalg.cholesky(A)
logDetA = 0
for i in range(self.m):
logDetA = 2 * np.log(LA[i][i])
datafit = (np.dot(self.train_y, self.train_y.T) -
np.dot(Phi_y.T, chol_inv(LA, Phi_y))) / sn2
neg_likelihood = 0.5 * (datafit +
self.num_train * np.log(2 * np.pi * sn2) +
logDetA - self.m * np.log(self.m * sn2 / sp2))
if (np.isnan(neg_likelihood)):
neg_likelihood = np.inf
w_nobias = self.nn.w_nobias(w, self.dim)
l1_reg = self.l1 * np.abs(w_nobias).sum()
l2_reg = self.l2 * np.dot(w_nobias, w_nobias.T)
neg_likelihood += l1_reg + l2_reg
if neg_likelihood < self.loss:
self.loss = neg_likelihood
self.theta = theta.copy()
self.A = A.copy()
self.LA = LA.copy()
return neg_likelihood
def conditional_probability_alive(self, frequency, recency, T):
"""
Compute conditional probability alive.
Compute the probability that a customer with history
(frequency, recency, T) is currently alive.
From http://www.brucehardie.com/notes/021/palive_for_BGNBD.pdf
Parameters
----------
frequency: array or scalar
historical frequency of customer.
recency: array or scalar
historical recency of customer.
T: array or scalar
age of the customer.
Returns
-------
array
value representing a probability
"""
r, alpha, a, b = self._unload_params("r", "alpha", "a", "b")
log_div = (r + frequency) * np.log(
(alpha + T) /
(alpha + recency)) + np.log(a / (b + np.maximum(frequency, 1) - 1))
return np.atleast_1d(np.where(frequency == 0, 1.0, expit(-log_div)))
def _negative_log_likelihood(log_params, freq, rec, T, weights,
penalizer_coef):
"""
The following method for calculatating the *log-likelihood* uses the method
specified in section 7 of [2]_. More information can also be found in [3]_.
References
----------
.. [2] Fader, Peter S., Bruce G.S. Hardie, and Ka Lok Lee (2005a),
"Counting Your Customers the Easy Way: An Alternative to the
Pareto/NBD Model," Marketing Science, 24 (2), 275-84.
.. [3] http://brucehardie.com/notes/004/
"""
warnings.simplefilter(action="ignore", category=FutureWarning)
params = np.exp(log_params)
r, alpha, a, b = params
A_1 = gammaln(r + freq) - gammaln(r) + r * np.log(alpha)
A_2 = gammaln(a + b) + gammaln(b + freq) - gammaln(b) - gammaln(a + b +
freq)
A_3 = -(r + freq) * np.log(alpha + T)
A_4 = np.log(a) - np.log(b + np.maximum(freq, 1) -
1) - (r + freq) * np.log(rec + alpha)
penalizer_term = penalizer_coef * sum(params**2)
ll = weights * (A_1 + A_2 +
np.log(np.exp(A_3) + np.exp(A_4) * (freq > 0)))
return -ll.sum() / weights.sum() + penalizer_term
def predict(samples_q, X):
# First layer
K = samples_q.shape[0]
# print('shapes', list(shapes))
(m, n) = shapes[0]
# print('m={},n={},k={}'.format(m,n,K))
# print('samples_q shape', samples_q.shape)
# print('-------------------after shape------------')
# samples_q[ : , : m * n ]
# print('samples_q shape', samples_q[ : , : m * n ].shape)
# print('-------------------after samples_q------------')
# samples_q[ : , : m * n ].reshape((n * K, m))
# print('-------------------after samples_q reshape------------')
W = samples_q[:, :m * n].reshape((n * K, m)).T
# print('-------------------after W------------')
b = samples_q[:, m * n:m * n + n].reshape((1, n * K))
# print('-------------------after b------------')
a = np.dot(X, W) + b
h = np.maximum(a, 0)
# Second layer
samples_q = samples_q[:, m * n + n:]
(m, n) = shapes[1]
b = samples_q[:, m * n:m * n + n].T
a = np.sum((samples_q[:, :m * n].reshape((1, -1)) * h).reshape(
(K * X.shape[0], m)), 1).reshape((X.shape[0], K)) + b
return a
def ye_limit(x, trackwidth):
k = TrackCurvature(x)
N = len(k)
lowlimit = -trackwidth/2 * np.ones(N)
highlimit = trackwidth/2 * np.ones(N)
# use a 5% margin so we can't actually hit the center of curvature
lowlimit[k < 0] = np.maximum(0.95/k[k < 0], -trackwidth/2)
highlimit[k > 0] = np.minimum(0.95/k[k > 0], trackwidth/2)
return lowlimit, highlimit
def projectLB(v, lb):
""" project vector v onto constraint v >= lb, used for nonnegative
constraint
Parameter
---------
v: shape(nVars,)
input vector
lb: float
lower bound
Return
------
w: shape(nVars,)
projection of v to constraint v >= lb
"""
return np.maximum(v, lb)
def projectSimplex(mat):
""" project each row vector to the simplex
"""
nPoints, nVars = mat.shape
mu = np.fliplr(np.sort(mat, axis=1))
sum_hist = np.cumsum(mu, axis=1)
flag = (mu - 1./np.tile(np.arange(1,nVars+1),(nPoints,1))*(sum_hist-1) > 0)
f_flag = lambda flagPoint: len(flagPoint) - 1 - \
flagPoint[::-1].argmax()
lastTrue = map(f_flag, flag)
sm_row = sum_hist[np.arange(nPoints), lastTrue]
theta = (sm_row - 1)*1./(np.array(lastTrue)+1.)
w = np.maximum(mat - np.tile(theta, (nVars,1)).T, 0.)
return w
def predict(samples_q, X):
# First layer
K = samples_q.shape[ 0 ]
(m, n) = shapes[ 0 ]
W = samples_q[ : , : m * n ].reshape(n * K, m).T
b = samples_q[ : , m * n : m * n + n ].reshape(1, n * K)
a = np.dot(X, W) + b
h = np.maximum(a, 0)
# Second layer
samples_q = samples_q[ : , m * n + n : ]
(m, n) = shapes[ 1 ]
b = samples_q[ : , m * n : m * n + n ].T
a = np.sum((samples_q[ : , : m * n ].reshape(1, -1) * h).reshape((K * X.shape[ 0 ], m)), 1).reshape((X.shape[ 0 ], K)) + b
return a
def fit_gaussian_draw(X, J, seed=28, reg=1e-7, eig_pow=1.0):
"""
Fit a multivariate normal to the data X (n x d) and draw J points
from the fit.
- reg: regularizer to use with the covariance matrix
- eig_pow: raise eigenvalues of the covariance matrix to this power to construct
a new covariance matrix before drawing samples. Useful to shrink the spread
of the variance.
"""
with NumpySeedContext(seed=seed):
d = X.shape[1]
mean_x = np.mean(X, 0)
cov_x = np.cov(X.T)
if d==1:
cov_x = np.array([[cov_x]])
[evals, evecs] = np.linalg.eig(cov_x)
evals = np.maximum(0, np.real(evals))
assert np.all(np.isfinite(evals))
evecs = np.real(evecs)
shrunk_cov = evecs.dot(np.diag(evals**eig_pow)).dot(evecs.T) + reg*np.eye(d)
V = np.random.multivariate_normal(mean_x, shrunk_cov, J)
return V
def projectSimplex_vec(v):
""" project vector v onto the probability simplex
Parameter
---------
v: shape(nVars,)
input vector
Returns
-------
w: shape(nVars,)
projection of v onto the probability simplex
"""
nVars = v.shape[0]
mu = np.sort(v,kind='quicksort')[::-1]
sm_hist = np.cumsum(mu)
flag = (mu - 1./np.arange(1,nVars+1)*(sm_hist-1) > 0)
lastTrue = len(flag) - 1 - flag[::-1].argmax()
sm_row = sm_hist[lastTrue]
theta = 1./(lastTrue+1) * (sm_row - 1)
w = np.maximum(v-theta, 0.)
return w
def relu(x): return np.maximum(0, x) def init_net_params(scale, layer_sizes, rs=npr.RandomState(0)):
def relu(z):
return np.maximum(0, z)
def relu(x):
return np.maximum(x, 0.0)
def FFNN(X, P):
H = np.dot(X,P[0]) + P[1]
H = np.maximum(H, H*0.01)
return np.dot(H, P[2])
def NN_Obj(W): # shallow net of the form : max( X^T W[:-1,] )^T W[-1,]
return 0.001*np.linalg.norm(np.dot(np.maximum( np.dot( X, W[:-1,] ), 0 ) , W[-1,]) - Y, 2) ** 2; # MSE
def relu(x):
return _np.maximum(0, x)
def fun(x): return to_scalar(np.maximum(
np.array( [x, x, 0.5]),
np.array([[x, 0.5, x],
[x, x, 0.5]])))
d_fun = lambda x : to_scalar(grad(fun)(x))
def fun(x): return to_scalar(np.maximum(x, x)) d_fun = lambda x : to_scalar(grad(fun)(x))
def relu(x):
np.maximum(0, x)
D = 1
rs = npr.RandomState(0)
inputs = np.concatenate([np.linspace(0, 2, num=n_data/2),
np.linspace(6, 8, num=n_data/2)])
targets = np.cos(inputs) + rs.randn(n_data) * noise_std
inputs = (inputs - 4.0) / 4.0
inputs = inputs.reshape((len(inputs), D))
targets = targets.reshape((len(targets), D))
return inputs, targets
if __name__ == '__main__':
# Specify inference problem by its unnormalized log-posterior.
rbf = lambda x: np.exp(-x**2)
relu = lambda x: np.maximum(x, 0.)
num_weights, predictions, logprob = \
make_nn_funs(layer_sizes=[1, 20, 20, 1], L2_reg=0.1,
noise_variance=0.01, nonlinearity=rbf)
inputs, targets = build_toy_dataset()
log_posterior = lambda weights, t: logprob(weights, inputs, targets)
# Build variational objective.
objective, gradient, unpack_params = \
black_box_variational_inference(log_posterior, num_weights,
num_samples=20)
# Set up figure.
fig = plt.figure(figsize=(12, 8), facecolor='white')
ax = fig.add_subplot(111, frameon=False)
def relu(v):
return np.maximum(v, 0)
def relu(x): return np.maximum(0, x) def sigmoid(x): return 0.5 * (np.tanh(x) + 1.0)
def forward_pass(self, data, params):
return np.maximum(data, np.zeros(data.shape))
def nn_predict(inputs, t, params):
for W, b in params:
outputs = np.dot(inputs, W) + b
inputs = np.maximum(0, outputs)
return outputs
