def rank_2_H_update(H, s, y, d):
Hy = np.matmul(H, y)
temp = np.outer(s - Hy, d)
ddT = np.outer(d, d)
dTy = np.inner(d, y)
return H + (temp + np.transpose(temp)) / dTy - np.inner(
y, s - Hy) * ddT / (dTy**2)
def rank_2_H_update(H,s,y,d):
Hy = np.matmul(H,y)
temp = np.outer(s-Hy,d)
ddT = np.outer(d,d)
dTy = np.inner(d,y)
if dTy**2 == 0:
raise ZeroDivisionError
return H + (temp + np.transpose(temp))/dTy - np.inner(y,s-Hy)*ddT/(dTy**2)
def test_inner_product(self):
x = self.manifold.random_point() / 2
u = self.manifold.random_tangent_vector(x)
v = self.manifold.random_tangent_vector(x)
np_testing.assert_allclose(
(2 / (1 - np.linalg.norm(x) ** 2)) ** 2 * np.inner(u, v),
self.manifold.inner_product(x, u, v),
)
# Test that angles are preserved.
x = self.manifold.random_point() / 2
u = self.manifold.random_tangent_vector(x)
v = self.manifold.random_tangent_vector(x)
cos_eangle = np.sum(u * v) / np.linalg.norm(u) / np.linalg.norm(v)
cos_rangle = (
self.manifold.inner_product(x, u, v)
/ self.manifold.norm(x, u)
/ self.manifold.norm(x, v)
)
np_testing.assert_allclose(cos_rangle, cos_eangle)
# Test symmetry.
np_testing.assert_allclose(
self.manifold.inner_product(x, u, v),
self.manifold.inner_product(x, v, u),
)
def h1_mean_var(X, Y, k, l):
"""
X: nxd numpy array
Y: nxd numpy array
k, l: a Kernel object
return (HSIC, var[HSIC]) under H1
"""
m = X.shape[0]
n = Y.shape[0]
if m != n:
raise ValueError('length X should be the same as length Y')
K = k.eval(X, X)
L = l.eval(Y, Y)
Kmean = np.mean(K, 0)
Lmean = np.mean(L, 0)
HK = K - Kmean
HL = L - Lmean
# t = trace(KHLH)
HKf = HK.flatten() / (n - 1)
HLf = HL.T.flatten() / (n - 1)
hsic = HKf.dot(HLf)
#hsic = np.sum(centering(KX) * centering(KY)) / m**2
# variance computation
var = 0
for i in range(m):
var1 = np.inner(K[i, :], L[i, :]) # inner product between vectors
var2 = (L.sum() * K[i, :]).sum()
var3 = np.outer(K[i, :], L[i, :]).sum()
var += ((var1 / m + var2 / (m**3) - 2 * var3 / (m**2))**2) / m
hsic_var = max(16 * (var - hsic**2), 1e-3)
# variance computation 2
#KX = KX - np.diag(KX); KY = KY - np.diag(KY)
#KXKY = KX @ KY; KYKX = KY @ KX
#ones = np.ones(m)
#h1 = (m-2)**2 * (np.multiply(KX,KY) @ ones)
#h2 = (m-2) * (np.trace(KXKY) - KXKY @ ones - KYKX @ ones )
#print(h2)
#h3 = m * np.multiply(KX @ ones, KY @ ones)
#print(h3)
#h4 = (ones @ KY @ ones) * (KX @ ones)
#print(h4)
#h5 = (ones @ KX @ ones) * (KY @ ones)
#print(h5)
#h6 = (ones @ KXKY @ ones)
#h = h1 + h2 -h3 + h4 + h5 - h6
#print(h @ h / (4*m*((m-1)*(m-2)*(m-3))**2))
return hsic, hsic_var
def mvnlogpdf(x, mu, L):
"""
not really logpdf. we need to use the weights
to keep track of normalizing factors that differ
across clusters
L cholesky decomposition of covariance matrix
"""
D = L.shape[0]
logdet = 2 * np.sum(np.log(np.diagonal(L)))
quad = np.inner(x - mu, solve(L.T, solve(L, (x - mu))))
return -0.5 *(D * np.log(2 * np.pi) + logdet + quad)
def test_inner(self):
x = self.man.rand() / 2
u = self.man.randvec(x)
v = self.man.randvec(x)
np_testing.assert_allclose(
(2 / (1 - la.norm(x)**2))**2 * np.inner(u, v),
self.man.inner(x, u, v),
)
# test that angles are preserved
x = self.man.rand() / 2
u = self.man.randvec(x)
v = self.man.randvec(x)
cos_eangle = np.sum(u * v) / la.norm(u) / la.norm(v)
cos_rangle = (self.man.inner(x, u, v) / self.man.norm(x, u) /
self.man.norm(x, v))
np_testing.assert_allclose(cos_rangle, cos_eangle)
elif isinstance(axis, tuple):
for ax in sorted(axis):
ans = anp.expand_dims(ans, ax)
chosen_locations = x == ans
return anp.sum(g * chosen_locations, axis=axis, keepdims=keepdims)
anp.max.defjvp(fwd_grad_chooser)
anp.min.defjvp(fwd_grad_chooser)
anp.amax.defjvp(fwd_grad_chooser)
anp.amin.defjvp(fwd_grad_chooser)
anp.cumsum.defjvp(
lambda g, ans, gvs, vs, x, axis=None: anp.cumsum(g, axis=axis))
anp.inner.defjvp(lambda g, ans, gvs, vs, A, B: anp.inner(g, B))
anp.inner.defjvp(lambda g, ans, gvs, vs, A, B: anp.inner(A, g), argnum=1)
anp.matmul.defjvp(lambda g, ans, gvs, vs, A, B: anp.matmul(g, B))
anp.matmul.defjvp(lambda g, ans, gvs, vs, A, B: anp.matmul(A, g), argnum=1)
anp.dot.defjvp(lambda g, ans, gvs, vs, A, B: anp.dot(g, B))
anp.dot.defjvp(lambda g, ans, gvs, vs, A, B: anp.dot(A, g), argnum=1)
anp.tensordot.defjvp(
lambda g, ans, gvs, vs, A, B, axes=2: anp.tensordot(g, B, axes=axes))
anp.tensordot.defjvp(
lambda g, ans, gvs, vs, A, B, axes=2: anp.tensordot(A, g, axes=axes),
argnum=1)
anp.outer.defjvp(lambda g, ans, gvs, vs, a, b: anp.outer(g, b))
def g4(self,angles4):
# output = np.ones((4,3))
# determine distances between points
# angles411 = np.arccos(angles4[1, 1])
# angles410 = np.arccos(angles4[1, 0])
# angles421 = np.arccos(angles4[2, 1])
# angles431 = np.arccos(angles4[3, 1])
# angles420 = np.arccos(angles4[2, 0])
# angles430 = np.arccos(angles4[3, 0])
# angles400 = np.arccos(angles4[0, 0])
# angles401 = np.arccos(angles4[0, 1])
angles411 = (angles4[1, 1])
angles410 = (angles4[1, 0])
angles421 = (angles4[2, 1])
angles431 = (angles4[3, 1])
angles420 = (angles4[2, 0])
angles430 = (angles4[3, 0])
angles400 = (angles4[0, 0])
#angles401 = (angles4[0, 1])
c2d2 = 1 / np.sin(angles411)
c1d2 = 1 / np.sin(angles410)
d2_d1c2 = c2d2 * np.sin(angles421)
d2_c1d1 = np.sin(angles431) * c1d2
d1c2 = (d2_d1c2 / np.tan(angles420)) + (d2_d1c2 / np.tan(angles421))
d1c1 = (d2_c1d1 / np.tan(angles430)) + (d2_c1d1 / np.tan(angles431))
d1d2 = d2_d1c2 / np.sin(angles420)
#d1_c1c2 = np.sin(angles401) * d1c1
# law of cosines
#c1c2 = (d1c1 ** 2 + d1c2 ** 2 - 2 * d1c2 * d1c1 * np.cos(angles400)) ** 0.5
result = []
result.append(0.0)
result.append(0.0)
result.append(0.0)
result.append(d1c1)
result.append(0.0)
result.append(0.0)
result.append(d1c2 * np.cos(angles400))
result.append(d1c2 * np.sin(angles400))
result.append(0.0)
s = c1d2 * np.cos(angles410) / (c2d2 * np.cos(angles411) + c1d2 * np.cos(angles410))
asdf = [(1 - s) * d1c1 + s * d1c2 * np.cos(angles400), s * d1c2 * np.sin(angles400), 0.]
slope = (result[3] - result[6]) / (result[7])
x = d1d2 * np.cos(angles430)
result.append(x)
# rise = slope * (x - d1c1)
rise = slope * (x - asdf[0])
y = asdf[1] + rise
result.append(y)
z = (d1d2 ** 2 - (x ** 2 + y ** 2)) ** 0.5
result.append(z)
# return(asdf[2])
# print(result)
result = np.asarray(result)
# return(result[11])
d1 = result[0:3]
# return(d1[])
c1 = result[3:6]
c2 = result[6:9]
d2 = result[9:12]
cc = (c2 - c1)
ip = np.inner((d1 - c1), (c2 - c1)) / (np.sum((c2 - c1) ** 2))
# return(output)
tilded1 = [d1[0] - ip * cc[0], d1[1] - ip * cc[1], d1[2] - ip * cc[2]]
iq = np.inner((d2 - c2), (c1 - c2)) / (np.sum((c1 - c2) ** 2))
cc2 = (c1 - c2)
tilded2 = [d2[0] - iq * cc2[0], d2[1] - iq * cc2[1], d2[2] - iq * cc2[2]]
tilded2star = [tilded2[0] + cc2[0], tilded2[1] + cc2[1], tilded2[2] + cc2[2]]
ab = np.sqrt((tilded2star[0] - c1[0]) ** 2 + (tilded2star[1] - c1[1]) ** 2 + (tilded2star[2] - c1[2]) ** 2)
bc = np.sqrt((tilded2star[0] - tilded1[0]) ** 2 + (tilded2star[1] - tilded1[1]) ** 2 + (tilded2star[2] - tilded1[2]) ** 2)
ca = np.sqrt((tilded1[0] - c1[0]) ** 2 + (tilded1[1] - c1[1]) ** 2 + (tilded1[2] - c1[2]) ** 2)
output = (ab ** 2 - bc ** 2 + ca ** 2) / (2 * ab * ca)
return (output)
def cost(x):
return np.inner(x, A @ x)
def rank_1_H_update(H, s, y, d):
return H + np.outer((s - np.matmul(H, y)), d) / np.inner(d, y)
print(X.shape)
exactCV, exactParams, sets = retrainingPlans.leave_k_out_cv(
model, k=1, method="exact", B=100, hold_outs='stochastic')
sets = np.array(sets).squeeze().astype(np.int32)
exact = np.einsum('nd,nd->n', exactParams, X[sets])
NS = np.zeros(sets.shape[0])
IJ = np.zeros(sets.shape[0])
NSAppx = np.zeros(sets.shape[0])
IJAppx = np.zeros(sets.shape[0])
IJAppxBnd = np.zeros(sets.shape[0])
for idx, n in enumerate(sets):
IJParams = thetaHat + D1[n] * iprods[n]
NS[idx] = np.inner(thetaHat, X[n]) + D1[n] * Q[n] / (1 - D2[n] * Q[n])
IJ[idx] = np.inner(thetaHat, X[n]) + D1[n] * Q[n]
IJAppx[idx] = np.inner(thetaHat, X[n]) + D1[n] * Qappx[n]
IJAppxBnd[idx] = np.abs(D1[n]) * estQErr[n]
model.compute_bounds()
NSBnd = model.NSBnd
IJBnd = model.IJBnd
totalIJBnd = IJAppxBnd + IJBnd[sets]
print('NS bound holds:', np.all(np.abs(NS - exact) < NSBnd[sets]))
print('IJ bound holds:', np.all(np.abs(IJ - exact) < IJBnd[sets]))
predErrsExact = (Y[sets] - np.exp(exact))**2
predErrsNS = (Y[sets] - np.exp(NS))**2
predErrsNSUpper = (Y[sets] - np.exp(NS + NSBnd[sets]))**2
def c(x):
inner = np.inner(x[i], quat)
print "Inner: ", inner
return thresh - (1 - 2 * (inner**2))
moving points collision avoidance
State: x,y,vx,vy
Input: ax,ay
constraint:
Do not collide into bad area
limited ax, and ay
"""
# import numpy as np
import autograd.numpy as np
import scipy.linalg as linalg
from autograd import grad, jacobian
MAX_INPUT = 20
sqeuclidean = lambda x: np.inner(x, x) # L2 NORM SQUARE
dt = 0.1
Dyn_A = np.array([[0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 0, 0], [0, 0, 0, 0]])
Dyn_B = np.array([[0, 0], [0, 0], [0.1, 0], [0, 0.1]])
# change to distrete time system
sysdt = linalg.expm(
np.concatenate(
[np.concatenate([Dyn_A, Dyn_B], axis=1) * dt,
np.zeros((2, 6))],
axis=0))
sys_A = sysdt[:4, :4]
sys_B = sysdt[:4, -2:]
def calc_loss__slda(
param_vec=None,
dim_P=None,
dataset=None,
topics_KV=None,
w_CK=None,
alpha=None,
tau=1.1,
delta=0.1,
lambda_w=0.001,
weight_x=1.0,
weight_y=1.0,
weight_pi=1.0,
return_dict=False,
rescale_total_loss_by_n_tokens=True,
pi_estimation_mode='missing_y',
frac_train_laps_completed=1.0,
pi_max_iters_first_train_lap=DefaultDocTopicOptKwargs['pi_max_iters'],
pi_max_iters=DefaultDocTopicOptKwargs['pi_max_iters'],
active_proba_thr=0.005,
**unused_kwargs):
''' Compute loss of provided dataset under sLDA topic model.
Returns
-------
loss : float
Total loss (-1 * log proba.) of dataset under provided sLDA model.
By default, rescaled by number of word tokens in the dataset.
'''
# Unpack common parameters
if param_vec is not None:
param_dict = unflatten_to_common_param_dict(param_vec, **dim_P)
topics_KV = param_dict['topics_KV']
w_CK = param_dict['w_CK']
assert topics_KV is not None
assert w_CK is not None
# Unpack hyperparams
delta = float(delta)
tau = float(tau)
lambda_w = float(lambda_w)
convex_alpha_minus_1 = make_convex_alpha_minus_1(alpha=alpha)
assert convex_alpha_minus_1 >= 0.0
assert convex_alpha_minus_1 < 1.0
n_docs = int(dataset['n_docs'])
n_labels = int(dataset['n_labels'])
if 'y_rowmask' in dataset:
y_finite_DC = dataset['y_DC'][dataset['y_rowmask'] == 1]
u_y_vals = np.unique(y_finite_DC.flatten())
else:
u_y_vals = np.unique(dataset['y_DC'].flatten())
assert np.all(np.isfinite(u_y_vals))
if u_y_vals.size <= 2 and np.union1d([0.0, 1.0], u_y_vals).size == 2:
output_data_type = 'binary'
else:
output_data_type = 'real'
K = w_CK.shape[1]
if return_dict:
start_time_sec = time.time()
pi_DK = np.zeros((n_docs, K))
n_docs_converged = 0
n_docs_restarted = 0
iters_per_doc = np.zeros(n_docs, dtype=np.int32)
dist_per_doc = np.zeros(n_docs, dtype=np.float32)
step_size_per_doc = np.zeros(n_docs, dtype=np.float32)
n_active_per_doc = np.zeros(n_docs, dtype=np.int32)
restarts_per_doc = np.zeros(n_docs, dtype=np.int32)
if return_dict and weight_y > 0:
y_proba_DC = np.zeros((n_docs, n_labels))
## Establish kwargs for pi optimization step
# Use 'ramp up' strategy to gradually increase per-doc iteration costs.
# At first, perform only pi_max_iters_first_train_lap.
# Linearly increase until reaching pi_max_iters,
# which is designed to happen 50% of way through training.
#
# frac_progress : float within (0.0, 1.0)
# 0.0 when frac_lap == 0
# 0.5 when frac_lap == 0.25
# 1.0 when frac_lap >= 0.5
# cur_pi_max_iters : int
# Number of pi iters to run now
assert pi_max_iters_first_train_lap <= pi_max_iters
frac_progress = np.minimum(1.0, 2 * frac_train_laps_completed)
cur_pi_max_iters = int(
pi_max_iters_first_train_lap +
np.ceil(frac_progress * (pi_max_iters - pi_max_iters_first_train_lap)))
# Pack up into the kwargs handed to pi optimization
pi_opt_kwargs = dict(**DefaultDocTopicOptKwargs)
pi_opt_kwargs['pi_max_iters'] = cur_pi_max_iters
# Aggregators for different loss terms
loss_x = 0.0
loss_y = 0.0
loss_pi = 0.0
assert pi_estimation_mode == 'missing_y'
for d in xrange(n_docs):
start_d = dataset['doc_indptr_Dp1'][d]
stop_d = dataset['doc_indptr_Dp1'][d + 1]
word_id_d_U = dataset['word_id_U'][start_d:stop_d]
word_ct_d_U = dataset['word_ct_U'][start_d:stop_d]
pi_d_K, info_d = \
calc_nef_map_pi_d_K__autograd(
word_id_d_U,
word_ct_d_U,
topics_KV=topics_KV,
convex_alpha_minus_1=convex_alpha_minus_1,
**pi_opt_kwargs)
if return_dict:
pi_DK[d] = pi_d_K
n_active_per_doc[d] = np.sum(pi_d_K > active_proba_thr)
n_docs_restarted += info_d['n_restarts'] > 0
n_docs_converged += info_d['did_converge']
iters_per_doc[d] = info_d['n_iters']
step_size_per_doc[d] = info_d['pi_step_size']
dist_per_doc[d] = info_d.get('cur_L1_diff', -1.0)
restarts_per_doc[d] = info_d.get('n_restarts', -1)
if weight_x > 0:
logpdf_x_d = np.inner(
word_ct_d_U, np.log(np.dot(pi_d_K, topics_KV[:, word_id_d_U])))
logpdf_x_d += \
gammaln(1.0 + np.sum(word_ct_d_U)) - \
np.sum(gammaln(1.0 + word_ct_d_U))
loss_x -= weight_x * logpdf_x_d
if weight_pi > 0:
loss_pi -= weight_pi * np.sum(
(convex_alpha_minus_1) * np.log(1e-9 + pi_d_K))
# Semi-supervised case: skip examples with unknown labels
if 'y_rowmask' in dataset and dataset['y_rowmask'][d] == 0:
continue
if weight_y > 0 and output_data_type == 'binary':
y_d_C = dataset['y_DC'][d]
sign_y_d_C = np.sign(y_d_C - 0.01)
logpdf_y_d = np.sum(
log_logistic_sigmoid(sign_y_d_C * np.dot(w_CK, pi_d_K)))
loss_y -= weight_y * logpdf_y_d
if return_dict:
proba_y_eq_1_d_C = logistic_sigmoid(np.dot(w_CK, pi_d_K))
y_proba_DC[d] = proba_y_eq_1_d_C
if weight_y > 0 and output_data_type == 'real':
y_d_C = dataset['y_DC'][d]
y_est_d_C = np.dot(w_CK, pi_d_K)
logpdf_y_d = -0.5 / delta * np.sum(np.square(y_est_d_C - y_d_C))
loss_y -= weight_y * logpdf_y_d
if return_dict:
y_proba_DC[d] = y_est_d_C
# ... end loop over docs
# GLOBAL PARAM REGULARIZATION TERMS
# Loss for topic-word params
loss_topics = \
-1.0 * (tau - 1) * np.sum(np.log(topics_KV))
# Loss for regression weights
# Needs to scale with weight_y so lambda_w doesnt grow as weight_y grows
loss_w = \
float(weight_y) * lambda_w * np.sum(np.square(w_CK))
# RESCALING LOSS TERMS
if rescale_total_loss_by_n_tokens:
scale_ttl = float(np.sum(dataset['word_ct_U']))
else:
scale_ttl = 1.0
loss_x /= scale_ttl
loss_pi /= scale_ttl
loss_topics /= scale_ttl
loss_w /= scale_ttl
loss_y /= scale_ttl
# TOTAL LOSS
loss_ttl = loss_x + loss_y + loss_pi + loss_topics + loss_w
if return_dict:
# Compute unweighted loss
uw_x = np.maximum(weight_x, 1.0)
uloss_x__pertok = loss_x * scale_ttl / float(
uw_x * np.sum(dataset['word_ct_U']))
uw_y = np.maximum(weight_y, 1.0)
n_y_docs, C = dataset['y_DC'].shape
n_y_docs = 1e-10 + float(n_y_docs)
if 'y_rowmask' in dataset:
n_y_docs = 1e-10 + float(np.sum(dataset['y_rowmask']))
uloss_y__perdoc = loss_y * scale_ttl / float(uw_y * C * n_y_docs)
ans_dict = dict(
loss_ttl=loss_ttl,
loss_x=loss_x,
loss_y=loss_y,
loss_pi=loss_pi,
loss_topics=loss_topics,
loss_w=loss_w,
rescale_total_loss_by_n_tokens=rescale_total_loss_by_n_tokens,
uloss_x__pertok=uloss_x__pertok,
uloss_y__perdoc=uloss_y__perdoc,
output_data_type=output_data_type,
pi_DK=pi_DK,
n_docs_converged=n_docs_converged,
n_docs_restarted=n_docs_restarted,
iters_per_doc=iters_per_doc,
dist_per_doc=dist_per_doc,
step_size_per_doc=step_size_per_doc,
n_active_per_doc=n_active_per_doc,
summary_msg=make_readable_summary_for_pi_DK_estimation(
elapsed_time_sec=time.time() - start_time_sec,
n_docs=n_docs,
n_docs_converged=n_docs_converged,
n_docs_restarted=n_docs_restarted,
iters_per_doc=iters_per_doc,
dist_per_doc=dist_per_doc,
step_size_per_doc=step_size_per_doc,
restarts_per_doc=restarts_per_doc,
n_active_per_doc=n_active_per_doc,
),
)
if weight_y > 0:
ans_dict['y_proba_DC'] = y_proba_DC
return ans_dict
else:
return loss_ttl
def cost(x):
return -np.inner(x, matrix @ x)
from __future__ import absolute_import import autograd.scipy.stats.dirichlet as di import autograd.numpy as np from autograd.scipy.special import digamma di.logpdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, (alpha - 1) / x), argnum=0) di.logpdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, (digamma(np.sum(alpha)) - digamma(alpha) + np.log(x))), argnum=1) di.pdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, ans * (alpha - 1) / x), argnum=0) di.pdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, ans * (digamma(np.sum(alpha)) - digamma(alpha) + np.log(x))), argnum=1)
