def rank_2_H_update(H, s, y, d): Hy = np.matmul(H, y) temp = np.outer(s - Hy, d) ddT = np.outer(d, d) dTy = np.inner(d, y) return H + (temp + np.transpose(temp)) / dTy - np.inner( y, s - Hy) * ddT / (dTy**2)
def rank_2_H_update(H,s,y,d): Hy = np.matmul(H,y) temp = np.outer(s-Hy,d) ddT = np.outer(d,d) dTy = np.inner(d,y) if dTy**2 == 0: raise ZeroDivisionError return H + (temp + np.transpose(temp))/dTy - np.inner(y,s-Hy)*ddT/(dTy**2)
def test_inner_product(self): x = self.manifold.random_point() / 2 u = self.manifold.random_tangent_vector(x) v = self.manifold.random_tangent_vector(x) np_testing.assert_allclose( (2 / (1 - np.linalg.norm(x) ** 2)) ** 2 * np.inner(u, v), self.manifold.inner_product(x, u, v), ) # Test that angles are preserved. x = self.manifold.random_point() / 2 u = self.manifold.random_tangent_vector(x) v = self.manifold.random_tangent_vector(x) cos_eangle = np.sum(u * v) / np.linalg.norm(u) / np.linalg.norm(v) cos_rangle = ( self.manifold.inner_product(x, u, v) / self.manifold.norm(x, u) / self.manifold.norm(x, v) ) np_testing.assert_allclose(cos_rangle, cos_eangle) # Test symmetry. np_testing.assert_allclose( self.manifold.inner_product(x, u, v), self.manifold.inner_product(x, v, u), )
def h1_mean_var(X, Y, k, l): """ X: nxd numpy array Y: nxd numpy array k, l: a Kernel object return (HSIC, var[HSIC]) under H1 """ m = X.shape[0] n = Y.shape[0] if m != n: raise ValueError('length X should be the same as length Y') K = k.eval(X, X) L = l.eval(Y, Y) Kmean = np.mean(K, 0) Lmean = np.mean(L, 0) HK = K - Kmean HL = L - Lmean # t = trace(KHLH) HKf = HK.flatten() / (n - 1) HLf = HL.T.flatten() / (n - 1) hsic = HKf.dot(HLf) #hsic = np.sum(centering(KX) * centering(KY)) / m**2 # variance computation var = 0 for i in range(m): var1 = np.inner(K[i, :], L[i, :]) # inner product between vectors var2 = (L.sum() * K[i, :]).sum() var3 = np.outer(K[i, :], L[i, :]).sum() var += ((var1 / m + var2 / (m**3) - 2 * var3 / (m**2))**2) / m hsic_var = max(16 * (var - hsic**2), 1e-3) # variance computation 2 #KX = KX - np.diag(KX); KY = KY - np.diag(KY) #KXKY = KX @ KY; KYKX = KY @ KX #ones = np.ones(m) #h1 = (m-2)**2 * (np.multiply(KX,KY) @ ones) #h2 = (m-2) * (np.trace(KXKY) - KXKY @ ones - KYKX @ ones ) #print(h2) #h3 = m * np.multiply(KX @ ones, KY @ ones) #print(h3) #h4 = (ones @ KY @ ones) * (KX @ ones) #print(h4) #h5 = (ones @ KX @ ones) * (KY @ ones) #print(h5) #h6 = (ones @ KXKY @ ones) #h = h1 + h2 -h3 + h4 + h5 - h6 #print(h @ h / (4*m*((m-1)*(m-2)*(m-3))**2)) return hsic, hsic_var
def mvnlogpdf(x, mu, L): """ not really logpdf. we need to use the weights to keep track of normalizing factors that differ across clusters L cholesky decomposition of covariance matrix """ D = L.shape[0] logdet = 2 * np.sum(np.log(np.diagonal(L))) quad = np.inner(x - mu, solve(L.T, solve(L, (x - mu)))) return -0.5 *(D * np.log(2 * np.pi) + logdet + quad)
def test_inner(self): x = self.man.rand() / 2 u = self.man.randvec(x) v = self.man.randvec(x) np_testing.assert_allclose( (2 / (1 - la.norm(x)**2))**2 * np.inner(u, v), self.man.inner(x, u, v), ) # test that angles are preserved x = self.man.rand() / 2 u = self.man.randvec(x) v = self.man.randvec(x) cos_eangle = np.sum(u * v) / la.norm(u) / la.norm(v) cos_rangle = (self.man.inner(x, u, v) / self.man.norm(x, u) / self.man.norm(x, v)) np_testing.assert_allclose(cos_rangle, cos_eangle)
elif isinstance(axis, tuple): for ax in sorted(axis): ans = anp.expand_dims(ans, ax) chosen_locations = x == ans return anp.sum(g * chosen_locations, axis=axis, keepdims=keepdims) anp.max.defjvp(fwd_grad_chooser) anp.min.defjvp(fwd_grad_chooser) anp.amax.defjvp(fwd_grad_chooser) anp.amin.defjvp(fwd_grad_chooser) anp.cumsum.defjvp( lambda g, ans, gvs, vs, x, axis=None: anp.cumsum(g, axis=axis)) anp.inner.defjvp(lambda g, ans, gvs, vs, A, B: anp.inner(g, B)) anp.inner.defjvp(lambda g, ans, gvs, vs, A, B: anp.inner(A, g), argnum=1) anp.matmul.defjvp(lambda g, ans, gvs, vs, A, B: anp.matmul(g, B)) anp.matmul.defjvp(lambda g, ans, gvs, vs, A, B: anp.matmul(A, g), argnum=1) anp.dot.defjvp(lambda g, ans, gvs, vs, A, B: anp.dot(g, B)) anp.dot.defjvp(lambda g, ans, gvs, vs, A, B: anp.dot(A, g), argnum=1) anp.tensordot.defjvp( lambda g, ans, gvs, vs, A, B, axes=2: anp.tensordot(g, B, axes=axes)) anp.tensordot.defjvp( lambda g, ans, gvs, vs, A, B, axes=2: anp.tensordot(A, g, axes=axes), argnum=1) anp.outer.defjvp(lambda g, ans, gvs, vs, a, b: anp.outer(g, b))
def g4(self,angles4): # output = np.ones((4,3)) # determine distances between points # angles411 = np.arccos(angles4[1, 1]) # angles410 = np.arccos(angles4[1, 0]) # angles421 = np.arccos(angles4[2, 1]) # angles431 = np.arccos(angles4[3, 1]) # angles420 = np.arccos(angles4[2, 0]) # angles430 = np.arccos(angles4[3, 0]) # angles400 = np.arccos(angles4[0, 0]) # angles401 = np.arccos(angles4[0, 1]) angles411 = (angles4[1, 1]) angles410 = (angles4[1, 0]) angles421 = (angles4[2, 1]) angles431 = (angles4[3, 1]) angles420 = (angles4[2, 0]) angles430 = (angles4[3, 0]) angles400 = (angles4[0, 0]) #angles401 = (angles4[0, 1]) c2d2 = 1 / np.sin(angles411) c1d2 = 1 / np.sin(angles410) d2_d1c2 = c2d2 * np.sin(angles421) d2_c1d1 = np.sin(angles431) * c1d2 d1c2 = (d2_d1c2 / np.tan(angles420)) + (d2_d1c2 / np.tan(angles421)) d1c1 = (d2_c1d1 / np.tan(angles430)) + (d2_c1d1 / np.tan(angles431)) d1d2 = d2_d1c2 / np.sin(angles420) #d1_c1c2 = np.sin(angles401) * d1c1 # law of cosines #c1c2 = (d1c1 ** 2 + d1c2 ** 2 - 2 * d1c2 * d1c1 * np.cos(angles400)) ** 0.5 result = [] result.append(0.0) result.append(0.0) result.append(0.0) result.append(d1c1) result.append(0.0) result.append(0.0) result.append(d1c2 * np.cos(angles400)) result.append(d1c2 * np.sin(angles400)) result.append(0.0) s = c1d2 * np.cos(angles410) / (c2d2 * np.cos(angles411) + c1d2 * np.cos(angles410)) asdf = [(1 - s) * d1c1 + s * d1c2 * np.cos(angles400), s * d1c2 * np.sin(angles400), 0.] slope = (result[3] - result[6]) / (result[7]) x = d1d2 * np.cos(angles430) result.append(x) # rise = slope * (x - d1c1) rise = slope * (x - asdf[0]) y = asdf[1] + rise result.append(y) z = (d1d2 ** 2 - (x ** 2 + y ** 2)) ** 0.5 result.append(z) # return(asdf[2]) # print(result) result = np.asarray(result) # return(result[11]) d1 = result[0:3] # return(d1[]) c1 = result[3:6] c2 = result[6:9] d2 = result[9:12] cc = (c2 - c1) ip = np.inner((d1 - c1), (c2 - c1)) / (np.sum((c2 - c1) ** 2)) # return(output) tilded1 = [d1[0] - ip * cc[0], d1[1] - ip * cc[1], d1[2] - ip * cc[2]] iq = np.inner((d2 - c2), (c1 - c2)) / (np.sum((c1 - c2) ** 2)) cc2 = (c1 - c2) tilded2 = [d2[0] - iq * cc2[0], d2[1] - iq * cc2[1], d2[2] - iq * cc2[2]] tilded2star = [tilded2[0] + cc2[0], tilded2[1] + cc2[1], tilded2[2] + cc2[2]] ab = np.sqrt((tilded2star[0] - c1[0]) ** 2 + (tilded2star[1] - c1[1]) ** 2 + (tilded2star[2] - c1[2]) ** 2) bc = np.sqrt((tilded2star[0] - tilded1[0]) ** 2 + (tilded2star[1] - tilded1[1]) ** 2 + (tilded2star[2] - tilded1[2]) ** 2) ca = np.sqrt((tilded1[0] - c1[0]) ** 2 + (tilded1[1] - c1[1]) ** 2 + (tilded1[2] - c1[2]) ** 2) output = (ab ** 2 - bc ** 2 + ca ** 2) / (2 * ab * ca) return (output)
def cost(x): return np.inner(x, A @ x)
def rank_1_H_update(H, s, y, d): return H + np.outer((s - np.matmul(H, y)), d) / np.inner(d, y)
print(X.shape) exactCV, exactParams, sets = retrainingPlans.leave_k_out_cv( model, k=1, method="exact", B=100, hold_outs='stochastic') sets = np.array(sets).squeeze().astype(np.int32) exact = np.einsum('nd,nd->n', exactParams, X[sets]) NS = np.zeros(sets.shape[0]) IJ = np.zeros(sets.shape[0]) NSAppx = np.zeros(sets.shape[0]) IJAppx = np.zeros(sets.shape[0]) IJAppxBnd = np.zeros(sets.shape[0]) for idx, n in enumerate(sets): IJParams = thetaHat + D1[n] * iprods[n] NS[idx] = np.inner(thetaHat, X[n]) + D1[n] * Q[n] / (1 - D2[n] * Q[n]) IJ[idx] = np.inner(thetaHat, X[n]) + D1[n] * Q[n] IJAppx[idx] = np.inner(thetaHat, X[n]) + D1[n] * Qappx[n] IJAppxBnd[idx] = np.abs(D1[n]) * estQErr[n] model.compute_bounds() NSBnd = model.NSBnd IJBnd = model.IJBnd totalIJBnd = IJAppxBnd + IJBnd[sets] print('NS bound holds:', np.all(np.abs(NS - exact) < NSBnd[sets])) print('IJ bound holds:', np.all(np.abs(IJ - exact) < IJBnd[sets])) predErrsExact = (Y[sets] - np.exp(exact))**2 predErrsNS = (Y[sets] - np.exp(NS))**2 predErrsNSUpper = (Y[sets] - np.exp(NS + NSBnd[sets]))**2
def c(x): inner = np.inner(x[i], quat) print "Inner: ", inner return thresh - (1 - 2 * (inner**2))
moving points collision avoidance State: x,y,vx,vy Input: ax,ay constraint: Do not collide into bad area limited ax, and ay """ # import numpy as np import autograd.numpy as np import scipy.linalg as linalg from autograd import grad, jacobian MAX_INPUT = 20 sqeuclidean = lambda x: np.inner(x, x) # L2 NORM SQUARE dt = 0.1 Dyn_A = np.array([[0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 0, 0], [0, 0, 0, 0]]) Dyn_B = np.array([[0, 0], [0, 0], [0.1, 0], [0, 0.1]]) # change to distrete time system sysdt = linalg.expm( np.concatenate( [np.concatenate([Dyn_A, Dyn_B], axis=1) * dt, np.zeros((2, 6))], axis=0)) sys_A = sysdt[:4, :4] sys_B = sysdt[:4, -2:]
def calc_loss__slda( param_vec=None, dim_P=None, dataset=None, topics_KV=None, w_CK=None, alpha=None, tau=1.1, delta=0.1, lambda_w=0.001, weight_x=1.0, weight_y=1.0, weight_pi=1.0, return_dict=False, rescale_total_loss_by_n_tokens=True, pi_estimation_mode='missing_y', frac_train_laps_completed=1.0, pi_max_iters_first_train_lap=DefaultDocTopicOptKwargs['pi_max_iters'], pi_max_iters=DefaultDocTopicOptKwargs['pi_max_iters'], active_proba_thr=0.005, **unused_kwargs): ''' Compute loss of provided dataset under sLDA topic model. Returns ------- loss : float Total loss (-1 * log proba.) of dataset under provided sLDA model. By default, rescaled by number of word tokens in the dataset. ''' # Unpack common parameters if param_vec is not None: param_dict = unflatten_to_common_param_dict(param_vec, **dim_P) topics_KV = param_dict['topics_KV'] w_CK = param_dict['w_CK'] assert topics_KV is not None assert w_CK is not None # Unpack hyperparams delta = float(delta) tau = float(tau) lambda_w = float(lambda_w) convex_alpha_minus_1 = make_convex_alpha_minus_1(alpha=alpha) assert convex_alpha_minus_1 >= 0.0 assert convex_alpha_minus_1 < 1.0 n_docs = int(dataset['n_docs']) n_labels = int(dataset['n_labels']) if 'y_rowmask' in dataset: y_finite_DC = dataset['y_DC'][dataset['y_rowmask'] == 1] u_y_vals = np.unique(y_finite_DC.flatten()) else: u_y_vals = np.unique(dataset['y_DC'].flatten()) assert np.all(np.isfinite(u_y_vals)) if u_y_vals.size <= 2 and np.union1d([0.0, 1.0], u_y_vals).size == 2: output_data_type = 'binary' else: output_data_type = 'real' K = w_CK.shape[1] if return_dict: start_time_sec = time.time() pi_DK = np.zeros((n_docs, K)) n_docs_converged = 0 n_docs_restarted = 0 iters_per_doc = np.zeros(n_docs, dtype=np.int32) dist_per_doc = np.zeros(n_docs, dtype=np.float32) step_size_per_doc = np.zeros(n_docs, dtype=np.float32) n_active_per_doc = np.zeros(n_docs, dtype=np.int32) restarts_per_doc = np.zeros(n_docs, dtype=np.int32) if return_dict and weight_y > 0: y_proba_DC = np.zeros((n_docs, n_labels)) ## Establish kwargs for pi optimization step # Use 'ramp up' strategy to gradually increase per-doc iteration costs. # At first, perform only pi_max_iters_first_train_lap. # Linearly increase until reaching pi_max_iters, # which is designed to happen 50% of way through training. # # frac_progress : float within (0.0, 1.0) # 0.0 when frac_lap == 0 # 0.5 when frac_lap == 0.25 # 1.0 when frac_lap >= 0.5 # cur_pi_max_iters : int # Number of pi iters to run now assert pi_max_iters_first_train_lap <= pi_max_iters frac_progress = np.minimum(1.0, 2 * frac_train_laps_completed) cur_pi_max_iters = int( pi_max_iters_first_train_lap + np.ceil(frac_progress * (pi_max_iters - pi_max_iters_first_train_lap))) # Pack up into the kwargs handed to pi optimization pi_opt_kwargs = dict(**DefaultDocTopicOptKwargs) pi_opt_kwargs['pi_max_iters'] = cur_pi_max_iters # Aggregators for different loss terms loss_x = 0.0 loss_y = 0.0 loss_pi = 0.0 assert pi_estimation_mode == 'missing_y' for d in xrange(n_docs): start_d = dataset['doc_indptr_Dp1'][d] stop_d = dataset['doc_indptr_Dp1'][d + 1] word_id_d_U = dataset['word_id_U'][start_d:stop_d] word_ct_d_U = dataset['word_ct_U'][start_d:stop_d] pi_d_K, info_d = \ calc_nef_map_pi_d_K__autograd( word_id_d_U, word_ct_d_U, topics_KV=topics_KV, convex_alpha_minus_1=convex_alpha_minus_1, **pi_opt_kwargs) if return_dict: pi_DK[d] = pi_d_K n_active_per_doc[d] = np.sum(pi_d_K > active_proba_thr) n_docs_restarted += info_d['n_restarts'] > 0 n_docs_converged += info_d['did_converge'] iters_per_doc[d] = info_d['n_iters'] step_size_per_doc[d] = info_d['pi_step_size'] dist_per_doc[d] = info_d.get('cur_L1_diff', -1.0) restarts_per_doc[d] = info_d.get('n_restarts', -1) if weight_x > 0: logpdf_x_d = np.inner( word_ct_d_U, np.log(np.dot(pi_d_K, topics_KV[:, word_id_d_U]))) logpdf_x_d += \ gammaln(1.0 + np.sum(word_ct_d_U)) - \ np.sum(gammaln(1.0 + word_ct_d_U)) loss_x -= weight_x * logpdf_x_d if weight_pi > 0: loss_pi -= weight_pi * np.sum( (convex_alpha_minus_1) * np.log(1e-9 + pi_d_K)) # Semi-supervised case: skip examples with unknown labels if 'y_rowmask' in dataset and dataset['y_rowmask'][d] == 0: continue if weight_y > 0 and output_data_type == 'binary': y_d_C = dataset['y_DC'][d] sign_y_d_C = np.sign(y_d_C - 0.01) logpdf_y_d = np.sum( log_logistic_sigmoid(sign_y_d_C * np.dot(w_CK, pi_d_K))) loss_y -= weight_y * logpdf_y_d if return_dict: proba_y_eq_1_d_C = logistic_sigmoid(np.dot(w_CK, pi_d_K)) y_proba_DC[d] = proba_y_eq_1_d_C if weight_y > 0 and output_data_type == 'real': y_d_C = dataset['y_DC'][d] y_est_d_C = np.dot(w_CK, pi_d_K) logpdf_y_d = -0.5 / delta * np.sum(np.square(y_est_d_C - y_d_C)) loss_y -= weight_y * logpdf_y_d if return_dict: y_proba_DC[d] = y_est_d_C # ... end loop over docs # GLOBAL PARAM REGULARIZATION TERMS # Loss for topic-word params loss_topics = \ -1.0 * (tau - 1) * np.sum(np.log(topics_KV)) # Loss for regression weights # Needs to scale with weight_y so lambda_w doesnt grow as weight_y grows loss_w = \ float(weight_y) * lambda_w * np.sum(np.square(w_CK)) # RESCALING LOSS TERMS if rescale_total_loss_by_n_tokens: scale_ttl = float(np.sum(dataset['word_ct_U'])) else: scale_ttl = 1.0 loss_x /= scale_ttl loss_pi /= scale_ttl loss_topics /= scale_ttl loss_w /= scale_ttl loss_y /= scale_ttl # TOTAL LOSS loss_ttl = loss_x + loss_y + loss_pi + loss_topics + loss_w if return_dict: # Compute unweighted loss uw_x = np.maximum(weight_x, 1.0) uloss_x__pertok = loss_x * scale_ttl / float( uw_x * np.sum(dataset['word_ct_U'])) uw_y = np.maximum(weight_y, 1.0) n_y_docs, C = dataset['y_DC'].shape n_y_docs = 1e-10 + float(n_y_docs) if 'y_rowmask' in dataset: n_y_docs = 1e-10 + float(np.sum(dataset['y_rowmask'])) uloss_y__perdoc = loss_y * scale_ttl / float(uw_y * C * n_y_docs) ans_dict = dict( loss_ttl=loss_ttl, loss_x=loss_x, loss_y=loss_y, loss_pi=loss_pi, loss_topics=loss_topics, loss_w=loss_w, rescale_total_loss_by_n_tokens=rescale_total_loss_by_n_tokens, uloss_x__pertok=uloss_x__pertok, uloss_y__perdoc=uloss_y__perdoc, output_data_type=output_data_type, pi_DK=pi_DK, n_docs_converged=n_docs_converged, n_docs_restarted=n_docs_restarted, iters_per_doc=iters_per_doc, dist_per_doc=dist_per_doc, step_size_per_doc=step_size_per_doc, n_active_per_doc=n_active_per_doc, summary_msg=make_readable_summary_for_pi_DK_estimation( elapsed_time_sec=time.time() - start_time_sec, n_docs=n_docs, n_docs_converged=n_docs_converged, n_docs_restarted=n_docs_restarted, iters_per_doc=iters_per_doc, dist_per_doc=dist_per_doc, step_size_per_doc=step_size_per_doc, restarts_per_doc=restarts_per_doc, n_active_per_doc=n_active_per_doc, ), ) if weight_y > 0: ans_dict['y_proba_DC'] = y_proba_DC return ans_dict else: return loss_ttl
def cost(x): return -np.inner(x, matrix @ x)
from __future__ import absolute_import import autograd.scipy.stats.dirichlet as di import autograd.numpy as np from autograd.scipy.special import digamma di.logpdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, (alpha - 1) / x), argnum=0) di.logpdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, (digamma(np.sum(alpha)) - digamma(alpha) + np.log(x))), argnum=1) di.pdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, ans * (alpha - 1) / x), argnum=0) di.pdf.defjvp(lambda g, ans, gvs, vs, x, alpha: np.inner(g, ans * (digamma(np.sum(alpha)) - digamma(alpha) + np.log(x))), argnum=1)