def renyii(pk0, pk1, a):
"""
Compute the renyii divergence between two Gaussian distributions.
"""
# Check dimensions
assert (pk0.S.shape == pk1.S.shape)
# Check diagonal
p0S_is_diag = np.all(np.diag(np.diag(pk0.S)) == pk0.S)
p1S_is_diag = np.all(np.diag(np.diag(pk1.S)) == pk1.S)
Sa = (1 - a) * pk0.S + a * pk1.S
# make sure eigenvalues are positive
if np.any(np.isfinite(Sa) == 0):
print(Sa)
w, v = np.linalg.eig(Sa)
#assert(np.all(w > 0))
assert np.linalg.det(Sa) != 0
#if np.linalg.det(Sa) == 0:
# print Sa
# return float('Inf')
dm = pk1.m - pk0.m
# Use precise computation for diagonal covariance matrices
if p0S_is_diag and p1S_is_diag:
r = a / 2. * np.dot(np.dot(dm, np.linalg.inv(Sa)), dm) + \
(np.sum(np.log(np.diag(Sa))) - (1-a)*np.sum(np.log(np.diag(pk0.S))) - a*np.sum(np.log(np.diag(pk1.S)))) \
/ (1 - a) / 2.
else:
r = a / 2. * np.dot(np.dot(dm, np.linalg.inv(Sa)), dm) + \
(np.log(np.linalg.det(Sa)) - (1-a)*np.log(np.linalg.det(pk0.S)) - a*np.log(np.linalg.det(pk1.S))) \
/ (1 - a) / 2.
#assert(r > -1e-10)
return max(r, 0)
def __init__(self, plb, pub, qlb, qub):
"""
plb: a numpy array of lower bounds of p
pub: a numpy array of upper bounds of p
qlb: a numpy array of lower bounds of q
qub: a numpy array of upper bounds of q
"""
convertif = lambda a: np.array(a) if isinstance(a, list) else a
plb, pub, qlb, qub = list(map(convertif, [plb, pub, qlb, qub]))
if not np.all([
plb.shape[0] == pub.shape[0], pub.shape[0] == qlb.shape[0],
qlb.shape[0] == qub.shape[0]
]):
raise ValueError(
'all lower and upper bounds must have the same length')
if not np.all(pub - plb > 0):
raise ValueError(
'Require upper - lower to be positive. False for p')
if not np.all(qub - qlb > 0):
raise ValueError(
'Require upper - lower to be positive. False for q')
self.plb = plb
self.pub = pub
self.qlb = qlb
self.qub = qub
def get_stick_breaking_entropy(stick_propn_mean, stick_propn_info,
gh_loc, gh_weights):
# return the entropy of logitnormal distriibution on the sticks whose
# logit has mean stick_propn_mean and information stick_propn_info
# Integration is done on the real line with respect to the Lesbegue measure
# integration is done numerical with Gauss Hermite quadrature.
# gh_loc and gh_weights specifiy the location and weights of the
# quadrature points
# we seek E[log q(V)], where q is the density of a logit-normal, and
# V ~ logit-normal. Let W := logit(V), so W ~ Normal. Hence,
# E[log q(W)]; we can then decompose log q(x) into the terms of a normal
# distribution and the jacobian term. The expectation of the normal term
# evaluates to the normal entropy, and we add the jacobian term to it.
# The jacobian term is 1/(x(1-x)), so we simply add -EV - E(1-V) to the normal
# entropy.
assert np.all(gh_weights > 0)
assert stick_propn_mean.shape == stick_propn_info.shape
assert np.all(stick_propn_info) > 0
e_log_v, e_log_1mv =\
ef.get_e_log_logitnormal(
lognorm_means = stick_propn_mean,
lognorm_infos = stick_propn_info,
gh_loc = gh_loc,
gh_weights = gh_weights)
return np.sum(ef.univariate_normal_entropy(stick_propn_info)) + \
np.sum(e_log_v + e_log_1mv)
def m_step(self, expectations, datas, inputs, masks, tags, **kwargs):
from sklearn.linear_model import LinearRegression
D, M = self.D, self.M
for k in range(self.K):
xs, ys, weights = [], [], []
for (Ez, _), data, input in zip(expectations, datas, inputs):
xs.append(
np.hstack([
data[self.lags - l - 1:-l - 1]
for l in range(self.lags)
] + [input[self.lags:]]))
ys.append(data[self.lags:])
weights.append(Ez[self.lags:, k])
xs = np.concatenate(xs)
ys = np.concatenate(ys)
weights = np.concatenate(weights)
# Fit a weighted linear regression
lr = LinearRegression()
lr.fit(xs, ys, sample_weight=weights)
self.As[k], self.Vs[k], self.bs[
k] = lr.coef_[:, :D *
self.lags], lr.coef_[:, D *
self.lags:], lr.intercept_
assert np.all(np.isfinite(self.As))
assert np.all(np.isfinite(self.Vs))
assert np.all(np.isfinite(self.bs))
# Update the variances
yhats = lr.predict(xs)
sqerr = (ys - yhats)**2
self.inv_sigmas[k] = np.log(
np.average(sqerr, weights=weights, axis=0))
def planar_flow(z: np.ndarray,
w: np.ndarray,
u: np.ndarray,
b: Union[int, float],
h=np.tanh) -> np.ndarray:
"""Apply a planar flow to each element of `samples`
:param z: numpy array, samples to be transformed
Shape: (n_samples, n_dim)
:param u: numpy array, parameter of flow (N, D)
:param w: numpy array, parameter of flow (N, D)
:param b: numeric, parameter of flow (N,)
:param h: callable, non-linear function (default tanh)
:returns: numpy array, transformed samples
Transforms given samples according to the planar flow
:math:`f(z) = z + uh(w^Tz + b)`
"""
assert np.all(
np.array([w.shape[0], u.shape[0], b.shape[0]]) ==
z.shape[0]), 'Incorrect first dimension'
assert np.all(np.array([w.shape[1], u.shape[1]]) ==
z.shape[1]), "Incorrect second dimension"
u = _get_uhat(u, w)
assert np.all(
np.sum(u * w, axis=1)
) >= -1, f'Flow is not guaranteed to be invertible (u^Tw < -1: {w._value, u._value})'
assert np.all(
np.sum(u * w, axis=1)
) >= -1, f'Flow is not guaranteed to be invertible (u^Tw < -1: {w._value, u._value})'
res = z + np.sum(u * np.tanh(np.sum(z * w, axis=1) + b).reshape(-1, 1),
axis=1).reshape(z.shape[0], -1)
assert res.shape == z.shape, f'Incorrect output shape: {(res.shape)}'
return res
def myentropy(nn_model, weightlist, xdata, returnallp=False):
'''
Usage: for NN_Dropout, use the same weights, duplicated N times
for MFVI, pass the sampled weights
'''
#assert xdata.shape[0]==2
n_samples = xdata.shape[1]
p1narray = np.zeros((len(weightlist), n_samples)) #NWeightSamples x NPoints
if type(nn_model) != list:
for i, w in enumerate(weightlist):
w = np.reshape(w, (1, nn_model.D))
p1narray[i, :] = nn_model.forward(w, xdata) #assumes that the 'model.forward' is dropout-like and has generates different outputs for each i
elif type(nn_model) == list: # deterministic
for i, nn in enumerate(nn_model):
p1narray[i, :] = nn.forward(weightlist[i], xdata)
#print (p_here.shape)
# <<<<<<< HEAD
certainpts = np.logical_or(np.all(p1narray==0, axis=0), np.all(p1narray==1, axis=0))
p2narray = 1 - p1narray
p1narraym = np.mean(p1narray, axis=0)
p2narraym = np.mean(p2narray, axis=0)
Hpredcheck = -p1narraym*np.log(p1narraym) - p2narraym*np.log(p2narraym)
Hpredcheck[certainpts] = 0.0
if returnallp:
return p1narray, p1narraym, Hpredcheck
else:
return p1narraym, Hpredcheck
def BFGS(f, grad_f, hess_0, x):
dims = len(x)
H = hess_0
assert np.all(np.linalg.eig(H)[0] > 0) and np.all(H == H.T), "Initial hessian must be SPD"
positions = [x]
while np.linalg.norm(grad_f(x)) > 1e-7:
p = -np.matmul(np.linalg.inv(hess_0), grad_f(x))
# p /= np.linalg.norm(p)
a = get_line_length(f, grad_f, x, p, a_max=10)
x_next = x + a*p
s = x_next - x
y = grad_f(x_next) - grad_f(x)
rho = 1 / np.matmul(y, s)
# Note the second term is a scalar (rho) multiplied by a matrix
t1 = (np.identity(dims) - rho * np.matmul(s[:,np.newaxis], y[np.newaxis,:]))
t2 = (np.identity(dims) - rho * np.matmul(y[:,np.newaxis], s[np.newaxis,:]))
t3 = rho * np.matmul(s[:,np.newaxis], s[np.newaxis,:])
H = many_matmul(t1, H, t2) + t3
x = x_next
positions.append(x)
return np.array(positions)
def test_entropy_bound_vectorized_vs_not():
D = 2
N = 1
rs = npr.RandomState(0)
stepsize = 0.1
approx = False
xs = rs.randn(N,D)
gradfun = grad(logprob_two_moons)
gradfun_vec = elementwise_grad(logprob_two_moons)
new_xs = []
new_es = []
for i in xrange(N):
cur_x = np.reshape(xs[i], (1, D))
cur_new_x, cur_new_e = gradient_step_track_entropy_non_vectorized(gradfun, x=cur_x.copy(), stepsize=stepsize, rs=None, approx=approx)
new_xs.append(cur_new_x)
new_es.append(cur_new_e)
vec_new_xs, vec_new_es = gradient_step_track_entropy(gradfun_vec, xs=xs.copy(), stepsize=stepsize, rs=None, approx=approx)
for i in xrange(N):
assert np.all(vec_new_xs[i] == new_xs[i]), "vectorized: {} non-vectorized: {}".format(vec_new_xs[i], new_xs[i])
assert np.all(vec_new_es[i] == new_es[i]), "vectorized: {} non-vectorized: {}".format(vec_new_es, new_es)
def test_sigmoid():
x = np.linspace(-2, 2, 100)
y = sigmoid(x)
yidx = np.argsort(y)
xidx = np.argsort(x)
assert (np.all(np.logical_and(np.greater_equal(y, 0), np.less_equal(y,
1))))
assert (np.all(np.equal(yidx, xidx)))
def test_batch_softmax():
X = np.random.randint(1, 10, size=(10, 5))
p0 = np.stack(softmax(x_) for i, x_ in enumerate(X))
p1 = batch_softmax(X, axis=1)
p0_0 = np.stack(softmax(x_) for i, x_ in enumerate(X.T))
p1_0 = batch_softmax(X, axis=0)
assert np.all(np.equal(p0, p1))
assert np.all(np.equal(p0_0.round(4), p1_0.round(4).T))
def test_batch_transform():
X = np.random.normal(0, 5, size=(10, 2))
Y = batch_transform(X, [sigmoid, stable_exp])
assert (np.all(np.equal(X.shape, Y.shape)))
assert (np.all(
np.logical_and(np.greater_equal(Y[:, 0], 0), np.less_equal(Y[:, 0],
1))))
assert (np.all(np.greater_equal(Y[:, 0], 0)))
def grad_gtilde(z):
"""get the value of grad of gtilde at -z by intersection"""
assert np.all(z <= 0.0)
lognegz = np.log(-z)
assert np.all(lognegz <= _gtilde_neglogz_range[1]), (min(lognegz), max(lognegz), _gtilde_neglogz_range)
rval = _grad_gtilde_interp(lognegz)
rval[z==0] = _grad_gtilde_value_0
assert not np.any(np.isnan(rval).flatten()), (np.min(z), np.max(z), np.min(lognegz), np.max(lognegz))
return rval
def _get_permutations(from_ABs, to_ABs):
def recode(ABs):
return np.array([{"A": -1, "B": 1}[c] for c in ABs.upper()], dtype=int)
from_ABs = recode(from_ABs)
to_ABs = recode(to_ABs)
for permutation in it.permutations(range(len(from_ABs))):
curr = from_ABs[np.array(permutation)]
if np.all(curr == to_ABs) or np.all(curr == -1 * to_ABs):
yield permutation
def one_hot(z, K):
z = np.atleast_1d(z).astype(int)
assert np.all(z >= 0) and np.all(z < K)
shp = z.shape
N = z.size
zoh = np.zeros((N, K))
zoh[np.arange(N), np.arange(K)[np.ravel(z)]] = 1
zoh = np.reshape(zoh, shp + (K,))
return zoh
def gtilde(z):
"""get the value of gtilde at -z by intersection"""
assert isinstance(z, np.ndarray)
assert np.all(z <= 0.0)
lognegz = np.log(-z)
assert np.all(lognegz <= _gtilde_neglogz_range[1]), (min(lognegz), max(lognegz), _gtilde_neglogz_range)
rval = _gtilde_interp(lognegz)
rval[z==0] = _gtilde_value_0
rval[lognegz < _gtilde_neglogz[0]] = 0.0
assert np.all(~np.isnan(rval).flatten())
return rval
def test_optimize_locs_width(self):
"""
Test the function optimize_locs_width(..). Make sure it does not return
unusual results.
"""
# sample source
n = 600
dim = 2
seed = 17
ss = data.SSGaussMeanDiff(dim, my=1.0)
#ss = data.SSGaussVarDiff(dim)
#ss = data.SSSameGauss(dim)
# ss = data.SSBlobs()
dim = ss.dim()
dat = ss.sample(n, seed=seed)
tr, te = dat.split_tr_te(tr_proportion=0.5, seed=10)
xy_tr = tr.stack_xy()
# initialize test_locs by drawing the a Gaussian fitted to the data
# number of test locations
J = 3
V0 = util.fit_gaussian_draw(xy_tr, J, seed=seed + 1)
med = util.meddistance(xy_tr, subsample=1000)
gwidth0 = med**2
assert gwidth0 > 0
# optimize
V_opt, gw2_opt, opt_info = tst.GaussUMETest.optimize_locs_width(
tr,
V0,
gwidth0,
reg=1e-2,
max_iter=100,
tol_fun=1e-5,
disp=False,
locs_bounds_frac=100,
gwidth_lb=None,
gwidth_ub=None)
# perform the test using the optimized parameters on the test set
alpha = 0.01
ume_opt = tst.GaussUMETest(V_opt,
gw2_opt,
n_simulate=2000,
alpha=alpha)
test_result = ume_opt.perform_test(te)
assert test_result['h0_rejected']
assert util.is_real_num(gw2_opt)
assert gw2_opt > 0
assert np.all(np.logical_not((np.isnan(V_opt))))
assert np.all(np.logical_not((np.isinf(V_opt))))
def test_bms_verbose():
X = np.random.uniform(0, 200, size=(100, 10))
pxp, xp, bor, q_m, alpha, f0, f1, niter = bms(X, verbose=True)
assert np.equal(pxp.sum().round(5), 1)
assert np.equal(xp.sum().round(5), 1)
assert (0 <= bor <= 1)
assert np.all(np.equal(q_m.sum(1).round(5), 1))
assert np.all(np.greater(alpha, 0))
assert np.greater_equal(f0, 0)
assert np.greater_equal(f1, 0)
assert np.greater(niter, 0)
def fit(self,X,T,ranked_pairs,smoothed_pairs,ranked_pair_weights=None,smoothed_pair_weights=None):
"""
Fit the DSSL loss
Args:
X - (n_samples,n_features) ndarray:
Design matrix
T - (n_samples,) ndarray of:
Vector of continuous timestamps
ranked_pairs - (n_ranked_pairs,2) integer ndarray:
Contains ranked pairs of samples. Model will try to find
parameters such that score(ranked_pairs[i,0]) > score(ranked_pairs[i,1])
for all i.
smoothed_pairs - (n_smoothed_pairs,2) integer ndarray:
Contains pairs of samples that are close in time. Model will
try to find parameters such that minimizes
(score(ranked_pairs[i,0]) - score(ranked_pairs[i,1]))**2/(T(ranked_pairs[i,0]) - T(ranked_pairs[i,1]))**2
for all i.
ranked_pair_weights - (n_ranked_pairs,) float ndarray:
Contains sample weights for each of the ranked pairs.
smoothed_pair_weights - (n_smoothed_pairs,) float ndarray:
Contains sample weights for each of the smoothed pairs.
"""
assert X.shape[0] > 0
assert T.shape == (X.shape[0],)
assert ranked_pairs is None or np.issubdtype(ranked_pairs.dtype, np.dtype(int).type)
assert smoothed_pairs is None or np.issubdtype(smoothed_pairs.dtype, np.dtype(int).type)
assert ranked_pairs is None or np.all(np.logical_and(ranked_pairs >= 0,ranked_pairs <= X.shape[0]))
assert smoothed_pairs is None or np.all(np.logical_and(smoothed_pairs >= 0,smoothed_pairs <= X.shape[0]))
assert ranked_pairs is None or np.all(ranked_pairs[:,0] != ranked_pairs[:,1])
assert smoothed_pairs is None or np.all(smoothed_pairs[:,0] != smoothed_pairs[:,1])
# get obj
obj = self.get_obj(X,T,ranked_pairs,smoothed_pairs,ranked_pair_weights,smoothed_pair_weights)
# get the gradient function using autograd
gfun = grad(obj)
# init params
w0 = np.zeros(X.shape[1])
# optimize objective
self.res = minimize(obj,w0,method="L-BFGS-B",jac=gfun,options={"gtol":self.gtol,"maxiter":self.maxiter,"disp":self.disp},tol=self.tol)
self.set_params(self.res.x)
return self
def test_comparison_grads():
compare_funs = [
lambda x, y: np.sum(x < x) + 0.0, lambda x, y: np.sum(x <= y) + 0.0,
lambda x, y: np.sum(x > y) + 0.0, lambda x, y: np.sum(x >= y) + 0.0,
lambda x, y: np.sum(x == y) + 0.0, lambda x, y: np.sum(x != y) + 0.0
]
for arg1, arg2 in arg_pairs():
zeros = (arg1 + arg2) * 0 # get correct shape
for fun in compare_funs:
assert np.all(grad(fun)(arg1, arg2) == zeros)
assert np.all(grad(fun, argnum=1)(arg1, arg2) == zeros)
def solve_sylvester(a, b, q):
if a.shape == b.shape:
axes = (0, 2, 1) if a.ndim == 3 else (1, 0)
if np.all(a == b) and np.all(np.abs(a - np.transpose(a, axes)) < 1e-12):
eigvals, eigvecs = eigh(a)
if np.all(eigvals >= 1e-12):
tilde_q = np.transpose(eigvecs, axes) @ q @ eigvecs
tilde_x = tilde_q / (eigvals[..., :, None] + eigvals[..., None, :])
return eigvecs @ tilde_x @ np.transpose(eigvecs, axes)
return np.vectorize(
scipy.linalg.solve_sylvester, signature="(m,m),(n,n),(m,n)->(m,n)"
)(a, b, q)
def x_star(self):
if not hasattr(self, 'x_opt'):
self.x_opt = solve_qp(
P=self.Q,
q=self.q,
A=self.A if not np.all((self.A == 0)) else None,
b=self.b if not np.all((self.A == 0)) else
None, # check for A since b can be zero
G=self.G if not np.all((self.G == 0)) else None,
h=self.h if not np.all((self.G == 0)) else
None, # check for G since h can be zero
solver='cvxopt')
return self.x_opt
def test_comparison_grads():
compare_funs = [lambda x, y : np.sum(x < x) + 0.0,
lambda x, y : np.sum(x <= y) + 0.0,
lambda x, y : np.sum(x > y) + 0.0,
lambda x, y : np.sum(x >= y) + 0.0,
lambda x, y : np.sum(x == y) + 0.0,
lambda x, y : np.sum(x != y) + 0.0]
with warnings.catch_warnings(record=True) as w:
for arg1, arg2 in arg_pairs():
zeros = (arg1 + arg2) * 0 # get correct shape
for fun in compare_funs:
assert np.all(grad(fun)(arg1, arg2) == zeros)
assert np.all(grad(fun, argnum=1)(arg1, arg2) == zeros)
def init_multicomponent_source(sky_coord, frame, observation, bg_rms, flux_percentiles=None,
thresh=1., symmetric=True, monotonic=True):
"""Initialize multiple components
See `MultiComponentSource` for a description of the parameters
"""
if flux_percentiles is None:
flux_percentiles = [25]
# Initialize the first component as an extended source
sed, morph = init_extended_source(sky_coord, frame, observation, bg_rms,
thresh, symmetric, monotonic)
# create a list of components from base morph by layering them on top of
# each other so that they sum up to morph
K = len(flux_percentiles) + 1
Ny, Nx = morph.shape
morphs = np.zeros((K, Ny, Nx), dtype=morph.dtype)
morphs[0, :, :] = morph[:, :]
max_flux = morph.max()
percentiles_ = np.sort(flux_percentiles)
last_thresh = 0
for k in range(1, K):
perc = percentiles_[k - 1]
flux_thresh = perc * max_flux / 100
mask_ = morph > flux_thresh
morphs[k - 1][mask_] = flux_thresh - last_thresh
morphs[k][mask_] = morph[mask_] - flux_thresh
last_thresh = flux_thresh
# renormalize morphs: initially Smax
for k in range(K):
if np.all(morphs[k] <= 0):
msg = "Zero or negative morphology for component {} at y={}, x={}"
logger.warning(msg.format(k, *skycoords))
morphs[k] /= morphs[k].max()
# optimal SEDs given the morphologies, assuming img only has that source
seds = get_best_fit_seds(morphs, frame, observation)
for k in range(K):
if np.any(seds[k] <= 0):
# If the flux in all channels is <=0,
# the new sed will be filled with NaN values,
# which will cause the code to crash later
msg = "Zero or negative SED {} for component {} at y={}, x={}".format(seds[k], k, *sky_coord)
if np.all(sed <= 0):
logger.warning(msg)
else:
logger.info(msg)
return seds, morphs
def check_if_selection(r_to_keep, r, k_to_keep, k, L, new_Lt):
''' Check if the architecture has changed during the selection procedure
r_to_keep (dict): The dimensions selected in the network
r (dict): The original dimensions of the network layers
k_to_keep (dict): The components selected in the network
k (dict): The original number of component on each layer
L (dict): The number of layers in the networks
new_Lt (int): The selected number of layers on the common tail.
--------------------------------------------------------------------------
returns (Bool): Whether or not the procedure has selected a new architecture
'''
is_L_unchanged = (L['t'] == new_Lt)
is_rd_unchanged = np.all([len(r_to_keep['d'][l]) == r['d'][l] for l in range(L['d'])])
is_rt_unchanged = np.all([len(r_to_keep['t'][l]) == r['t'][l] for l in range(new_Lt)])
is_rc_unchanged = np.all([len(r_to_keep['c'][l]) == r['c'][l] for l in range(L['c'] + 1)])
is_r_unchanged = np.all([is_rd_unchanged, is_rc_unchanged, is_rt_unchanged])
is_kd_unchanged = np.all([len(k_to_keep['d'][l]) == k['d'][l] for l in range(L['d'])])
is_kt_unchanged = np.all([len(k_to_keep['t'][l]) == k['t'][l] for l in range(new_Lt)])
is_kc_unchanged = np.all([len(k_to_keep['c'][l]) == k['c'][l] for l in range(L['c'] + 1)])
is_k_unchanged = np.all([is_kd_unchanged, is_kc_unchanged, is_kt_unchanged])
is_selection = not(is_r_unchanged & is_k_unchanged & is_L_unchanged)
return is_selection
def is_min_architecture_reached(k, r, n_clusters):
'''
k (dict of list): The number of components on each layer of each head and tail
r (dict of list): The dimensions of each layer of each head and tail
n_clusters (int): The number of clusters to look for in the data
------------------------------------------------------------------------
returns (Bool): True if the minimal network architecture has been reached
False otherwise
'''
# Check that common tail is minimal
first_layer_k_min = k['t'][0] == n_clusters # geq or eq ?
following_layers_k_min = np.all([kk <= 2 for kk in k['t'][1:]])
is_tail_k_min = first_layer_k_min & following_layers_k_min
is_tail_r_min = r['t'] == [2, 1]
is_tail_min = is_tail_k_min & is_tail_r_min
# Check that the heads are minimal
is_head_k_min = {'c': True, 'd': True}
is_head_r_min = {'c': True, 'd': True}
Lt = len(k['t'])
for h in ['c', 'd']:
Lh = len(k[h])
# If more than one layer on head, then arch is not minimal
if Lh >= 2:
is_head_k_min[h] = False
is_head_r_min[h] = False
continue
for l in range(Lh):
# If all k >= 2
if k[h][l] > 1:
is_head_k_min[h] = False
# The first dimension of the continuous dimension is imposed
if (h == 'd') | (l > 0):
# k is min if r3 = r2 + 1 = r1 + 2 ...
if r[h][l] > Lh + Lt - l:
is_head_r_min[h] = False
are_heads_min = np.all(list(is_head_k_min.values())) \
& np.all(list(is_head_r_min.values()))
is_arch_min = are_heads_min & is_tail_min
return is_arch_min
def params(self, value):
Ps, rs, ps = value
assert Ps.shape == (self.K, self.K)
assert np.allclose(np.diag(Ps), 0)
assert np.allclose(Ps.sum(1), 1)
assert rs.shape == (self.K)
assert rs.dtype == int
assert np.all(rs > 0)
assert ps.shape == (self.K)
assert np.all(ps > 0)
assert np.all(ps < 1)
self.Ps, self.rs, self.ps = Ps, rs, ps
# Reset the transition matrix
self._transition_matrix = None
def test_basic(self):
"""
Nothing special. Just test basic things.
"""
# sample
n = 10
d = 3
with util.NumpySeedContext(seed=29):
X = np.random.randn(n, d) * 3
k = kernel.KGauss(sigma2=1)
K = k.eval(X, X)
self.assertEqual(K.shape, (n, n))
self.assertTrue(np.all(K >= 0 - 1e-6))
self.assertTrue(np.all(K <= 1 + 1e-6), "K not bounded by 1")
def _invert(self, data, input=None, mask=None, tag=None):
"""
Approximate invert the linear emission model with the pseudoinverse
y = Cx + d + noise; C orthogonal.
xhat = (C^T C)^{-1} C^T (y-d)
"""
assert self.single_subspace, "Can only invert with a single emission model"
C, F, d = self.Cs[0], self.Fs[0], self.ds[0]
C_pseudoinv = np.linalg.solve(C.T.dot(C), C.T).T
# Account for the bias
bias = input.dot(F.T) + d
if not np.all(mask):
data = interpolate_data(data, mask)
# We would like to find the PCA coordinates in the face of missing data
# To do so, alternate between running PCA and imputing the missing entries
for itr in range(25):
mu = (data - bias).dot(C_pseudoinv)
data[:, ~mask[0]] = (mu.dot(C.T) + bias)[:, ~mask[0]]
# Project data to get the mean
return (data - bias).dot(C_pseudoinv)
def test_softmax_partition_potential():
x = np.arange(10).astype(np.float)
B = 1.5
potential, partition = softmax_components(B * x)
component_sm = potential / partition
original_sm = softmax(B * x)
assert (np.all(component_sm == original_sm))
def test_getquantile():
x = np.linspace(0, 1, 100)
y = getquantile(x, lower=0.25, upper=0.5)
assert (np.equal(y.mean(), 0.25))
y = getquantile(x, lower=0.25, upper=0.5, return_indices=True)
assert (np.all(np.equal(y, np.arange(25, 50))))
def initialize(self, datas, inputs=None, masks=None, tags=None):
data = np.concatenate(datas)
ddata = np.concatenate([np.gradient(d, axis=0) for d in datas])
ddata = (ddata - ddata.mean(0)) / ddata.std(0)
input = np.concatenate(inputs)
T = data.shape[0]
# Cluster the data and its gradient before initializing
from sklearn.cluster import KMeans
km = KMeans(self.K)
# km.fit(np.column_stack((data, ddata)))
km.fit(data)
z = km.labels_[:-self.lags]
from sklearn.linear_model import LinearRegression
for k in range(self.K):
ts = np.where(z == k)[0]
x = np.column_stack([data[ts + l]
for l in range(self.lags)] + [input[ts]])
y = data[ts + self.lags]
lr = LinearRegression().fit(x, y)
self.As[k] = lr.coef_[:, :self.D * self.lags]
self.Vs[k] = lr.coef_[:, self.D * self.lags:]
self.bs[k] = lr.intercept_
resid = y - lr.predict(x)
sigmas = np.var(resid, axis=0)
self.inv_sigmas[k] = np.log(sigmas + 1e-16)
assert np.all(np.isfinite(self.inv_sigmas))
def test_flatten_dict():
val = {'k': npr.random((4, 4)),
'k2': npr.random((3, 3)),
'k3': 3.0,
'k4': [1.0, 4.0, 7.0, 9.0]}
vect, unflatten = flatten(val)
val_recovered = unflatten(vect)
vect_2, _ = flatten(val_recovered)
assert np.all(vect == vect_2)
def test_optimize_locs_width(self):
"""
Test the function optimize_locs_width(..). Make sure it does not return
unusual results.
"""
# sample source
n = 600
dim = 2
seed = 17
ss = data.SSGaussMeanDiff(dim, my=1.0)
#ss = data.SSGaussVarDiff(dim)
#ss = data.SSSameGauss(dim)
# ss = data.SSBlobs()
dim = ss.dim()
dat = ss.sample(n, seed=seed)
tr, te = dat.split_tr_te(tr_proportion=0.5, seed=10)
xy_tr = tr.stack_xy()
# initialize test_locs by drawing the a Gaussian fitted to the data
# number of test locations
J = 3
V0 = util.fit_gaussian_draw(xy_tr, J, seed=seed+1)
med = util.meddistance(xy_tr, subsample=1000)
gwidth0 = med**2
assert gwidth0 > 0
# optimize
V_opt, gw2_opt, opt_info = tst.GaussUMETest.optimize_locs_width(tr, V0, gwidth0, reg=1e-2,
max_iter=100, tol_fun=1e-5, disp=False, locs_bounds_frac=100,
gwidth_lb=None, gwidth_ub=None)
# perform the test using the optimized parameters on the test set
alpha = 0.01
ume_opt = tst.GaussUMETest(V_opt, gw2_opt, n_simulate=2000, alpha=alpha)
test_result = ume_opt.perform_test(te)
assert test_result['h0_rejected']
assert util.is_real_num(gw2_opt)
assert gw2_opt > 0
assert np.all(np.logical_not((np.isnan(V_opt))))
assert np.all(np.logical_not((np.isinf(V_opt))))
def test_flatten():
r = np.random.randn
x = (1.0, r(2,3), [r(1,4), {'x': 2.0, 'y': r(4,2)}])
x_flat, unflatten = flatten(x)
assert x_flat.shape == (20,)
assert x_flat[0] == 1.0
assert np.all(x_flat == flatten(unflatten(x_flat))[0])
y = (1.0, 2.0, [3.0, {'x': 2.0, 'y': 4.0}])
y_flat, unflatten = flatten(y)
assert y_flat.shape == (5,)
assert y == unflatten(y_flat)
def log_likelihood(self, u=None, fluxes=None, shape=None):
""" conditional likelihood of source conditioned on photon
sampled images """
# args passed in or from source's own params
u = self.params.u if u is None else u
fluxes = self.params.fluxes if fluxes is None else fluxes
shape = self.params.shape if shape is None else shape
assert np.all(~np.isnan(fluxes)), 'passing in NAN fluxes.'
# for each source image, compute per pixel poisson likelihood term
ll = 0
for n, (samp_img, fits_img, pixel_grid) in enumerate(self.sample_image_list):
# grab the patch the sample_image corresponds to (every other
# pixel is a zero value)
ylim, xlim = (samp_img.y0, samp_img.y1), \
(samp_img.x0, samp_img.x1)
# photon scatter image (from psf and galaxy extent) aligned w/ patch
psf_ns, _, _ = \
self.compute_scatter_on_pixels(fits_image=fits_img,
u=u, shape=shape,
xlim=xlim, ylim=ylim,
pixel_grid=pixel_grid)
# convert parameter flux to fits_image specific photon count flux
band_flux = self.flux_in_image(fits_img, fluxes=fluxes)
if psf_ns is None:
ll_img = - band_flux * np.sum(fits_img.weights)
ll += ll_img
continue
# compute model patch means and the sum of means outside patch (should be small...)
model_patch = band_flux * psf_ns
# compute poisson likelihood of each pixel - note that
# the last term would be sum(model_patch) - model_outside, which is
# just equal to band_flux * sum(fits_img.weights)
mask = (model_patch > 0.)
ll_img = np.sum( np.log(model_patch[mask]) *
np.array(samp_img.data)[mask] ) - \
band_flux*np.sum(fits_img.weights)
### debug
if np.isnan(ll_img):
print "NAN"
print "model patch zeros: ", np.sum(model_patch==0)
print "band flux ", band_flux
ll += ll_img
return ll
def natural_predict(J, h, J11, J12, J22, logZ):
# convert from natural parameter to the usual J definitions
J, J11, J12, J22 = -2*J, -2*J11, -J12, -2*J22
L = np.linalg.cholesky(J + J11)
v = solve_triangular(L, h)
lognorm = 1./2*np.dot(v,v) - np.sum(np.log(np.diag(L)))
h_predict = -np.dot(J12.T, solve_triangular(L, v, trans='T'))
temp = solve_triangular(L, J12)
J_predict = J22 - np.dot(temp.T, temp)
assert np.all(np.linalg.eigvals(J_predict) > 0)
return (-1./2*J_predict, h_predict), lognorm + logZ
def fit(self, samples):
import sklearn.mixture
m = sklearn.mixture.GMM(self.num_components, "full")
m.fit(samples)
self.comp_lprior = log(m.weights_)
self.dist_cat = categorical(exp(self.comp_lprior))
self.comp_dist = [mvt(m.means_[i], m.covars_[i], self.df) for i in range(self.comp_lprior.size)]
self.dim = m.means_[0].size
#self._e_step()
if False:
old = -1
i = 0
while not np.all(old == self.resp):
i += 1
old = self.resp.copy()
self._e_step()
self._m_step()
print(np.sum(old == self.resp)/self.resp.size)
#print("Convergence after",i,"iterations")
self.dist_cat = categorical(exp(self.comp_lprior))
def _set_transmat(self, transmat_val):
if transmat_val is None:
transmat = np.tile(1.0 / self.n_components,
(self.n_components, self.n_components))
else:
transmat_val[np.isnan(transmat_val)] = 0.0
normalize(transmat_val, axis=1)
if (np.asarray(transmat_val).shape == (self.n_components,
self.n_components)):
transmat = np.copy(transmat_val)
elif transmat_val.shape[0] == self.n_unique:
transmat = self._ntied_transmat(transmat_val)
else:
raise ValueError("cannot match shape of transmat")
if not np.all(np.allclose(np.sum(transmat, axis=1), 1.0)):
raise ValueError('Rows of transmat must sum to 1.0')
self._log_transmat = np.log(np.asarray(transmat).copy())
underflow_idx = np.isnan(self._log_transmat)
self._log_transmat[underflow_idx] = NEGINF
def fit_gaussian_draw(X, J, seed=28, reg=1e-7, eig_pow=1.0):
"""
Fit a multivariate normal to the data X (n x d) and draw J points
from the fit.
- reg: regularizer to use with the covariance matrix
- eig_pow: raise eigenvalues of the covariance matrix to this power to construct
a new covariance matrix before drawing samples. Useful to shrink the spread
of the variance.
"""
with NumpySeedContext(seed=seed):
d = X.shape[1]
mean_x = np.mean(X, 0)
cov_x = np.cov(X.T)
if d==1:
cov_x = np.array([[cov_x]])
[evals, evecs] = np.linalg.eig(cov_x)
evals = np.maximum(0, np.real(evals))
assert np.all(np.isfinite(evals))
evecs = np.real(evecs)
shrunk_cov = evecs.dot(np.diag(evals**eig_pow)).dot(evecs.T) + reg*np.eye(d)
V = np.random.multivariate_normal(mean_x, shrunk_cov, J)
return V
def compute_rotated_map(self, rotation):
"""
Compute stellar maps projected on the plane of the sky for a given rotation of the star
Args:
rotation (float) : rotation around the star in degrees given as [longitude, latitude] in degrees
Returns:
pixel_unique (int) : vector with the "active" healpix pixels
pixel_map (int) : map showing the healpix pixel projected on the plane of the sky
mu_pixel (float): map of the astrocentric angle for each pixel on the plane of the sky (zero for pixels not in the star)
T_pixel (float): map of temperatures for each pixel on the plane of the sky
"""
mu_pixel = np.zeros_like(self.mu_angle)
T_pixel = np.zeros_like(self.mu_angle)
# Get the projection of the healpix pixel indices on the plane of the sky
pixel_map = self.projector.projmap(self.indices, self.f_vec2pix, rot=rotation)[:,0:int(self.npix/2)]
# Get the unique elements in the vector
pixel_unique = np.unique(pixel_map)
# Now loop over all unique pixels, filling up the array of the projected map with the mu and temeperature values
for j in range(len(pixel_unique)):
ind = np.where(pixel_map == pixel_unique[j])
if (np.all(np.isfinite(self.mu_angle[ind[0],ind[1]]))):
if (self.mu_angle[ind[0],ind[1]].size == 0):
value = 0.0
else:
value = np.nanmean(self.mu_angle[ind[0],ind[1]])
mu_pixel[ind[0],ind[1]] = value
T_pixel[ind[0],ind[1]] = self.temperature_map[int(pixel_unique[j])]
else:
mu_pixel[ind[0],ind[1]] = 0.0
T_pixel[ind[0],ind[1]] = 0.0
return pixel_unique, pixel_map, mu_pixel, T_pixel
def test_comparison_grads():
compare_funs = [lambda x, y : np.sum(x < x),
lambda x, y : np.sum(x <= y),
lambda x, y : np.sum(x > y),
lambda x, y : np.sum(x >= y),
lambda x, y : np.sum(x == y),
lambda x, y : np.sum(x != y)]
for arg1, arg2 in arg_pairs():
zeros = (arg1 + arg2) * 0 # get correct shape
for fun in compare_funs:
d_fun_0 = lambda x, y : to_scalar(grad(fun, 0)(x, y))
d_fun_1 = lambda x, y : to_scalar(grad(fun, 1)(x, y))
assert np.all(grad(fun)(arg1, arg2) == zeros)
assert np.all(grad(fun, argnum=1)(arg1, arg2) == zeros)
assert np.all(grad(d_fun_0)(arg1, arg2) == zeros)
assert np.all(grad(d_fun_1, argnum=1)(arg1, arg2) == zeros)
assert np.all(grad(d_fun_0)(arg1, arg2) == zeros)
assert np.all(grad(d_fun_1, argnum=1)(arg1, arg2) == zeros)
def precompute_rotation_maps(self, rotations=None):
"""
Compute the averaged spectrum on the star for a given temperature map and for a given rotation
Args:
rotations (float) : [N_phases x 2] giving [longitude, latitude] in degrees for each phase
Returns:
None
"""
if (rotations is None):
print("Use some angles for the rotations")
return
self.n_phases = rotations.shape[0]
self.avg_mu = [None] * self.n_phases
self.avg_v = [None] * self.n_phases
self.velocity = [None] * self.n_phases
self.n_pixel_unique = [None] * self.n_phases
self.n_pixels = [None] * self.n_phases
self.pixel_unique = [None] * self.n_phases
for loop in range(self.n_phases):
mu_pixel = np.zeros_like(self.mu_angle)
v_pixel = np.zeros_like(self.vel_projection)
pixel_map = self.projector.projmap(self.indices, self.f_vec2pix, rot=rotations[loop,:])[:,0:int(self.npix/2)]
pixel_unique = np.unique(pixel_map[np.isfinite(pixel_map)])
for j in range(len(pixel_unique)):
ind = np.where(pixel_map == pixel_unique[j])
if (np.all(np.isfinite(self.mu_angle[ind[0],ind[1]]))):
if (self.mu_angle[ind[0],ind[1]].size == 0):
mu_pixel[ind[0],ind[1]] = 0.0
v_pixel[ind[0],ind[1]] = 0.0
else:
if (self.clv):
value = np.nanmean(self.mu_angle[ind[0],ind[1]])
else:
value = 1.0
mu_pixel[ind[0],ind[1]] = value
value = np.nanmean(self.vel_projection[ind[0],ind[1]])
v_pixel[ind[0],ind[1]] = value
else:
mu_pixel[ind[0],ind[1]] = 0.0
v_pixel[ind[0],ind[1]] = 0.0
self.n_pixel_unique[loop] = len(pixel_unique)
self.avg_mu[loop] = np.zeros(self.n_pixel_unique[loop])
self.avg_v[loop] = np.zeros(self.n_pixel_unique[loop])
self.velocity[loop] = np.zeros(self.n_pixel_unique[loop])
self.n_pixels[loop] = np.zeros(self.n_pixel_unique[loop], dtype='int')
self.pixel_unique[loop] = pixel_unique.astype('int')
for i in range(len(pixel_unique)):
ind = np.where(pixel_map == pixel_unique[i])
self.n_pixels[loop][i] = len(ind[0])
self.avg_mu[loop][i] = np.unique(mu_pixel[ind[0], ind[1]])
self.avg_v[loop][i] = np.unique(v_pixel[ind[0], ind[1]])
self.velocity[loop][i] = self.avg_mu[loop][i] * self.avg_v[loop][i]
def test_flatten_complex():
val = 1 + 1j
flat, unflatten = flatten(val)
assert np.all(val == unflatten(flat))
def test_flatten_unflatten():
for vs in everything:
v = npr.randn(vs.size)
v2 = vs.flatten(vs.unflatten(v))
assert np.all(v2 == v), \
report_flatten_unflatten(vs, v, v2)
def test_flatten_empty():
val = (npr.randn(4), [npr.randn(3,4), 2.5], (), (2.0, [1.0, npr.randn(2)]))
vect, unflatten = flatten(val)
val_recovered = unflatten(vect)
vect_2, _ = flatten(val_recovered)
assert np.all(vect == vect_2)
test_sampler = np.random.permutation(len(test_labels))[:100]
train_sampler = np.random.permutation(len(train_labels))[:1000]
train_images = train_images[train_sampler]
train_labels = train_labels[train_sampler]
test_images = test_images[test_sampler]
test_labels = test_labels[test_sampler]
N_data = train_images.shape[0]
from sklearn import neighbors
train_images_3 = np.reshape(train_images, (1000,28*28))
test_images_3 = np.reshape(test_images, (100,28*28))
clf = neighbors.KNeighborsClassifier()
clf.fit(train_images_3,train_labels)
res = clf.predict(test_images_3)
res2 = [np.all(x) for x in (res==test_labels)]
print ('KNN')
print ( np.sum(res2) / 100.)
# Make neural net functions
N_weights, pred_fun, loss_fun, \
frac_err = make_nn_funs(input_shape, layer_specs, L2_reg)
loss_grad = grad(loss_fun)
# Initialize weights
rs = npr.RandomState()
W = rs.randn(N_weights) * param_scale
print(" Epoch | Train err | Test error ")
def print_perf(epoch, W):
test_perf = frac_err(W, test_images, test_labels)
def unflatten_tracing():
val = [npr.randn(4), [npr.randn(3,4), 2.5], (), (2.0, [1.0, npr.randn(2)])]
vect, unflatten = flatten(val)
def f(vect): return unflatten(vect)
flatten2, _ = make_vjp(f)(vect)
assert np.all(vect == flatten2(val))
def is_posdef(A):
return np.allclose(A, A.T) and np.all(np.linalg.eigvalsh(A) > 0.)
def is_psd(A):
return np.allclose(A, A.T) and np.all(np.linalg.eigvalsh(A) >= 0.)
def natural_condition_on_general(J, h, Jo, ho, logZo):
Jo = Jo if Jo.ndim == 2 else np.diag(Jo)
assert np.all(np.linalg.eigvals(-2*(J + Jo)) > 0)
return (J + Jo, h + ho), logZo