def sampler_w(n, wts, pts):
if wts is None or pts is None or pts.shape[0] == 0:
muw = mu0
USigw = np.linalg.cholesky(Sig0) #Note: USigw is lower triangular here, below is upper tri. Doesn't matter, just need Sigw = MM^T
else:
muw, USigw, _ = model_linreg.weighted_post(mu0, Sig0inv, datastd**2, pts, wts)
return muw + np.random.randn(n, muw.shape[0]).dot(USigw.T)
def sampler_w(n, wts, pts):
if pts.shape[0] == 0:
wts = np.zeros(1)
pts = np.zeros((1, Z.shape[1]))
muw, LSigw, LSigwInv = model_linreg.weighted_post(mu0, Sig0inv, datastd**2,
pts, wts)
return muw + np.random.randn(n, muw.shape[0]).dot(LSigw.T)
def update(self, wts, pts):
if wts is None or pts is None or pts.shape[0] == 0:
self.muw = mu0
self.USigw = np.linalg.cholesky(
Sig0
) #Note: USigw here is lower triangular, but keeping naming convention for below stuff. Doesn't matter, just need Sigw = MM^T
else:
self.muw, self.USigw, _ = model_linreg.weighted_post(
mu0, Sig0inv, datastd**2, pts, wts)
def tsf_exact_w(wts, idcs):
w = np.zeros(X.shape[0])
w[idcs] = wts
muw, Sigw = model_linreg.weighted_post(mu0, Sig0inv, datastd ** 2, X, Y, w)
lmb, V = np.linalg.eigh(Sigw)
beta = X.dot(V * np.sqrt(np.maximum(lmb, 0.)))
nu = Y - X.dot(muw)
# project the matrix term down to 20*20 = 400 dimensions
lmb, V = np.linalg.eigh(beta.T.dot(beta))
n_dim = 20
beta_proj = beta.dot(V[:, -n_dim:])
return np.hstack((nu[:, np.newaxis] * beta,
1. / np.sqrt(2.) * (beta_proj[:, :, np.newaxis] * beta_proj[:, np.newaxis, :]).reshape(
beta.shape[0], n_dim ** 2))) / datastd ** 2
idcs = np.random.choice(np.arange(x.shape[0]),
replace=False,
size=basis_unique_counts[i])
basis_locs = np.vstack((basis_locs, x[idcs, :2]))
print('Converting bases and observations into X/Y matrices')
#convert basis functions + observed data locations into a big X matrix
X = np.zeros((x.shape[0], basis_scales.shape[0]))
for i in range(basis_scales.shape[0]):
X[:, i] = np.exp(-((x[:, :2] - basis_locs[i, :])**2).sum(axis=1) /
(2 * basis_scales[i]**2))
Y = x[:, 2]
#get true posterior
print('Computing true posterior')
mup, Sigp = model_linreg.weighted_post(mu0, Sig0inv, datastd**2, X, Y,
np.ones(X.shape[0]))
Sigpinv = np.linalg.inv(Sigp)
#create function to output log_likelihood given param samples
print('Creating log-likelihood function')
log_likelihood = lambda samples: model_linreg.potentials(
datastd**2, X, Y, samples)
#create tangent space for well-tuned Hilbert coreset alg
print('Creating tuned tangent space for Hilbert coreset construction')
sampler_optimal = lambda n, w, ids: np.random.multivariate_normal(mup, Sigp, n)
tsf_optimal = bc.BayesianTangentSpaceFactory(log_likelihood, sampler_optimal,
proj_dim)
#create tangent space for poorly-tuned Hilbert coreset alg
print('Creating untuned tangent space for Hilbert coreset construction')
def run(arguments):
# check if result already exists for this run, and if so, quit
if results.check_exists(arguments):
print('Results already exist for arguments ' + str(arguments))
print('Quitting.')
quit()
#######################################
#######################################
## Step 0: Setup
#######################################
#######################################
np.random.seed(arguments.trial)
bc.util.set_verbosity(arguments.verbosity)
if arguments.coreset_size_spacing == 'log':
Ms = np.unique(
np.logspace(0.,
np.log10(arguments.coreset_size_max),
arguments.coreset_num_sizes,
dtype=np.int32))
else:
Ms = np.unique(
np.linspace(1,
arguments.coreset_size_max,
arguments.coreset_num_sizes,
dtype=np.int32))
#make sure the first size to record is 0
if Ms[0] != 0:
Ms = np.hstack((0, Ms))
#######################################
#######################################
## Step 1: Load and preprocess data
#######################################
#######################################
#load data and compute true posterior
#each row of x is [lat, lon, price]
print('Loading data')
x = np.load('../data/prices2018.npy')
print('dataset size : ', x.shape)
print('Subsampling down to ' + str(arguments.data_num) + ' points')
idcs = np.arange(x.shape[0])
np.random.shuffle(idcs)
x = x[idcs[:arguments.data_num], :]
#log transform the prices
x[:, 2] = np.log10(x[:, 2])
#get empirical mean/std
datastd = x[:, 2].std()
datamn = x[:, 2].mean()
#bases of increasing size; the last one is effectively a constant
basis_unique_scales = np.array([.2, .4, .8, 1.2, 1.6, 2., 100])
basis_unique_counts = np.hstack(
(arguments.n_bases_per_scale * np.ones(6, dtype=np.int64), 1))
#the dimension of the scaling vector for the above bases
d = basis_unique_counts.sum()
print('Basis dimension: ' + str(d))
#model params
mu0 = datamn * np.ones(d)
Sig0 = (datastd**2 + datamn**2) * np.eye(d)
Sig0inv = np.linalg.inv(Sig0)
#generate basis functions by uniformly randomly picking locations in the dataset
print('Trial ' + str(arguments.trial))
print('Creating bases')
basis_scales = np.array([])
basis_locs = np.zeros((0, 2))
for i in range(basis_unique_scales.shape[0]):
basis_scales = np.hstack(
(basis_scales,
basis_unique_scales[i] * np.ones(basis_unique_counts[i])))
idcs = np.random.choice(np.arange(x.shape[0]),
replace=False,
size=basis_unique_counts[i])
basis_locs = np.vstack((basis_locs, x[idcs, :2]))
print('Converting bases and observations into X/Y matrices')
#convert basis functions + observed data locations into a big X matrix
X = np.zeros((x.shape[0], basis_scales.shape[0]))
for i in range(basis_scales.shape[0]):
X[:, i] = np.exp(-((x[:, :2] - basis_locs[i, :])**2).sum(axis=1) /
(2 * basis_scales[i]**2))
Y = x[:, 2]
Z = np.hstack((X, Y[:, np.newaxis]))
_, bV = np.linalg.eigh(X.T.dot(X))
bV = bV[:, -arguments.proj_dim:]
#######################################
#######################################
## Step 2: Calculate Likelihoods/Projectors
#######################################
#######################################
#get true posterior
print('Computing true posterior')
mup, USigp, LSigpInv = model_linreg.weighted_post(mu0, Sig0inv, datastd**2,
Z, np.ones(X.shape[0]))
Sigp = USigp.dot(USigp.T)
SigpInv = LSigpInv.dot(LSigpInv.T)
#create function to output log_likelihood given param samples
print('Creating log-likelihood function')
log_likelihood = lambda z, th: model_linreg.log_likelihood(
z, th, datastd**2)
print('Creating gradient log-likelihood function')
grad_log_likelihood = lambda z, th: model_linreg.grad_x_log_likelihood(
z, th, datastd**2)
#create tangent space for well-tuned Hilbert coreset alg
print('Creating tuned projector for Hilbert coreset construction')
sampler_optimal = lambda n, w, pts: mup + np.random.randn(n, mup.shape[0]
).dot(USigp.T)
prj_optimal = bc.BlackBoxProjector(sampler_optimal, arguments.proj_dim,
log_likelihood, grad_log_likelihood)
#create tangent space for poorly-tuned Hilbert coreset alg
print('Creating untuned projector for Hilbert coreset construction')
Zhat = Z[np.random.randint(0, Z.shape[0], int(np.sqrt(Z.shape[0]))), :]
muhat, USigHat, LSigHatInv = model_linreg.weighted_post(
mu0, Sig0inv, datastd**2, Zhat, np.ones(Zhat.shape[0]))
sampler_realistic = lambda n, w, pts: muhat + np.random.randn(
n, muhat.shape[0]).dot(USigHat.T)
prj_realistic = bc.BlackBoxProjector(sampler_realistic, arguments.proj_dim,
log_likelihood, grad_log_likelihood)
print('Creating black box projector')
def sampler_w(n, wts, pts):
if wts is None or pts is None or pts.shape[0] == 0:
muw = mu0
USigw = np.linalg.cholesky(
Sig0
) #Note: USigw is lower triangular here, below is upper tri. Doesn't matter, just need Sigw = MM^T
else:
muw, USigw, _ = model_linreg.weighted_post(mu0, Sig0inv,
datastd**2, pts, wts)
return muw + np.random.randn(n, muw.shape[0]).dot(USigw.T)
prj_bb = bc.BlackBoxProjector(sampler_w, arguments.proj_dim,
log_likelihood, grad_log_likelihood)
print('Creating exact projectors')
##############################
###Exact projection in SparseVI for gradient computation
#for this model we can do the tangent space projection exactly
class LinRegProjector(bc.Projector):
def __init__(self, bV):
self.bV = bV
def project(self, pts, grad=False):
X = pts[:, :-1]
Y = pts[:, -1]
#beta = X.dot(self.V*np.sqrt(np.maximum(self.lmb, 0.)))
beta = X.dot(self.USigw)
nu = Y - X.dot(self.muw)
#approximation to avoid high memory cost: project the matrix term down to bV.shape[1]**2 dimensions
beta_proj = beta.dot(self.bV)
#lmb2, V2 = np.linalg.eigh(beta.T.dot(beta))
#beta_proj = beta.dot(V2[:, -arguments.proj_dim:])
return np.hstack(
(nu[:, np.newaxis] * beta, 1. / np.sqrt(2.) *
(beta_proj[:, :, np.newaxis] * beta_proj[:, np.newaxis, :]).
reshape(beta.shape[0], arguments.proj_dim**2))) / datastd**2
def update(self, wts, pts):
if wts is None or pts is None or pts.shape[0] == 0:
self.muw = mu0
self.USigw = np.linalg.cholesky(
Sig0
) #Note: USigw here is lower triangular, but keeping naming convention for below stuff. Doesn't matter, just need Sigw = MM^T
else:
self.muw, self.USigw, _ = model_linreg.weighted_post(
mu0, Sig0inv, datastd**2, pts, wts)
#if pts.shape[0] == 0:
# self.muw = mu0
# self.Sigw = Sig0
#else:
# self.muw, self.Sigw = model_linreg.weighted_post(mu0, Sig0inv, datastd**2, pts, wts)
#self.lmb, self.V = np.linalg.eigh(self.LSigw.dot(self.LSigw.T))
prj_optimal_exact = LinRegProjector(bV)
prj_optimal_exact.update(np.ones(Z.shape[0]), Z)
prj_realistic_exact = LinRegProjector(bV)
prj_realistic_exact.update(np.ones(Zhat.shape[0]), Zhat)
#######################################
#######################################
## Step 3: Construct Coreset
#######################################
#######################################
##############################
print('Creating coreset construction objects')
#create coreset construction objects
sparsevi_exact = bc.SparseVICoreset(Z,
LinRegProjector(bV),
opt_itrs=arguments.opt_itrs,
step_sched=eval(arguments.step_sched))
sparsevi = bc.SparseVICoreset(Z,
prj_bb,
opt_itrs=arguments.opt_itrs,
step_sched=eval(arguments.step_sched))
giga_optimal = bc.HilbertCoreset(Z, prj_optimal)
giga_optimal_exact = bc.HilbertCoreset(Z, prj_optimal_exact)
giga_realistic = bc.HilbertCoreset(Z, prj_realistic)
giga_realistic_exact = bc.HilbertCoreset(Z, prj_realistic_exact)
unif = bc.UniformSamplingCoreset(Z)
algs = {
'SVI-EXACT': sparsevi_exact,
'SVI': sparsevi,
'GIGA-OPT': giga_optimal,
'GIGA-OPT-EXACT': giga_optimal_exact,
'GIGA-REAL': giga_realistic,
'GIGA-REAL-EXACT': giga_realistic_exact,
'US': unif
}
alg = algs[arguments.alg]
print('Building coreset')
w = []
p = []
cputs = np.zeros(Ms.shape[0])
t_build = 0
for m in range(Ms.shape[0]):
print('M = ' + str(Ms[m]) + ': coreset construction, ' +
arguments.alg + ' ' + str(arguments.trial))
t0 = time.process_time()
itrs = (Ms[m] if m == 0 else Ms[m] - Ms[m - 1])
alg.build(itrs)
t_build += time.process_time() - t0
wts, pts, idcs = alg.get()
#store weights/pts/runtime
w.append(wts)
p.append(pts)
cputs[m] = t_build
##############################
##############################
## Step 4: Evaluate coreset
##############################
##############################
# computing kld and saving results
muw = np.zeros((Ms.shape[0], mu0.shape[0]))
Sigw = np.zeros((Ms.shape[0], mu0.shape[0], mu0.shape[0]))
rklw = np.zeros(Ms.shape[0])
fklw = np.zeros(Ms.shape[0])
mu_errs = np.zeros(Ms.shape[0])
Sig_errs = np.zeros(Ms.shape[0])
csizes = np.zeros(Ms.shape[0])
for m in range(Ms.shape[0]):
csizes[m] = (w[m] > 0).sum()
muw[m, :], USigw, LSigwInv = model_linreg.weighted_post(
mu0, Sig0inv, datastd**2, p[m], w[m])
Sigw[m, :, :] = USigw.dot(USigw.T)
rklw[m] = model_linreg.KL(muw[m, :], Sigw[m, :, :], mup, SigpInv)
fklw[m] = model_linreg.KL(mup, Sigp, muw[m, :],
LSigwInv.dot(LSigwInv.T))
mu_errs[m] = np.sqrt(((mup - muw[m, :])**2).sum()) / np.sqrt(
(mup**2).sum())
Sig_errs[m] = np.sqrt(((Sigp - Sigw[m, :, :])**2).sum()) / np.sqrt(
(Sigp**2).sum())
results.save(arguments,
csizes=csizes,
Ms=Ms,
cputs=cputs,
rklw=rklw,
fklw=fklw,
mu_errs=mu_errs,
Sig_errs=Sig_errs)
#also save muw/Sigw/etc for plotting coreset visualizations
f = open('results/coreset_data.pk', 'wb')
res = (x, mu0, Sig0, datastd, mup, Sigp, w, p, muw, Sigw)
pk.dump(res, f)
f.close()
w_true = np.random.randn(d)
X, Y = build_synthetic_dataset(N, w_true)
Z = np.hstack((X, Y[:, np.newaxis]))
#get empirical mean/std
datastd = Y.std()
datamn = Y.mean()
#model params
mu0 = datamn * np.ones(d)
ey = np.eye(d)
Sig0 = (datastd**2 + datamn**2) * ey
Sig0inv = np.linalg.inv(Sig0)
#get true posterior
mup, LSigp, LSigpInv = model_linreg.weighted_post(mu0, Sig0inv, datastd**2, Z,
np.ones(X.shape[0]))
Sigp = LSigp.dot(LSigp.T)
SigpInv = LSigpInv.T.dot(LSigpInv)
#create function to output log_likelihood given param samples
print('Creating log-likelihood function')
log_likelihood = lambda z, th: model_linreg.gaussian_loglikelihood(
z, th, datastd**2)
print('Creating gradient log-likelihood function')
grad_log_likelihood = lambda z, th: model_linreg.gaussian_grad_x_loglikelihood(
z, th, datastd**2)
#create tangent space for well-tuned Hilbert coreset alg
print('Creating tuned projector for Hilbert coreset construction')
sampler_optimal = lambda n, w, pts: mup + np.random.randn(n, mup.shape[0]).dot(