def __negloglikgrad(hyp, info):
"""Return gradient of the penalized log likelihood of single demensional
GP model."""
R0, dR = __covmat(info['theta'], info['theta'], hyp[:-1], True)
nug = np.exp(hyp[-1]) / (1 + np.exp(hyp[-1]))
R = (1 - nug) * R0 + nug * np.eye(info['theta'].shape[0])
if info['gvar'] is not None:
R += np.diag(info['gvar'])
dR = (1 - nug) * dR
dRappend = nug / (
(1 + np.exp(hyp[-1]))) * (-R0 + np.eye(info['theta'].shape[0]))
dR = np.append(dR, dRappend[:, :, None], axis=2)
W, V = np.linalg.eigh(R)
Vh = V / np.sqrt(np.abs(W))
fcenter = Vh.T @ info['g']
n = info['g'].shape[0]
sig2hat = (n * np.mean(fcenter**2) + 10) / (n + 10)
dnegloglik = np.zeros(dR.shape[2])
Rinv = Vh @ Vh.T
for k in range(0, dR.shape[2]):
dsig2hat = -np.sum((Vh @ np.multiply.outer(fcenter, fcenter) @ Vh.T) *
dR[:, :, k]) / (n + 10)
dnegloglik[k] += 0.5 * n * dsig2hat / sig2hat
dnegloglik[k] += 0.5 * np.sum(Rinv * dR[:, :, k])
dnegloglik += (10**(-8) + hyp - info['hypregmean']) / (
(info['hypregstd'])**2)
return dnegloglik
def __negloglik(hyp, info):
"""Return penalized log likelihood of single demensional GP model."""
R0 = __covmat(info['theta'], info['theta'], hyp[:-1])
nug = np.exp(hyp[-1]) / (1 + np.exp(hyp[-1]))
R = (1 - nug) * R0 + nug * np.eye(info['theta'].shape[0])
if info['gvar'] is not None:
R += np.diag(info['gvar'])
W, V = np.linalg.eigh(R)
Vh = V / np.sqrt(np.abs(W))
fcenter = Vh.T @ info['g']
n = info['g'].shape[0]
sig2hat = (n * np.mean(fcenter**2) + 10) / (n + 10)
negloglik = 1 / 2 * np.sum(np.log(np.abs(W))) + 1 / 2 * n * np.log(sig2hat)
negloglik += 0.5 * np.sum(
(((10**(-8) + hyp - info['hypregmean'])) / (info['hypregstd']))**2)
return negloglik
def predict(predinfo, fitinfo, x, theta, **kwargs):
r"""
Finds prediction at theta and x given the dictionary fitinfo.
This [emulationpredictdocstring] automatically filled by docinfo.py when
running updatedocs.py
Parameters
----------
predinfo : dict
An arbitary dictionary where you should place all of your prediction
information once complete. This dictionary is pass by reference, so
there is no reason to return anything. Keep only stuff that will be
used by predict. Key elements are
- `predinfo['mean']` : `predinfo['mean'][k]` is mean of the prediction
at all x at `theta[k]`.
- `predinfo['var']` : `predinfo['var'][k]` is variance of the
prediction at all x at `theta[k]`.
- `predinfo['cov']` : `predinfo['cov'][k]` is mean of the prediction
at all x at `theta[k]`.
- `predinfo['covhalf']` : if `A = predinfo['covhalf'][k]` then
`A.T @ A = predinfo['cov'][k]`.
fitinfo : dict
An arbitary dictionary where you placed all your important fitting
information from the fit function above.
x : array of objects
An matrix (vector) of inputs for prediction.
theta : array of objects
An matrix (vector) of parameters to prediction.
kwargs : dict
A dictionary containing options passed to you.
"""
return_grad = False
if (kwargs is not None) and ('return_grad' in kwargs.keys()) and \
(kwargs['return_grad'] is True):
return_grad = True
return_covx = True
if (kwargs is not None) and ('return_covx' in kwargs.keys()) and \
(kwargs['return_covx'] is False):
return_covx = False
infos = fitinfo['emulist']
predvecs = np.zeros((theta.shape[0], len(infos)))
predvars = np.zeros((theta.shape[0], len(infos)))
if return_grad:
predvecs_gradtheta = np.zeros(
(theta.shape[0], len(infos), theta.shape[1]))
predvars_gradtheta = np.zeros(
(theta.shape[0], len(infos), theta.shape[1]))
drsave = np.array(np.ones(len(infos)), dtype=object)
if predvecs.ndim < 1.5:
predvecs = predvecs.reshape((1, -1))
predvars = predvars.reshape((1, -1))
try:
if x is None or np.all(np.equal(x, fitinfo['x'])) or \
np.allclose(x, fitinfo['x']):
xind = np.arange(0, x.shape[0])
xnewind = np.arange(0, x.shape[0])
else:
raise
except Exception:
matchingmatrix = np.ones((x.shape[0], fitinfo['x'].shape[0]))
for k in range(0, x[0].shape[0]):
try:
matchingmatrix *= np.isclose(x[:, k][:, None], fitinfo['x'][:,
k])
except Exception:
matchingmatrix *= np.equal(x[:, k][:, None], fitinfo['x'][:,
k])
xind = np.argwhere(matchingmatrix > 0.5)[:, 1]
xnewind = np.argwhere(matchingmatrix > 0.5)[:, 0]
rsave = np.array(np.ones(len(infos)), dtype=object)
# loop over principal components
for k in range(0, len(infos)):
if infos[k]['hypind'] == k:
# covariance matrix between new theta and thetas from fit.
if return_grad:
rsave[k], drsave[k] = __covmat(theta,
fitinfo['theta'],
infos[k]['hypcov'],
return_gradx1=True)
else:
rsave[k] = __covmat(theta, fitinfo['theta'],
infos[k]['hypcov'])
# adjusted covariance matrix
r = (1 - infos[k]['nug']) * np.squeeze(rsave[infos[k]['hypind']])
try:
rVh = r @ infos[k]['Vh']
rVh2 = rVh @ (infos[k]['Vh']).T
except Exception:
for i in range(0, len(infos)):
print((i, infos[i]['hypind']))
raise ValueError('Something went wrong with fitted components')
if rVh.ndim < 1.5:
rVh = rVh.reshape((1, -1))
if rVh2.ndim < 1.5:
rVh2 = np.reshape(rVh2, (1, -1))
predvecs[:, k] = r @ infos[k]['pw']
if return_grad:
drsave_hypind = np.squeeze(drsave[infos[k]['hypind']])
if drsave_hypind.ndim < 2.5 and theta.shape[1] < 1.5:
drsave_hypind = np.reshape(drsave_hypind,
(*drsave_hypind.shape, 1))
elif drsave_hypind.ndim < 2.5 and theta.shape[1] > 1.5:
drsave_hypind = np.reshape(drsave_hypind,
(1, *drsave_hypind.shape))
dr = (1 - infos[k]['nug']) * drsave_hypind
if dr.ndim == 2:
drVh = dr.T @ infos[k]['Vh']
predvecs_gradtheta[:, k, :] = dr.T @ infos[k]['pw']
predvars_gradtheta[:, k, :] = \
-infos[k]['sig2'] * 2 * np.sum(rVh * drVh, 1)
else:
drpw = np.squeeze(dr.transpose(0, 2, 1) @ infos[k]['pw'])
if drpw.ndim < 1.5 and theta.shape[1] < 1.5:
drpw = np.reshape(drpw, (-1, 1))
elif drpw.ndim < 1.5 and theta.shape[1] > 1.5:
drpw = np.reshape(drpw, (1, -1))
predvecs_gradtheta[:, k, :] = (1 - infos[k]['nug']) * drpw
predvars_gradtheta[:, k, :] = \
-(infos[k]['sig2'] * 2) * np.einsum("ij,ijk->ik", rVh2, dr)
predvars[:, k] = infos[k]['sig2'] * np.abs(1 - np.sum(rVh**2, 1))
# calculate predictive mean and variance
predinfo['mean'] = np.full((x.shape[0], theta.shape[0]), np.nan)
predinfo['var'] = np.full((x.shape[0], theta.shape[0]), np.nan)
pctscale = (fitinfo['pct'].T * fitinfo['scale']).T
predinfo['mean'][xnewind, :] = ((predvecs @ pctscale[xind, :].T) +
fitinfo['offset'][xind]).T
predinfo['var'][xnewind, :] = ((fitinfo['extravar'][xind] +
predvars @ (pctscale[xind, :]**2).T)).T
predinfo['extravar'] = 1 * fitinfo['extravar'][xind]
predinfo['predvars'] = 1 * predvars
predinfo['predvecs'] = 1 * predvecs
predinfo['phi'] = 1 * pctscale[xind, :]
if return_covx:
CH = (np.sqrt(predvars)[:, :, None] *
(pctscale[xind, :].T)[None, :, :])
predinfo['covxhalf'] = np.full(
(theta.shape[0], CH.shape[1], x.shape[0]), np.nan)
predinfo['covxhalf'][:, :, xnewind] = CH
predinfo['covxhalf'] = predinfo['covxhalf'].transpose((2, 0, 1))
if return_grad:
predinfo['mean_gradtheta'] = np.full(
(x.shape[0], theta.shape[0], theta.shape[1]), np.nan)
predinfo['mean_gradtheta'][xnewind, :, :] = \
((predvecs_gradtheta.transpose(0, 2, 1) @
pctscale[xind, :].T)).transpose((2, 0, 1))
predinfo['predvars_gradtheta'] = 1 * predvars_gradtheta
predinfo['predvecs_gradtheta'] = 1 * predvecs_gradtheta
if return_covx:
dsqrtpredvars = 0.5 * (predvars_gradtheta.transpose(2, 0, 1) /
np.sqrt(predvars)).transpose(1, 2, 0)
if np.allclose(xnewind, xind):
predinfo['covxhalf_gradtheta'] = \
(dsqrtpredvars.transpose(2, 0, 1)[:, :, :, None] *
(pctscale[xind, :].T)[None, :, :]).transpose(3, 1, 2, 0)
else:
predinfo['covxhalf_gradtheta'] = np.full(
(x.shape[0], theta.shape[0], CH.shape[1], theta.shape[1]),
np.nan)
predinfo['covxhalf_gradtheta'][xnewind] = \
(dsqrtpredvars.transpose(2, 0, 1)[:, :, :, None] *
(pctscale[xind, :].T)[None, :, :]).transpose(3, 1, 2, 0)
return
def __fitGP1d(theta,
g,
hyp1,
hyp2,
gvar=None,
hypstarts=None,
hypinds=None,
prevsubmodel=None):
"""Return a fitted model from the emulator model using smart method."""
subinfo = {}
subinfo['hypregmean'] = np.append(
0 + 0.5 * np.log(theta.shape[1]) + np.log(np.std(theta, 0)), (0, hyp1))
subinfo['hypregLB'] = np.append(
-4 + 0.5 * np.log(theta.shape[1]) + np.log(np.std(theta, 0)),
(-12, hyp2))
subinfo['hypregUB'] = np.append(
4 + 0.5 * np.log(theta.shape[1]) + np.log(np.std(theta, 0)), (2, 0))
subinfo['hypregstd'] = (subinfo['hypregUB'] - subinfo['hypregLB']) / 8
subinfo['hypregstd'][-2] = 2
subinfo['hypregstd'][-1] = 4
subinfo['hyp'] = 1 * subinfo['hypregmean']
nhyptrain = np.max(np.min((20 * theta.shape[1], theta.shape[0])))
if theta.shape[0] > nhyptrain:
thetac = np.random.choice(theta.shape[0], nhyptrain, replace=False)
else:
thetac = range(0, theta.shape[0])
subinfo['theta'] = theta[thetac, :]
subinfo['g'] = g[thetac]
subinfo['gvar'] = gvar[thetac]
hypind0 = -1
L0 = __negloglik(subinfo['hyp'], subinfo)
if hypstarts is not None:
L0 = __negloglik(subinfo['hyp'], subinfo)
for k in range(0, hypstarts.shape[0]):
L1 = __negloglik(hypstarts[k, :], subinfo)
if L1 < L0:
subinfo['hyp'] = hypstarts[k, :]
L0 = 1 * L1
hypind0 = hypinds[k]
if hypind0 > -0.5 and hypstarts.ndim > 1:
dL = __negloglikgrad(subinfo['hyp'], subinfo)
scalL = np.std(hypstarts, 0) * hypstarts.shape[0] / \
(1 + hypstarts.shape[0]) + (1 / (1 + hypstarts.shape[0]) * subinfo['hypregstd'])
if np.sum((dL * scalL) ** 2) < 1.25 * \
(subinfo['hyp'].shape[0] + 5 * np.sqrt(subinfo['hyp'].shape[0])):
skipop = True
else:
skipop = False
else:
skipop = False
if (not skipop):
def scaledlik(hypv):
hyprs = subinfo['hypregmean'] + hypv * subinfo['hypregstd']
return __negloglik(hyprs, subinfo)
def scaledlikgrad(hypv):
hyprs = subinfo['hypregmean'] + hypv * subinfo['hypregstd']
return __negloglikgrad(hyprs, subinfo) * subinfo['hypregstd']
newLB = (subinfo['hypregLB'] -
subinfo['hypregmean']) / subinfo['hypregstd']
newUB = (subinfo['hypregUB'] -
subinfo['hypregmean']) / subinfo['hypregstd']
newhyp0 = (subinfo['hyp'] -
subinfo['hypregmean']) / subinfo['hypregstd']
opval = spo.minimize(scaledlik,
newhyp0,
method='L-BFGS-B',
options={'gtol': 0.1},
jac=scaledlikgrad,
bounds=spo.Bounds(newLB, newUB))
hypn = subinfo['hypregmean'] + opval.x * subinfo['hypregstd']
likdiff = (L0 - __negloglik(hypn, subinfo))
else:
likdiff = 0
if hypind0 > -0.5 and (2 * likdiff) < 1.25 * \
(subinfo['hyp'].shape[0] + 5 * np.sqrt(subinfo['hyp'].shape[0])):
subinfo['hypcov'] = subinfo['hyp'][:-1]
subinfo['hypind'] = hypind0
subinfo['nug'] = np.exp(
subinfo['hyp'][-1]) / (1 + np.exp(subinfo['hyp'][-1]))
R = __covmat(theta, theta, subinfo['hypcov'])
subinfo['R'] = (1 - subinfo['nug']) * R + subinfo['nug'] * np.eye(
R.shape[0])
if gvar is not None:
subinfo['R'] += np.diag(gvar)
W, V = np.linalg.eigh(subinfo['R'])
Vh = V / np.sqrt(np.abs(W))
fcenter = Vh.T @ g
subinfo['Vh'] = Vh
n = subinfo['R'].shape[0]
subinfo['sig2'] = (np.mean(fcenter**2) * n + 1) / (n + 1)
subinfo['Rinv'] = V @ np.diag(1 / W) @ V.T
else:
subinfo['hyp'] = hypn
subinfo['hypind'] = -1
subinfo['hypcov'] = subinfo['hyp'][:-1]
subinfo['nug'] = np.exp(
subinfo['hyp'][-1]) / (1 + np.exp(subinfo['hyp'][-1]))
R = __covmat(theta, theta, subinfo['hypcov'])
subinfo['R'] = (1 - subinfo['nug']) * R + subinfo['nug'] * np.eye(
R.shape[0])
if gvar is not None:
subinfo['R'] += np.diag(gvar)
n = subinfo['R'].shape[0]
W, V = np.linalg.eigh(subinfo['R'])
Vh = V / np.sqrt(np.abs(W))
fcenter = Vh.T @ g
subinfo['sig2'] = (np.mean(fcenter**2) * n + 1) / (n + 1)
subinfo['Rinv'] = Vh @ Vh.T
subinfo['Vh'] = Vh
subinfo['pw'] = subinfo['Rinv'] @ g
return subinfo
def supplementtheta(fitinfo, size, theta, thetachoices, choicecosts, cal,
**kwargs):
r'''
Suggests next parameters and obviates pending parameters for value
retrieval of `f`.
Parameters
----------
fitinfo : dict
An arbitary dictionary where you placed all your important fitting
information from the fit function above.
size : integer
The number of new thetas the user wants.
theta : array
An array of theta values where you want to predict.
thetachoices : array
An array of thetas to choose from.
choicecosts : array
The computation cost of each theta choice given to you.
cal : instance of emulator class
An emulator class instance as defined in calibration.
This will not always be provided.
**kwargs : dict
A dictionary containing additional options. Specific arguments:
- `'pending'`: a matrix (sized like `f`) to indicate pending value retrieval of `f`
- `'costpending'`: the cost to obviate pending thetas
- `'includepending'`: boolean to include pending values for obviation considerations
Example usage: `kwargs = {'includepending': True, 'costpending': 0.01+0.99*np.mean(pending,0),
'pending': pending}`.
Returns
----------
Note that we should have `theta.shape[0] * x.shape[0] < size`.
theta : array
Suggested parameters for further value retrievals of `f`.
info : dict
A dictionary to contain selection and obviation information. Contains arguments:
- `'crit'`: criteria associated with selected thetas
- `'obviatesugg'`: indices in `pending` for suggested obviations
'''
pending = None
if ('pending' in kwargs.keys()):
pending = kwargs['pending'].T
pendvar = __getnewvar(fitinfo, pending)
if ('includepending'
in kwargs.keys()) and (kwargs['includepending'] is True):
includepending = True
if ('costpending' in kwargs.keys()):
costpending = kwargs['costpending'] * \
np.ones(fitinfo['theta'].shape[0])
else:
costpending = np.mean(choicecosts) * \
np.ones(fitinfo['theta'].shape[0])
else:
includepending = False
if theta is None:
raise ValueError('this method is designed to take in the '
'theta values.')
infos = copy.copy(fitinfo['emulist'])
thetaold = copy.copy(fitinfo['theta'])
varpca = copy.copy(fitinfo['pcstdvar'])
thetaposs = thetachoices
rsave = np.array(np.ones(len(infos)), dtype=object)
rposssave = np.array(np.ones(len(infos)), dtype=object)
rnewsave = np.array(np.ones(len(infos)), dtype=object)
R = np.array(np.ones(len(infos)), dtype=object)
crit = np.zeros(thetaposs.shape[0])
weightma = np.mean(fitinfo['pct']**2, 0)
# covariance matrices between new thetas, thetachoices, and thetas in fit.
for k in range(0, len(infos)):
if infos[k]['hypind'] == k:
rsave[k] = (1 - infos[k]['nug']) * __covmat(
theta, thetaold, infos[k]['hypcov'])
rposssave[k] = (1 - infos[k]['nug']) * __covmat(
thetaposs, thetaold, infos[k]['hypcov'])
rnewsave[k] = (1 - infos[k]['nug']) * __covmat(
thetaposs, theta, infos[k]['hypcov'])
R[k] = __covmat(thetaold, thetaold, infos[k]['hypcov'])
R[k] = (1 - infos[k]['nug']) * R[k] + np.eye(R[k].shape[0]) * \
infos[k]['nug']
critsave = np.zeros(thetaposs.shape[0])
critcount = np.zeros((crit.shape[0], len(infos)))
if pending is None:
varpcause = 1 * varpca
else:
varpcause = 1 * pendvar
# calculation of selection criterion
thetachoicesave = np.zeros((size, fitinfo['theta'].shape[1]))
for j in range(0, size):
critcount = np.zeros((crit.shape[0], len(infos)))
if thetaposs.shape[0] < 1.5:
thetaold = np.vstack((thetaold, thetaposs))
break
for k in range(0, len(infos)):
if infos[k]['hypind'] == k: # this is to speed things up a bit...
Rh = R[infos[k]['hypind']] + np.diag(varpcause[:, k])
p = rnewsave[infos[k]['hypind']]
term1 = np.linalg.solve(Rh, rposssave[infos[k]['hypind']].T)
q = rsave[infos[k]['hypind']] @ term1
r = rposssave[infos[k]['hypind']].T * term1
critcount[:, k] = weightma[k] * np.mean((p.T - q) ** 2, 0) / \
np.abs(1 - np.sum(r, 0))
else:
critcount[:, k] = weightma[k] / weightma[infos[k]['hypind']] * \
critcount[:, infos[k]['hypind']]
crit = np.sum(critcount, 1)
jstar = np.argmax(crit / choicecosts)
critsave[j] = crit[jstar] / choicecosts[jstar]
thetaold = np.vstack((thetaold, thetaposs[jstar]))
thetachoicesave[j] = thetaposs[jstar]
thetaposs = np.delete(thetaposs, jstar, 0)
for k in range(0, len(infos)):
if infos[k]['hypind'] == k:
R[k] = np.vstack((R[k], rposssave[k][jstar, :]))
R[k] = np.vstack((R[k].T, np.append(rposssave[k][jstar, :],
1))).T
newr = (1 - infos[k]['nug']) * \
__covmat(thetaposs, thetaold[-1, :], infos[k]['hypcov'])
rposssave[k] = np.delete(rposssave[k], jstar, 0)
rposssave[k] = np.hstack((rposssave[k], newr))
rsave[k] = np.hstack((rsave[k], rnewsave[k][jstar, :][:,
None]))
rnewsave[k] = np.delete(rnewsave[k], jstar, 0)
crit = np.delete(crit, jstar)
critcount = np.delete(critcount, jstar, 0)
choicecosts = np.delete(choicecosts, jstar)
varpcause = np.vstack((varpcause, 0 * varpca[0, :]))
varpca = np.vstack((varpca, 0 * varpca[0, :]))
for k in range(0, len(infos)):
if infos[k]['hypind'] == k:
rsave[k] = (1 - infos[k]['nug']) * __covmat(
theta, thetaold, infos[k]['hypcov'])
R[k] = __covmat(thetaold, thetaold, infos[k]['hypcov'])
R[k] = (1 - infos[k]['nug']) * R[k] + np.eye(R[k].shape[0]) * \
infos[k]['nug']
# calculation of obviation criterion and suggests obviations.
info = {}
info['crit'] = critsave
if includepending:
critpend = np.zeros((fitinfo['theta'].shape[0], len(infos)))
for k in range(0, len(infos)):
if infos[k]['hypind'] == k: # this is to speed things up a bit...
Rh = R[infos[k]['hypind']] + np.diag(varpca[:, k])
term1 = np.linalg.solve(Rh, rsave[infos[k]['hypind']].T)
delta = (pendvar[:, k] - varpca[:fitinfo['theta'].shape[0], k])
term3 = np.diag(np.linalg.inv(Rh))[:fitinfo['theta'].shape[0]]
critpend[:, k] = -weightma[k] * delta * \
np.mean((term1[:fitinfo['theta'].shape[0], :] ** 2), 1) / (1 + delta * term3)
else:
critpend[:, k] = weightma[k] / weightma[infos[k]['hypind']] * \
critpend[:, infos[k]['hypind']]
critpend = np.sum(critpend, 1)
info['obviatesugg'] = np.where(
np.any(pending, 1) *
(np.mean(critsave[:size]) > critpend / costpending) > 0.5)[0]
return thetachoicesave, info
