def make_fits_images(run, camcol, field):
"""gets field files from local cache (or sdss), returns UGRIZ dict of
fits images"""
print """==================================================\n\n
Grabbing image files from the cache.
TODO: turn off the tractor printing... """
imgs = {}
for band in BANDS:
print "reading in band %s" % band
imgs[band] = sdss.get_tractor_image_dr9(run, camcol, field, band)
fn = asdss.DR9().retrieve('photoField', run, camcol, field)
F = aufits.fits_table(fn)
# convert to FitsImage's
imgfits = {}
for iband,band in enumerate(BANDS):
print "converting images %s" % band
frame = asdss.DR9().readFrame(run, camcol, field, band)
calib = np.median(frame.getCalibVec())
gain = F[0].gain[iband]
darkvar = F[0].dark_variance[iband]
sky = np.median(frame.getSky())
imgfits[band] = celeste.FitsImage(band,
timg=imgs[band],
calib=calib,
gain=gain,
darkvar=darkvar,
sky=sky)
return imgfits
def make_fits_images(run, camcol, field):
"""gets field files from local cache (or sdss), returns UGRIZ dict of
fits images"""
print """==================================================\n\n
Grabbing image files from the cache.
TODO: turn off the tractor printing... """
imgs = {}
for band in BANDS:
print "reading in band %s" % band
imgs[band] = sdss.get_tractor_image_dr9(run, camcol, field, band)
fn = asdss.DR9().retrieve('photoField', run, camcol, field)
F = aufits.fits_table(fn)
# convert to FitsImage's
imgfits = {}
for iband,band in enumerate(BANDS):
print "converting images %s" % band
frame = asdss.DR9().readFrame(run, camcol, field, band)
calib = np.median(frame.getCalibVec())
gain = F[0].gain[iband]
darkvar = F[0].dark_variance[iband]
sky = np.median(frame.getSky())
imgfits[band] = celeste.FitsImage(iband,
timg=imgs[band],
calib=calib,
gain=gain,
darkvar=darkvar,
sky=sky)
return imgfits
def plot(self, fits_image, ax, data_ax=None, diff_ax=None, unit_flux=False):
import matplotlib.pyplot as plt; import seaborn as sns;
from CelestePy.util.misc import plot_util
if unit_flux:
patch, ylim, xlim = self.compute_scatter_on_pixels(fits_image)
else:
patch, ylim, xlim = self.compute_model_patch(fits_image)
cim = ax.imshow(patch, extent=(xlim[0], xlim[1], ylim[0], ylim[1]))
plot_util.add_colorbar_to_axis(ax, cim)
ax.set_title("model")
if data_ax is not None:
dpatch = fits_image.nelec[ylim[0]:ylim[1], xlim[0]:xlim[1]].copy()
print "Data patch median: ", np.median(dpatch)
dpatch -= np.median(dpatch)
dpatch[dpatch<0] = 0.
dim = data_ax.imshow(dpatch, extent=(xlim[0], xlim[1], ylim[0], ylim[1]))
plot_util.add_colorbar_to_axis(data_ax, dim)
data_ax.set_title("data")
if diff_ax is not None:
dpatch = fits_image.nelec[ylim[0]:ylim[1], xlim[0]:xlim[1]].copy()
dpatch -= np.median(dpatch)
dpatch[dpatch<0] = 0.
dim = diff_ax.imshow((dpatch - patch), extent=(xlim[0], xlim[1], ylim[0], ylim[1]))
plot_util.add_colorbar_to_axis(diff_ax, dim)
msqe = np.mean((dpatch - patch)**2)
smsqe = np.mean((dpatch - patch)**2 / patch)
diff_ax.set_title("diff, mse = %2.3f"%msqe)
def _parameter_initialiser(self, x, c=None, n=None, t=None, offset=False):
log_x = np.log(x)
log_x[np.isnan(log_x)] = -np.inf
if (2 in c) or (-1 in c):
heuristic = "Turnbull"
else:
heuristic = "Fleming-Harrington"
data = {'x' : x, 'c' : c, 'n' : n, 't' : t}
model = para.Parametric(self, 'MPP', data, offset, False, False)
fitting_info = {}
fitting_info['rr'] = 'x'
fitting_info['heuristic'] = heuristic
fitting_info['on_d_is_0'] = True
fitting_info['turnbull_estimator'] = 'Fleming-Harrington'
fitting_info['init'] = None
model.fitting_info = fitting_info
if offset:
results = mpp(model)
return (results['gamma'], *results['params'])
else:
gumb = para.Gumbel.fit(log_x, c, n, t, how='MLE')
if not gumb.res.success:
gumb = para,Gumbel.fit(log_x, c, n, t, how='MPP', heuristic=heuristic)
mu, sigma = gumb.params
alpha, beta = np.exp(mu), 1. / sigma
if (np.isinf(alpha) | np.isnan(alpha)):
alpha = np.median(x)
if (np.isinf(beta) | np.isnan(beta)):
beta = 1.
return alpha, beta
def initialize(deep_map, X,num_pseudo_params):
smart_map = {}
for layer,layer_map in deep_map.iteritems():
smart_map[layer] = {}
for unit,gp_map in layer_map.iteritems():
smart_map[layer][unit] = {}
cov_params = gp_map['cov_params']
lengthscales = cov_params[1:]
if layer == 0:
pairs = itertools.combinations(X, 2)
dists = np.array([np.abs(p1-p2) for p1,p2 in pairs])
smart_lengthscales = np.array([np.log(np.median(dists[:,i])) for i in xrange(len(lengthscales))])
kmeans = KMeans(n_clusters = num_pseudo_params, init = 'k-means++')
fit = kmeans.fit(X)
smart_x0 = fit.cluster_centers_
#inds = npr.choice(len(X), num_pseudo_params, replace = False)
#smart_x0 = np.array(X)[inds,:]
smart_y0 = np.ndarray.flatten(smart_x0)
#smart_y0 = np.array(y)[inds]
smart_noise_scale = np.log(np.var(smart_y0))
else:
smart_x0 = gp_map['x0']
smart_y0 = np.ndarray.flatten(smart_x0[:,0])
smart_lengthscales = np.array([np.log(1) for i in xrange(len(lengthscales))])
smart_noise_scale = np.log(np.var(smart_y0))
gp_map['cov_params'] = np.append(cov_params[0],smart_lengthscales)
gp_map['x0'] = smart_x0
gp_map['y0'] = smart_y0
#gp_map['noise_scale'] = smart_noise_scale
smart_map[layer][unit] = gp_map
smart_params = pack_deep_params(smart_map)
return smart_params
def meddistance(X, subsample=None, mean_on_fail=True):
"""
Compute the median of pairwise distances (not distance squared) of points
in the matrix. Useful as a heuristic for setting Gaussian kernel's width.
Parameters
----------
X : n x d numpy array
mean_on_fail: True/False. If True, use the mean when the median distance is 0.
This can happen especially, when the data are discrete e.g., 0/1, and
there are more slightly more 0 than 1. In this case, the m
Return
------
median distance
"""
if subsample is None:
D = dist_matrix(X, X)
Itri = np.tril_indices(D.shape[0], -1)
Tri = D[Itri]
med = np.median(Tri)
if med <= 0:
# use the mean
return np.mean(Tri)
return med
else:
assert subsample > 0
rand_state = np.random.get_state()
np.random.seed(9827)
n = X.shape[0]
ind = np.random.choice(n, min(subsample, n), replace=False)
np.random.set_state(rand_state)
# recursion just one
return meddistance(X[ind, :], None, mean_on_fail)
def guess_params_intensity(self, f_kappa_hat):
# Is this correct?
med = np.median(f_kappa_hat)
return dict(
mu=med,
kappa=np.log(f_kappa_hat/med)
)
def initialize(deep_map, X,num_pseudo_params):
smart_map = {}
for layer,layer_map in deep_map.iteritems():
smart_map[layer] = {}
for unit,gp_map in layer_map.iteritems():
smart_map[layer][unit] = {}
cov_params = gp_map['cov_params']
lengthscales = cov_params[1:]
if layer == 0:
pairs = itertools.combinations(X, 2)
dists = np.array([np.abs(p1-p2) for p1,p2 in pairs])
smart_lengthscales = np.array([np.log(np.median(dists[:,i])) for i in xrange(len(lengthscales))])
kmeans = KMeans(n_clusters = num_pseudo_params, init = 'k-means++')
fit = kmeans.fit(X)
smart_x0 = fit.cluster_centers_
#inds = npr.choice(len(X), num_pseudo_params, replace = False)
#smart_x0 = np.array(X)[inds,:]
smart_y0 = np.ndarray.flatten(smart_x0)
#smart_y0 = np.array(y)[inds]
smart_noise_scale = np.log(np.var(smart_y0))
else:
smart_x0 = gp_map['x0']
smart_y0 = np.ndarray.flatten(smart_x0[:,0])
smart_lengthscales = np.array([np.log(1) for i in xrange(len(lengthscales))])
smart_noise_scale = np.log(np.var(smart_y0))
gp_map['cov_params'] = np.append(cov_params[0],smart_lengthscales)
gp_map['x0'] = smart_x0
gp_map['y0'] = smart_y0
#gp_map['noise_scale'] = smart_noise_scale
smart_map[layer][unit] = gp_map
smart_params = pack_deep_params(smart_map)
return smart_params
def meddistance(X, subsample=None, mean_on_fail=True):
"""
Compute the median of pairwise distances (not distance squared) of points
in the matrix. Useful as a heuristic for setting Gaussian kernel's width.
Parameters
----------
X : n x d numpy array
mean_on_fail: True/False. If True, use the mean when the median distance is 0.
This can happen especially, when the data are discrete e.g., 0/1, and
there are more slightly more 0 than 1. In this case, the m
Return
------
median distance
"""
if subsample is None:
D = dist_matrix(X, X)
Itri = np.tril_indices(D.shape[0], -1)
Tri = D[Itri]
med = np.median(Tri)
if med <= 0:
# use the mean
return np.mean(Tri)
return med
else:
assert subsample > 0
rand_state = np.random.get_state()
np.random.seed(9827)
n = X.shape[0]
ind = np.random.choice(n, min(subsample, n), replace=False)
np.random.set_state(rand_state)
# recursion just one
return meddistance(X[ind, :], None, mean_on_fail)
def _create_initial_point(self, Ts, E, *args):
if CensoringType.is_right_censoring(self):
log_T = np.log(Ts[0])
elif CensoringType.is_left_censoring(self):
log_T = np.log(Ts[1])
elif CensoringType.is_interval_censoring(self):
log_T = np.log(Ts[1])
return np.array([np.median(log_T), 1.0])
def _create_initial_point(self, Ts, E, *args):
if CensoringType.is_right_censoring(self):
T = Ts[0]
elif CensoringType.is_left_censoring(self):
T = Ts[1]
elif CensoringType.is_interval_censoring(self):
T = Ts[1] - Ts[0]
return np.array([np.median(T), 1.0])
def accuracy_stats(accs):
hist = {}
for l, a in accs:
temp = hist.get(l, [])
temp.append(a)
hist[l] = temp
print(hist)
for l, a_s in hist.items():
print(l, np.median(a_s))
def __init__(self, img_dict):
self.img_dict = img_dict
# set each image noise level to the median
for k, img in self.img_dict.iteritems():
img.epsilon = np.median(img.nelec)
# set the (gamma) prior over noise level
self.a_0 = 5 # convolution parameter - higher tends to avoid 0
self.b_0 = .005 # inverse scale parameter
def __init__(self, img_dict):
self.img_dict = img_dict
# set each image noise level to the median
for k, img in self.img_dict.iteritems():
img.epsilon = np.median(img.nelec)
# set the (gamma) prior over noise level
self.a_0 = 5 # convolution parameter - higher tends to avoid 0
self.b_0 = .005 # inverse scale parameter
def get_median_inter_mnist(x):
# x2 = np.sum(x*x,axis=1,keepdims=True)
# sqdist = x2+x2.T-2*[email protected]
# sqdist = (sqdist+abs(sqdist).T)/2
if x.shape[0] < 10000:
sqdist = _sqdist(x, None)
else:
M = int(x.shape[0]/400)
sqdist = Parallel(n_jobs=20)(delayed(_sqdist)(x[i:i+M], x) for i in range(0,x.shape[0],M))
dist = np.sqrt(sqdist)
return np.median(dist.flatten())
def compute_prob(self):
""" Compute prob as median over all trees """
T = self.x.shape[0]
self.single_P = np.zeros((self.n_trees, T, T))
for t in range(self.n_trees):
tree = mind_tree(self.x,
manifold_dim=self.manifold_dim,
rng=self.rng)
tree.fit_tree()
tree.compute_probs()
self.single_P[t] = tree.P
# un-normalized density
self.P = np.median(self.single_P, axis=0)
def compute_probs(self):
""" Compute prob as median over all trees """
T = self.x.shape[0]
self.single_P = np.zeros((self.n_trees, T, T))
for t in range(self.n_trees):
print("Fitting tree {} of {}".format(t, self.n_trees))
tree = mind_tree(self.x,
manifold_dim=self.manifold_dim,
rng=self.rng)
self.trees[t] = tree
tree.compute_probs()
self.single_P[t] = tree.P
# un-normalized density
self.P = np.median(self.single_P, axis=0)
def _create_initial_point(self, Ts, E, *args):
if CensoringType.is_right_censoring(self):
T = Ts[0]
elif CensoringType.is_left_censoring(self):
T = np.clip(0.0001, np.inf, Ts[1])
elif CensoringType.is_interval_censoring(self):
if E.sum() > 0:
# Ts[1] can contain infs, so ignore this data
okay_data = Ts[1] < 1e10
T = Ts[1]
T = T[okay_data]
else:
T = np.array([1.0])
return np.array([np.median(T), 1.0])
def unpack(self, theta):
# This will work, technically, but autograd won't like it
if theta == self.theta_cached:
return
if theta.shape != self.Ndim + self.Ndim * (self.Ndim + 1) / 2:
raise ParameterError("theta, mu and C have incompatible shapes")
self.mu = theta[:self.Ndim]
k = self.Ndim
for i, Crow in enumerate(self.C):
self.C[i, i:] = self.C[i:, i] = theta[k:k + (Ndim - i)]
k += Ndim - i
eps = 1e-10 * np.eye(np.median(np.diag(C)))
self.L = np.linalg.cholesky(C + eps)
self.theta_cached = theta
def plot(self,
fits_image,
ax,
data_ax=None,
diff_ax=None,
unit_flux=False):
import matplotlib.pyplot as plt
import seaborn as sns
from CelestePy.util.misc import plot_util
if unit_flux:
patch, ylim, xlim = self.compute_scatter_on_pixels(fits_image)
else:
patch, ylim, xlim = self.compute_model_patch(fits_image)
cim = ax.imshow(patch, extent=(xlim[0], xlim[1], ylim[0], ylim[1]))
plot_util.add_colorbar_to_axis(ax, cim)
ax.set_title("model")
if data_ax is not None:
dpatch = fits_image.nelec[ylim[0]:ylim[1], xlim[0]:xlim[1]].copy()
print "Data patch median: ", np.median(dpatch)
dpatch -= np.median(dpatch)
dpatch[dpatch < 0] = 0.
dim = data_ax.imshow(dpatch,
extent=(xlim[0], xlim[1], ylim[0], ylim[1]))
plot_util.add_colorbar_to_axis(data_ax, dim)
data_ax.set_title("data")
if diff_ax is not None:
dpatch = fits_image.nelec[ylim[0]:ylim[1], xlim[0]:xlim[1]].copy()
dpatch -= np.median(dpatch)
dpatch[dpatch < 0] = 0.
dim = diff_ax.imshow((dpatch - patch),
extent=(xlim[0], xlim[1], ylim[0], ylim[1]))
plot_util.add_colorbar_to_axis(diff_ax, dim)
msqe = np.mean((dpatch - patch)**2)
smsqe = np.mean((dpatch - patch)**2 / patch)
diff_ax.set_title("diff, mse = %2.3f" % msqe)
def is_outlier(points, thresh=3.5):
"""
Returns a boolean array with True if points are outliers and False
otherwise.
Parameters:
-----------
points : An numobservations by numdimensions array of observations
thresh : The modified z-score to use as a threshold. Observations with
a modified z-score (based on the median absolute deviation) greater
than this value will be classified as outliers.
Returns:
--------
mask : A numobservations-length boolean array.
References:
----------
Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and
Handle Outliers", The ASQC Basic References in Quality Control:
Statistical Techniques, Edward F. Mykytka, Ph.D., Editor.
https://stackoverflow.com/a/11886564/7479938
"""
if len(points.shape) == 1:
points = points[:, None]
median = np.median(points, axis=0)
diff = np.sum((points - median)**2, axis=-1)
diff = np.sqrt(diff)
med_abs_deviation = np.median(diff)
if (np.isclose(med_abs_deviation, 0.0)):
return np.zeros_like(diff, dtype=bool)
modified_z_score = 0.6745 * diff / med_abs_deviation
return modified_z_score > thresh
def get_basic_kernel(t, y, yerr):
kernel = terms.SHOTerm(
log_S0=np.log(np.var(y)),
log_Q=-np.log(4.0),
log_omega0=np.log(2 * np.pi / 10.),
bounds=dict(
log_S0=(-20.0, 10.0),
log_omega0=(np.log(2 * np.pi / 80.0), np.log(2 * np.pi / 2.0)),
),
)
kernel.freeze_parameter('log_Q')
# Finally some jitter
kernel += terms.JitterTerm(log_sigma=np.log(np.median(yerr)),
bounds=[(-20.0, 5.0)])
return kernel
def _parameter_initialiser(self, x, c=None, n=None, offset=False):
log_x = np.log(x)
log_x[np.isnan(log_x)] = 0
gumb = para.Gumbel.fit(log_x, c, n, how='MLE')
if not gumb.res.success:
gumb = para.Gumbel.fit(log_x, c, n, how='MPP')
mu, sigma = gumb.params
alpha, beta = np.exp(mu), 1. / sigma
if (np.isinf(alpha) | np.isnan(alpha)):
alpha = np.median(x)
if (np.isinf(beta) | np.isnan(beta)):
beta = 1.
if offset:
gamma = np.min(x) - (np.max(x) - np.min(x)) / 10.
return gamma, alpha, beta, 1.
else:
return alpha, beta, 1.
def calc_rolled_out_gradient(self, cnt=100):
np.set_printoptions(precision=2, linewidth=200)
scores = []
# normalize
for k in self.cur_grad_db:
for j in self.cur_grad_db[k]:
self.grad_db[k][j] = []
# perform cnt rollouts
for i in range(cnt):
self.run_model()
scores.append(self.cur_trace_score)
for k in self.cur_grad_db:
for j in self.cur_grad_db[k]:
self.grad_db[k][j].append(self.cur_grad_db[k][j])
# our baseline is just the median
scores = np.atleast_2d(scores)
total_score = np.sum(scores)
# scores = scores - np.mean( scores )
scores = scores - np.median(scores)
# normalize
for k in self.cur_grad_db:
for j in self.cur_grad_db[k]:
# print "%s - %s" % (k,j)
# tmp = self.grad_db[ k ][ j ]
# print tmp[0].shape
# self.grad_db[ k ][ j ] = np.dot( scores, np.vstack( tmp ) ) / cnt
tmp = self.grad_db[k][j]
tmpdims = len(tmp[0].shape)
grads = np.stack(tmp, axis=tmpdims)
newshape = np.ones((tmpdims + 1), dtype=int)
newshape[tmpdims] = scores.shape[1]
tmpscores = np.reshape(scores, newshape)
self.grad_db[k][j] = np.sum(tmpscores * grads,
axis=tmpdims) / cnt
return total_score / float(cnt)
def smart_initialize_params(init_params):
layer_params, x0, y0 = unpack_all_params(init_params)
# Initialize the first length scale parameter as the median distance between points
pairs = itertools.combinations(X, 2)
dists = np.array([np.linalg.norm(np.array([p1])- np.array([p2])) for p1,p2 in pairs])
layer_params[0][2] = np.log(np.var(y))
layer_params[0][3] = np.log(np.median(dists))
# Initialize the pseudo inputs for the first layer by sampling from the data, the pseudo outputs equal to the inputs
x0[0] = np.ndarray.flatten(np.array(X)[rs.choice(len(X), num_pseudo_params, replace=False),:])
y0[0] = x0[0]
# For every other layer, set the inducing outputs to the inducing inputs (which are sampled from N(0,.01)) and lengthscale large
for layer in xrange(1,n_layers):
y0[layer] = x0[layer]
layer_params[layer][3] = np.log(1)
return pack_all_params(layer_params, x0, y0)
def get_basic_kernel(t, y, yerr, period=False):
if not period:
period = 0.5
kernel = terms.SHOTerm(
log_S0=np.log(np.var(y)),
log_Q=-np.log(4.0),
log_omega0=np.log(2 * np.pi / 20.),
bounds=dict(
log_S0=(-20.0, 10.0),
log_omega0=(np.log(2 * np.pi / 100.), np.log(2 * np.pi / (10))),
),
)
kernel.freeze_parameter('log_Q')
## tau = 2*np.exp(-1*np.log(4.0))/np.exp(log_omega0)
# Finally some jitter
ls = np.log(np.median(yerr))
kernel += terms.JitterTerm(log_sigma=ls, bounds=[(ls - 5.0, ls + 5.0)])
return kernel
def read_image(fframe, data_set='BTLS', background_subtract=False):
''' read given image
:param fframe:
file name. Currently assumes that the file can be opened using
PIL.Image
:param background_subtract:
If True, sigmaclipped background subtraction (default: False)
'''
if data_set == 'BTLS':
im = Image.open(fframe)
imarr = np.array(im)
elif dataset == 'mock_alpha':
imarr = np.loadtxt(fframe)
noise_level = 0.
if background_subtract:
# if true, simple sigmaclipped background subtraction
noise_level = np.median(stats.sigmaclip(imarr.flatten(), high=3.)[0])
return imarr - noise_level
def smart_initialize_params(init_params):
layer_params, x0, y0 = unpack_all_params(init_params)
# Initialize the first length scale parameter as the median distance between points
pairs = itertools.combinations(X, 2)
dists = np.array([
np.linalg.norm(np.array([p1]) - np.array([p2])) for p1, p2 in pairs
])
layer_params[0][2] = np.log(np.var(y))
layer_params[0][3] = np.log(np.median(dists))
# Initialize the pseudo inputs for the first layer by sampling from the data, the pseudo outputs equal to the inputs
x0[0] = np.ndarray.flatten(
np.array(X)[
rs.choice(len(X), num_pseudo_params, replace=False), :])
y0[0] = x0[0]
# For every other layer, set the inducing outputs to the inducing inputs (which are sampled from N(0,.01)) and lengthscale large
for layer in xrange(1, n_layers):
y0[layer] = x0[layer]
layer_params[layer][3] = np.log(1)
return pack_all_params(layer_params, x0, y0)
def forward(
co2dict,
grads_of,
grads_wrt,
totals=None,
equilibria_in=None,
equilibria_out=None,
dx=1e-6,
dx_scaling="median",
dx_func=None,
):
"""Get forward finite-difference derivatives of CO2SYS outputs w.r.t. inputs.
Arguments:
co2dict -- output generated by `PyCO2SYS.CO2SYS`.
grads_of -- list of keys from `co2dict` that you want to calculate the derivatives
of, or a single key as a string, or "all".
grads_wrt -- list of `PyCO2SYS.CO2SYS` input variable names that you want to
calculate the derivatives with respect to, or a single name as a string, or
"all".
Keyword arguments:
totals -- dict of internal override total salt concentrations identical to that used
to generate the `co2dict` (default None).
equilibria_in -- dict of internal override equilibrium constants at input conditions
identical to that used to generate the `co2dict` (default None).
equilibria_out -- dict of internal override equilibrium constants at output
conditions identical to that used to generate the `co2dict` (default None).
dx -- the forward difference for the derivative estimation (default 1e-6).
dx_scaling -- method for scaling `dx` for each variable, can be one of: "median"
(default), "none", or "custom".
dx_func -- function of each variable to scale `dx` with if dx_scaling="custom".
"""
# Derivatives can be calculated w.r.t. these inputs only
inputs_wrt = [
"PAR1",
"PAR2",
"SAL",
"TEMPIN",
"TEMPOUT",
"PRESIN",
"PRESOUT",
"SI",
"PO4",
"NH3",
"H2S",
]
totals_wrt = ["TB", "TF", "TSO4", "TCa"]
Ks_wrt = [
"KSO4",
"KF",
"fH",
"KB",
"KW",
"KP1",
"KP2",
"KP3",
"KSi",
"K1",
"K2",
"KH2S",
"KNH3",
"K0",
"FugFac",
"KCa",
"KAr",
]
Kis_wrt = ["{}input".format(K) for K in Ks_wrt]
Kos_wrt = ["{}output".format(K) for K in Ks_wrt]
Kis_wrt.append("RGas")
Kos_wrt.append("RGas")
pKis_wrt = ["p{}input".format(K) for K in Ks_wrt if K.startswith("K")]
pKos_wrt = ["p{}output".format(K) for K in Ks_wrt if K.startswith("K")]
# If only a single `grads_wrt` is requested, check it's allowed & convert to list
groups_wrt = [
"all", "measurements", "totals", "equilibria_in", "equilibria_out"
]
all_wrt = inputs_wrt + totals_wrt + Kis_wrt + Kos_wrt + pKis_wrt + pKos_wrt
if isinstance(grads_wrt, str):
assert grads_wrt in (all_wrt + groups_wrt)
if grads_wrt == "all":
grads_wrt = all_wrt
elif grads_wrt == "measurements":
grads_wrt = inputs_wrt
elif grads_wrt == "totals":
grads_wrt = totals_wrt
elif grads_wrt == "equilibria_in":
grads_wrt = Kis_wrt
elif grads_wrt == "equilibria_out":
grads_wrt = Kos_wrt
else:
grads_wrt = [grads_wrt]
# Make sure all requested `grads_wrt` are allowed
assert np.all(np.isin(list(grads_wrt),
all_wrt)), "Invalid `grads_wrt` requested."
# If only a single `grads_of` is requested, check it's allowed & convert to list
if isinstance(grads_of, str):
assert grads_of in ["all"] + list(engine.gradables)
if grads_of == "all":
grads_of = engine.gradables
else:
grads_of = [grads_of]
# Final validity checks
assert np.all(np.isin(grads_of,
engine.gradables)), "Invalid `grads_of` requested."
assert dx > 0, "`dx` must be positive."
# Assemble input arguments for engine._CO2SYS()
co2args = {
arg: co2dict[arg]
for arg in [
"PAR1",
"PAR2",
"PAR1TYPE",
"PAR2TYPE",
"SAL",
"TEMPIN",
"TEMPOUT",
"PRESIN",
"PRESOUT",
"SI",
"PO4",
"NH3",
"H2S",
"pHSCALEIN",
"K1K2CONSTANTS",
"KSO4CONSTANT",
"KFCONSTANT",
"BORON",
"buffers_mode",
"WhichR",
]
}
co2kwargs = {
"KSO4CONSTANTS": co2dict["KSO4CONSTANTS"],
"totals": totals,
"equilibria_in": equilibria_in,
"equilibria_out": equilibria_out,
}
# Preallocate output dict to store the gradients
co2derivs = {of: {wrt: None for wrt in grads_wrt} for of in grads_of}
dxs = {wrt: None for wrt in grads_wrt}
# Estimate the gradients with central differences
for wrt in grads_wrt:
# Make copies of input args to modify
co2args_plus = copy.deepcopy(co2args)
co2kwargs_plus = copy.deepcopy(co2kwargs)
# Perturb if `wrt` is one of the main inputs to CO2SYS
if wrt in inputs_wrt:
dx_wrt = _get_dx_wrt(dx,
np.median(co2args_plus[wrt]),
dx_scaling,
dx_func=dx_func)
co2args_plus[wrt] = co2args_plus[wrt] + dx_wrt
# Perturb if `wrt` is one of the `totals` internal overrides
elif wrt in totals_wrt:
co2kwargs_plus, dx_wrt = _overridekwargs(co2dict,
co2kwargs_plus,
"totals",
wrt,
dx,
dx_scaling,
dx_func=dx_func)
# Perturb if `wrt` is one of the `equilibria_in` internal overrides
elif wrt in Kis_wrt:
co2kwargs_plus, dx_wrt = _overridekwargs(
co2dict,
co2kwargs_plus,
"equilibria_in",
wrt,
dx,
dx_scaling,
dx_func=dx_func,
)
# Perturb if `wrt` is one of the `equilibria_in` internal overrides and the pK
# derivative is requested
elif wrt in pKis_wrt:
co2kwargs_plus, dx_wrt = _overridekwargs(
co2dict,
co2kwargs_plus,
"equilibria_in",
wrt,
dx,
dx_scaling,
dx_func=dx_func,
)
# Perturb if `wrt` is one of the `equilibria_out` internal overrides
elif wrt in Kos_wrt:
co2kwargs_plus, dx_wrt = _overridekwargs(
co2dict,
co2kwargs_plus,
"equilibria_out",
wrt,
dx,
dx_scaling,
dx_func=dx_func,
)
# Solve CO2SYS with the perturbation applied
co2dict_plus = engine._CO2SYS(**co2args_plus, **co2kwargs_plus)
dxs[wrt] = dx_wrt
# Extract results and calculate forward finite difference derivatives
for of in grads_of:
if co2derivs[of][
wrt] is None: # don't overwrite existing derivatives
co2derivs[of][wrt] = (co2dict_plus[of] - co2dict[of]) / dx_wrt
return co2derivs, dxs
old_mean_grad = mean_grad
old_log_sigma_grad = log_sigma_grad
#if (itt+1) % 10== 0:
#print(mean_grads_running_dot_product)
#print(sigma_grads_running_dot_product)
criterion1 = mean_grads_running_dot_product < 0
criterion2 = sigma_grads_running_dot_product < 0
criterion3 = np.abs(elbo_prev - elbo) < np.abs(
elbo_threshold_swa * elbo_prev)
criterion7 = s_mean < 0
criterion8 = s_log_var < 0
elbo_diff_list.append(elbo - elbo_prev)
elbo_diff_median = np.median(np.array(elbo_diff_list[-21:-1]))
elbo_diff_mean = np.mean(np.array(elbo_diff_list[-21:-1]))
elbo_diff_last_20 = elbo_diff_list[-20:-1]
#elbo_diff_max = np.max(np.array(elbo_diff_list[-21:-1]))
elbo_diff_list_abs = [0 for i in elbo_diff_last_20 if i < 0]
val1 = len(elbo_diff_list_abs) - np.count_nonzero(
np.asarray(elbo_diff_list_abs))
criterion4 = val1 > 5
criterion6 = itt > 3000
if len(elbo_mean_list) > 6:
criterion5 = np.abs(elbo_mean_list[-1] - elbo_mean_list[-2]
) < np.abs(elbo_mean_list[-2] -
elbo_mean_list[-5]) * 0.10
#if criterion1 and criterion2 and criterion3 and criterion6 and start_swa is False:
def mat_median(m):
l = len(m)
v = np.arange(l).reshape((l,1))
return np.median(m[v != v.T])
## GRADIENT DESCENT
for i in range(gradient_iters_max):
objectPoints = np.copy(new_objectPoints)
gradient = gd.evaluate_gradient(gradient, objectPoints,
np.array(cam.P), normalize)
new_objectPoints = gd.update_points(gradient,
objectPoints,
limitx=0.15,
limity=0.15)
new_imagePoints = np.array(cam.project(new_objectPoints, False))
D5pWell.Camera.append(cam)
D5pWell.ObjectPoints.append(new_objectPoints)
D5pWell.calculate_metrics()
print np.median(np.array(D5pIll.Homo_CV_mean) / np.array(D5pWell.Homo_CV_mean))
print np.median(
np.array(D5pIll.pnp_tvec_error_mean) /
np.array(D5pWell.pnp_tvec_error_mean))
print np.median(np.array(D5pIll.CondNumber) / np.array(D5pWell.CondNumber))
#print np.mean(np.array(D4pSquare.pnp_tvec_error_mean)/np.array(D4pWell.pnp_tvec_error_mean))
#print np.mean(np.array(D4pSquare.Homo_CV_mean)/np.array(D4pWell.Homo_CV_mean))
pickle.dump([D5pIll, D5pWell], open("icra_sim_illvsWell_5points.p", "wb"))
#
#
#
#
#
# Admittedly, this is not a good way to do it
# If you have any tips on how to make this better please let me know
smart_params = np.array([])
all_layer_params = unpack_all_params(init_params)
for layer in xrange(n_layers):
layer_params = all_layer_params[layer]
layer_gp_params = unpack_layer_params[layer](layer_params)
for dim in xrange(dimensions[layer+1]):
gp_params = layer_gp_params[dim]
mean, cov_params, noise_scale, x0, y0 = unpack_gp_params_all[layer][dim](gp_params)
lengthscales = cov_params[1:]
if layer == 0:
pairs = itertools.combinations(X, 2)
dists = np.array([np.abs(p1-p2) for p1,p2 in pairs])
smart_lengthscales = np.array([np.log(np.median(dists[:,i])) for i in xrange(len(lengthscales))])
smart_x0 = np.array(X)[rs.choice(len(X), num_pseudo_params, replace=False),:]
smart_y0 = np.ndarray.flatten(smart_x0)
else:
smart_x0 = x0
smart_y0 = np.ndarray.flatten(x0)
smart_lengthscales = np.array([np.log(1) for i in xrange(len(lengthscales))])
cov_params = np.append(cov_params[0],smart_lengthscales)
params = pack_gp_params_all[layer][dim](mean, cov_params, noise_scale, smart_x0, smart_y0)
smart_params = np.append(smart_params, params)
init_params = smart_params
print("Optimizing covariance parameters...")
def guess_params_intensity(self, f_kappa_hat):
med = np.median(f_kappa_hat)
return dict(
mu=med,
kappa=f_kappa_hat-med
)
def rdc(x, y, f=np.sin, k=20, s=1 / 6., n=1):
"""
Computes the Randomized Dependence Coefficient
x,y: numpy arrays 1-D or 2-D
If 1-D, size (samples,)
If 2-D, size (samples, variables)
f: function to use for random projection
k: number of random projections to use
s: scale parameter
n: number of times to compute the RDC and
return the median (for stability)
According to the paper, the coefficient should be relatively insensitive to
the settings of the f, k, and s parameters.
Source: https://github.com/garydoranjr/rdc
"""
#x = x.reshape((len(x)))
#y = y.reshape((len(y)))
if n > 1:
values = []
for i in range(n):
try:
values.append(rdc(x, y, f, k, s, 1))
except np.linalg.linalg.LinAlgError:
pass
return np.median(values)
if len(x.shape) == 1: x = x.reshape((-1, 1))
if len(y.shape) == 1: y = y.reshape((-1, 1))
# Copula Transformation
cx = np.column_stack([rankdata(xc, method='ordinal')
for xc in x.T]) / float(x.size)
cy = np.column_stack([rankdata(yc, method='ordinal')
for yc in y.T]) / float(y.size)
# Add a vector of ones so that w.x + b is just a dot product
O = np.ones(cx.shape[0])
X = np.column_stack([cx, O])
Y = np.column_stack([cy, O])
# Random linear projections
Rx = (s / X.shape[1]) * np.random.randn(X.shape[1], k)
Ry = (s / Y.shape[1]) * np.random.randn(Y.shape[1], k)
X = np.dot(X, Rx)
Y = np.dot(Y, Ry)
# Apply non-linear function to random projections
fX = f(X)
fY = f(Y)
# Compute full covariance matrix
C = np.cov(np.hstack([fX, fY]).T)
# Due to numerical issues, if k is too large,
# then rank(fX) < k or rank(fY) < k, so we need
# to find the largest k such that the eigenvalues
# (canonical correlations) are real-valued
k0 = k
lb = 1
ub = k
while True:
# Compute canonical correlations
Cxx = C[:k, :k]
Cyy = C[k0:k0 + k, k0:k0 + k]
Cxy = C[:k, k0:k0 + k]
Cyx = C[k0:k0 + k, :k]
eigs = np.linalg.eigvals(
np.dot(np.dot(np.linalg.pinv(Cxx), Cxy),
np.dot(np.linalg.pinv(Cyy), Cyx)))
# Binary search if k is too large
if not (np.all(np.isreal(eigs)) and 0 <= np.min(eigs)
and np.max(eigs) <= 1):
ub -= 1
k = (ub + lb) // 2
continue
if lb == ub: break
lb = k
if ub == lb + 1:
k = ub
else:
k = (ub + lb) // 2
return np.sqrt(np.max(eigs))
#another figure for individual Pose errors
fig3 = plt.figure('Well conditioned Vs Ill conditioned configurations (Pose)',
figsize=(fig_width, 1.1 * fig_width))
ax_t_error_ippe = fig3.add_subplot(321)
ax_r_error_ippe = fig3.add_subplot(322)
ax_t_error_epnp = fig3.add_subplot(323, sharex=ax_t_error_ippe)
ax_r_error_epnp = fig3.add_subplot(324, sharex=ax_r_error_ippe)
ax_t_error_pnp = fig3.add_subplot(325, sharex=ax_t_error_ippe)
ax_r_error_pnp = fig3.add_subplot(3, 2, 6, sharex=ax_r_error_ippe)
ax_t_error_pnp.set_xticks([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
ax_r_error_pnp.set_xticks([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
#CONDITION NUMBER
mean4Well = np.median((D4pWell.CondNumber))
mean5Well = np.median((D5pWell.CondNumber))
mean6Well = np.median((D6pWell.CondNumber))
mean7Well = np.median((D7pWell.CondNumber))
mean8Well = np.median((D8pWell.CondNumber))
mean4Ill = np.median((D4pIll.CondNumber))
mean5Ill = np.median((D5pIll.CondNumber))
mean6Ill = np.median((D6pIll.CondNumber))
mean7Ill = np.median((D7pIll.CondNumber))
mean8Ill = np.median((D8pIll.CondNumber))
mean9Ill = np.median((D9pIll.CondNumber))
mean10Ill = np.median((D10pIll.CondNumber))
mean11Ill = np.median((D11pIll.CondNumber))
mean12Ill = np.median((D12pIll.CondNumber))
mean13Ill = np.median((D13pIll.CondNumber))
old_mean_grad = mean_grad
old_log_sigma_grad = log_sigma_grad
#if (itt+1) % 10== 0:
#print(mean_grads_running_dot_product)
#print(sigma_grads_running_dot_product)
criterion1 = mean_grads_running_dot_product < 0
criterion2 = sigma_grads_running_dot_product < 0
criterion3 = np.abs(elbo_prev - elbo) < np.abs(
elbo_threshold_swa * elbo_prev)
criterion7 = s_mean < 0
criterion8 = s_log_var < 0
elbo_diff_list.append(elbo - elbo_prev)
elbo_diff_median = np.median(np.array(elbo_diff_list[-21:-1]))
elbo_diff_mean = np.mean(np.array(elbo_diff_list[-21:-1]))
elbo_diff_last_20 = elbo_diff_list[-20:-1]
#elbo_diff_max = np.max(np.array(elbo_diff_list[-21:-1]))
elbo_diff_list_abs = [0 for i in elbo_diff_last_20 if i < 0]
val1 = len(elbo_diff_list_abs) - np.count_nonzero(
np.asarray(elbo_diff_list_abs))
criterion4 = val1 > 5
criterion6 = itt > 4000
if len(elbo_mean_list) > 6:
criterion5 = np.abs(elbo_mean_list[-1] - elbo_mean_list[-2]
) < np.abs(elbo_mean_list[-2] -
elbo_mean_list[-5]) * 0.10
#if criterion1 and criterion2 and criterion3 and criterion6 and start_swa is False:
