def low_to_high_frc(lr_da,lr_grd,hr_grd,gt,time_coord,fill_value=0.0):
print('set up empty hr data array')
N = time_coord.size
dummy = np.tile(np.zeros(hr_grd['lon_'+gt].shape),(N,1,1))
x = hr_grd['xi_'+gt]
y = hr_grd['eta_'+gt]
z = time_coord
hr_da = xr.DataArray(dummy,coords=[z,y,x],dims=[time_coord.name,'eta_'+gt,'xi_'+gt])
print('Fill in the mask of lr data, resample to high resolution grid and fill hr mask with fill value: ',fill_value)
for k in np.arange(N):
print('processing time: ',k)
data = lr_da[k].values
valid_mask = ~np.isnan(data)
coords = np.array(np.nonzero(valid_mask)).T
values = data[valid_mask]
it = interpolate.NearestNDInterpolator(coords,values)
filled = it(list(np.ndindex(data.shape))).reshape(data.shape)
# Fill in known values on high res grid
hr_da[k] = resample(lr_grd['lon_'+gt].values,lr_grd['lat_'+gt].values,hr_grd['lon_'+gt].values,hr_grd['lat_'+gt].values,filled)
# Fill with zeros where mask is present
hr_da[k].values[hr_grd['mask_'+gt].values == 0] = fill_value
return hr_da
def make_clouds(sampling=240, seed=42):
# Set up the power spectrum for the clouds.
rad_fov = 1.8 #degrees? or radians?
windowsize = numpy.sqrt(2. * (2 * rad_fov)**2)
clouds = Clouds(windowsize, sampling)
# Set up the power spectrum for the clouds.
clouds.setupPowerSpectrum_SF()
# Generate the clouds in 'real space'.
clouds.DirectClouds(randomSeed=seed)
cloudimage = clouds.clouds
# The cloud extinction image is just an 'image' numbered from 0/0 to samplesize/samplesize
# So we need to add x/y locations appropriate to stellar x/y locations.
# PROBABLY NEED TO CHANGE THIS SCALING, APPROPRIATELY FOR TANGENT PLANE
x, y = numpy.mgrid[0:sampling, 0:sampling]
x = x / (numpy.max(x) / 2.0) - 1.0 # if focal plane goes from -1 to 1
y = y / (numpy.max(y) / 2.0) - 1.0 # probably need to change this.
cloudxy = numpy.column_stack([numpy.ravel(x), numpy.ravel(y)])
# Create an object we could use for interpolating cloud density at a point.
print(numpy.shape(cloudxy), numpy.shape(numpy.ravel(cloudimage)))
cloud_interp = interpolate.NearestNDInterpolator(cloudxy,
cloudimage.ravel())
# and plot
pylab.figure()
pylab.imshow(cloudimage)
pylab.title('Cloud from PWS: sampling %d, seed %d' % (sampling, seed))
return cloud_interp
def RescaleCLM(self, CLM_lon, CLM_lat, CLM_map, LCC_lon, LCC_lat):
"""FIXME! briefly describe function
:param CLM_lon:
:param CLM_lat:
:param CLM_map:
:param LCC_lon:
:param LCC_lat:
:returns:
:rtype:
"""
x, y = np.meshgrid(CLM_lon, CLM_lat)
n = x.size
F = RegularGridInterpolator.NearestNDInterpolator(
(np.resize(x, n), np.resize(y, n)), np.resize(CLM_map, n))
loni, lati = np.meshgrid(LCC_lon, LCC_lat)
CLM_map_i = F(loni, lati)
if self.plotoption == 4:
plt.figure(figsize=[20, 20])
plt.subplot(2, 1, 1)
plt.imshow(CLM_map)
plt.subplot(2, 1, 2)
plt.imshow(CLM_map_i[0::2, 0::2])
return CLM_map_i
def nn_interp_2d(in_x, in_t, value, uncertainty, out_x, out_t):
final_sig = []
excluded_inds = np.isnan(value)
y = value[~excluded_inds]
x = in_x[~excluded_inds]
t = in_t[~excluded_inds]
err = uncertainty[~excluded_inds]
def scale(signal, basis):
return (signal - min(basis)) / (max(basis) - min(basis))
t_scaled = scale(t, out_t)
out_t_scaled = scale(out_t, out_t)
x_scaled = scale(x, [0, 1]) #out_x)
out_x_scaled = scale(out_x, [0, 1]) #out_x)
get_value = interpolate.NearestNDInterpolator(
np.stack((t_scaled, x_scaled)).T, y)
import itertools
coords = np.array(list(itertools.product(out_t_scaled, out_x_scaled))).T
final_sig = get_value(coords[0], coords[1])
final_sig = final_sig.reshape((len(out_t_scaled), len(out_x_scaled)))
return final_sig
def interp(disp):
valid_mask = disp >= 0
coords = np.array(np.nonzero(valid_mask)).T
values = disp[valid_mask]
it = interpolate.NearestNDInterpolator(coords, values)
# it = interpolate.LinearNDInterpolator(coords, values, fill_value=0)
return it(list(np.ndindex(disp.shape))).reshape(disp.shape)
def process_outer_old(self):
"""Apply nearest neighbour interpolation to the corners of
the array. You might wonder why we don't just use this
method with a linear interpolator to do the whole thing: it
is because we need to serialise for multi processing that we
end up with a big class.
"""
for slice in self.outer_slices:
logging.info('using {}'.format(slice))
nans = self.invalid[slice]
logging.info('found {} nans'.format(nans.sum()))
valid_points = np.vstack(c[slice][~nans] for c in self.coords).T
valid_values = np.vstack(d[slice][~nans] for d in self.data).T
dim = valid_points.shape
logging.info('creating interpolator over {}'.format(dim))
interpolator = interp.NearestNDInterpolator(
valid_points, valid_values)
invalid_points = np.vstack(c[slice][nans] for c in self.coords).T
idim = invalid_points.shape
logging.info('interpolating over {}'.format(idim))
invalid_values = interpolator(invalid_points).astype(np.float32)
logging.info('copying data')
for i, d in enumerate(self.data):
# only copy across values that aren't nan
values = invalid_values[:, i]
good = ~np.isnan(values)
# see https://stackoverflow.com/questions/7179532
d[slice].flat[np.flatnonzero(nans)[good]] = values[good]
def plotClassMapNoPoints(optimizedModel,
labels,
prior="estimated",
do_interpolate=True,
cname="Spectral_r"):
"""Plots GTM class map without 2D representations. Internal usage.
"""
k = np.sqrt(optimizedModel.matX.shape[0])
n = 100
x = optimizedModel.matX[:, 0]
y = optimizedModel.matX[:, 1]
uniqClasses, label_numeric = np.unique(labels, return_inverse=True)
z = ugtm_landscape.classMap(optimizedModel, label_numeric,
prior).activityModel
if do_interpolate:
ti = np.linspace(-1.1, 1.1, n)
XI, YI = np.meshgrid(ti, ti)
f = interpolate.NearestNDInterpolator(optimizedModel.matX, z)
ZI = f(XI, YI)
plt.pcolor(XI, YI, ZI, cmap=plt.get_cmap(cname))
else:
plt.scatter(x,
y,
175 * (10 / k),
z,
cmap=plt.get_cmap(cname),
marker="s",
alpha=0.3)
plt.title('Class Map')
plt.xlim(-1.1, 1.1)
plt.ylim(-1.1, 1.1)
plt.axis('tight')
plt.xticks([]), plt.yticks([])
def extract_points(self, pid, points):
"""Extract values at certain points in the grid from a given parameter
set. Cells are selected by interpolating the centroids of the cells
towards the line using a "nearest" scheme.
Note that data is only returned for the points provided. If you want to
extract multiple data points along a line, defined by start and end
point, use the **extract_along_line** function.
Parameters
----------
pid : int
The parameter id to extract values from
points : Nx2 numpy.ndarray
(x, y) pairs
Returns
-------
values : numpy.ndarray (n x 1)
data values for extracted data points
"""
xy = self.grid.get_element_centroids()
data = self.parsets[pid]
iobj = spi.NearestNDInterpolator(xy, data)
values = iobj(points)
return values
def fit(self, x, y):
t0 = time.time()
assert set(y.flatten()) == set(
[0, 1]), "y data (target data) needs to have only 0s and 1s"
# determine the clusters
self.clus = sk.cluster.MiniBatchKMeans(n_clusters=self.n_clusters)
self.clus.fit(x)
# determine the nearest neighbor for each data point
nns = sk.neighbors.NearestNeighbors(n_neighbors=self.n_neighbors)
nns.fit(x)
neighbor_dist, neighbor_idx = nns.kneighbors(
self.clus.cluster_centers_, n_neighbors=self.n_neighbors)
# compute the cluster means
self.cluster_x = self.clus.cluster_centers_
self.cluster_y = array(
map(lambda i: nanmean(y[neighbor_idx[i, :]]),
xrange(shape(self.cluster_x)[0])))
# make the interpolator
if self.interpolator == 'linear':
self.iterpol = interpolate.LinearNDInterpolator(self.cluster_x,
self.cluster_y,
fill_value=nan)
else:
self.iterpol = interpolate.CloughTocher2DInterpolator(
self.cluster_x, self.cluster_y, fill_value=nan)
self.nnb_iterpol = interpolate.NearestNDInterpolator(
self.cluster_x, self.cluster_y)
print "fit elapsed time: %.02f minutes" % ((time.time() - t0) / 60.)
def plotLandscapeNoPoints(optimizedModel,
labels,
do_interpolate=True,
cname="Spectral_r"):
""" Plots GTM landscape without 2D representations. Internal usage.
"""
k = np.sqrt(optimizedModel.matX.shape[0])
n = 100
x = optimizedModel.matX[:, 0]
y = optimizedModel.matX[:, 1]
z = ugtm_landscape.landscape(optimizedModel, labels)
if do_interpolate:
ti = np.linspace(-1.1, 1.1, n)
XI, YI = np.meshgrid(ti, ti)
f = interpolate.NearestNDInterpolator(optimizedModel.matX, z)
ZI = f(XI, YI)
plt.pcolor(XI, YI, ZI, cmap=plt.get_cmap(cname))
else:
plt.scatter(x,
y,
50 * (10 / k),
z,
cmap=plt.get_cmap(cname),
marker="s")
plt.title('Landscape')
plt.xlim(-1.1, 1.1)
plt.ylim(-1.1, 1.1)
plt.colorbar()
plt.axis('tight')
plt.xticks([]), plt.yticks([])
def test_interpolation(mesh):
from scipy import interpolate
lons, lats = mesh.lons, mesh.lats
Z = np.exp(-lons**2 - lats**2)
lon = 2.*np.pi*np.random.random(10)
lat = np.arccos(2.*np.random.random(10) - 1.) - np.pi/2
# Stripy
zn1 = mesh.interpolate_nearest(lon, lat, Z)
zl1 = mesh.interpolate_linear(lon, lat, Z)
zc1 = mesh.interpolate_cubic(lon, lat, Z)
# cKDTree
tree = interpolate.NearestNDInterpolator((lons,lats), Z)
zn2 = tree(lon, lat)
# Least-squares spline
tri = interpolate.LinearNDInterpolator((lons,lats), Z)
zl2 = tri((lon, lat))
# Cubic spline
spl = interpolate.SmoothSphereBivariateSpline(lats+np.pi/2, lons, Z, s=1)
zc2 = spl.ev(lat+np.pi/2,lon)
print("squared residual in interpolation\n \
- nearest neighbour = {}\n \
- linear = {}\n \
- cubic = {}".format(((zn1 - zn2)**2).max(), \
((zl1 - zl2)**2).max(), \
((zc1 - zc2)**2).max()))
def inpaint_nearest(X):
idx = np.isfinite(X)
RI, CI = np.meshgrid(np.arange(X.shape[0]), np.arange(X.shape[1]))
f_near = interpolate.NearestNDInterpolator((RI[idx], CI[idx]), X[idx])
idx = ~idx
X[idx] = f_near(RI[idx], CI[idx])
return X
def nn_interp(x, y, vals, coord_box, inc):
"""
Performs scipy nearest-neighbor interpolation on the data
Assumes Tape scripts were run on a finer grid (try npts = 250)
the mins, maxes, and incriment should match that of other methods for easy comparison.
"""
xmin, xmax = coord_box[0], coord_box[1]
ymin, ymax = coord_box[2], coord_box[3]
newx = np.arange(xmin, xmax + inc[0] / 2, inc[0])
# needed to match other grids somehow
newy = np.arange(ymin, ymax + inc[1], inc[1])
tempvals = []
nn_interpolator = interp.NearestNDInterpolator((x, y), vals)
for i in range(len(newx)):
for j in range(len(newy)):
val = nn_interpolator(newx[i], newy[j])
tempvals.append(val)
newvals = []
for i in np.arange(0, len(tempvals), len(newy)):
newvals.append(tempvals[i:i + len(newy)])
newvals = np.transpose(newvals)
return newx, newy, newvals
def interpolate_by_scipy_nearest(src_data):
"""
Use Scipy to interpolate input source data to target grid.
------
Input:
src_data: the metadata on src_grids
Output:
tgt_data: the interpolated data on target grids.
Note: the following parameters are required
lat_src or lat_tgt: the latitude 1d np.array from source or target grids
lon_src or lon_tgt: the longitude 1d np.array from source or target grids
method: the interpolate method used by scipy RegularGridInterpolator
only available "linear" & "nearest"
"""
import scipy.interpolate as interpolator
global lat_src, lon_src, lat_tgt, lon_tgt
# Need prepare grid_points and vestor for scipy interpolator
src_data_vec = np.ravel(src_data)
grid_points_src = get_grid_points(lat_src, lon_src)
lat_tgt_vec, lon_tgt_vec = get_grid_vector(lat_tgt, lon_tgt)
# Define the interpolator
scipy_interp = interpolator.NearestNDInterpolator(grid_points_src, src_data_vec)
# Derive
tgt_data = scipy_interp(lat_tgt_vec, lon_tgt_vec)
# Need reshape back to 2d
tgt_data = tgt_data.reshape(len(lat_tgt), len(lon_tgt))
return tgt_data
def plotClassMap(optimizedModel, labels, prior="equiprobable",
do_interpolate=True, cname="Spectral_r", pointsize=1.0,
alpha=0.3):
k = math.sqrt(optimizedModel.matX.shape[0])
n = 100
x = optimizedModel.matX[:, 0]
y = optimizedModel.matX[:, 1]
uniqClasses, label_numeric = np.unique(labels, return_inverse=True)
z = ugtm_landscape.classMap(optimizedModel,
label_numeric, prior).activityModel
if do_interpolate:
ti = np.linspace(-1.1, 1.1, n)
XI, YI = np.meshgrid(ti, ti)
f = interpolate.NearestNDInterpolator(optimizedModel.matX, z)
ZI = f(XI, YI)
plt.pcolor(XI, YI, ZI, cmap=plt.get_cmap(cname))
else:
plt.scatter(x, y, 175*(10/k), z, cmap=plt.get_cmap(cname),
marker="s", alpha=0.3)
plt.scatter(optimizedModel.matMeans[:, 0], optimizedModel.matMeans[:, 1],
s=20*pointsize, c=np.squeeze(label_numeric),
cmap=plt.get_cmap(cname),
edgecolor='black', marker="o")
plt.title('Class Map')
plt.xlim(-1.1, 1.1)
plt.ylim(-1.1, 1.1)
plt.axis('tight')
plt.xticks([]), plt.yticks([])
def image_inpaint_pixels(img, valid_mask):
assert img.shape == valid_mask.shape
coords = np.array(np.nonzero(valid_mask)).T
values = img[valid_mask]
it = interpolate.NearestNDInterpolator(coords, values)
img_paint = it(list(np.ndindex(img.shape))).reshape(img.shape)
return img_paint
def __init__(self,
x,
y,
z,
interp_method: str = "linear",
rescale: bool = False,
**kw):
"""
Args:
rescale (bool): Rescales `x` and `y` data to (-0.5, 0.5) range.
Useful when working small/large scales.
If `True` you must take the input range into account when interpolating.
"""
assert {interp_method} <= {"linear", "nearest", "deg"}
points = np.column_stack((x, y))
if rescale:
xy_mean, xy_scale = calc_mean_and_scale(x, y)
self.xy_mean, self.xy_scale = xy_mean, xy_scale
scaled_pnts = scale(points, xy_mean=xy_mean, xy_scale=xy_scale)
del points
else:
scaled_pnts = points
if interp_method == "linear":
ip = interpolate.LinearNDInterpolator(scaled_pnts, z, **kw)
elif interp_method == "nearest":
ip = interpolate.NearestNDInterpolator(scaled_pnts, z, **kw)
elif interp_method == "deg":
ip = DegInterpolator(scaled_pnts, z, **kw)
self.rescale = rescale
self.ip = ip
def __init__(self, filename, verbose=1):
"""
Initialisation of class to load templates from a file and create the interpolation
objects
Parameters
----------
filename: string
Location of Template file
verbose: int
Verbosity level,
0 = no logging
1 = File + interpolation point information
2 = Detailed description of interpolation points
"""
self.verbose = verbose
if self.verbose:
print("Loading lookup tables from", filename)
grid, bins, template = self.parse_fits_table(filename)
x_bins, y_bins = bins
self.interpolator = interpolate.LinearNDInterpolator(grid,
template,
fill_value=0)
self.nearest_interpolator = interpolate.NearestNDInterpolator(
grid, template)
self.grid_interp = interpolate.RegularGridInterpolator(
(x_bins, y_bins),
np.zeros([x_bins.shape[0], y_bins.shape[0]]),
method="linear",
bounds_error=False,
fill_value=0)
def nn_interpolation(lon, lat, values, newlon, newlat, missing_value=-999):
#take all as 1-D arrays
lon1 = np.reshape(lon, lon.size)
lat1 = np.reshape(lat, lon.size)
val1 = np.reshape(values, lon.size)
#find indices to keep:
ikeep = np.where((lon1 >= newlon.min() - 0.5)
& (lon1 <= newlon.max() + 0.5)
& (lat1 >= newlat.min() - 0.5)
& (lat1 <= newlat.max() + 0.5))
#trim the arrays
lon1 = lon1[ikeep]
lat1 = lat1[ikeep]
val1 = val1[ikeep]
if val1.size == 0:
return
coords = np.asarray(zip(lon1, lat1))
f = interpolate.NearestNDInterpolator(coords, val1)
print f
#f = interpolate.LinearNDInterpolator(coords, asarray(valuelist),fill_value=missing_value)
res = f(newlon, newlat)
print len(res)
#for whatever the reason, for indices falling out the data coverage,
#interpolation doesn't result in nan by -8.??e+38
maskdef = np.zeros(res.shape, dtype=bool)
if hasattr(res, 'mask'):
maskdef = res.mask
#res[where((maskdef) | (isnan(res)) | (res<0) | (extramask) )]=missing_value
res[np.where((maskdef) | (np.isnan(res)) | (res < 0))] = missing_value
return np.ma.masked_equal(res, missing_value)
def nearest_interp(lon, lat, z, LON, LAT, undef=np.nan):
lon[lon > 80.0] = lon[lon > 80.0] - 360.0
I = np.where(np.isfinite(z))
points = np.array((lon[I].flatten(), lat[I].flatten())).swapaxes(0, 1)
zout = interpolate.NearestNDInterpolator(points, z[I].flatten())(LON, LAT)
return zout
def __init__(self, point_estimates, dates, strikes, uses_dates=False):
super().__init__(point_estimates, dates, strikes, uses_dates)
strikes, expiries = np.meshgrid(self.strikes, self.dates)
self.f = interp.NearestNDInterpolator(np.vstack(
[strikes.flatten(), expiries.flatten()]).T,
self.point_estimates.flatten(),
rescale=True)
def nearest_interp(lon, lat, z, LON, LAT, undef=np.nan):
lon[lon>80.0]=lon[lon>80.0]-360.0
points=np.array( (lon.flatten(), lat.flatten()) ).swapaxes(0, 1)
zout=interpolate.NearestNDInterpolator(points, z.flatten())(LON, LAT)
return zout
def image_inpaint_pixels(img, valid_mask):
assert img.shape == valid_mask.shape, \
'image size %s and mask size %s should be equal' \
% (repr(img.shape), repr(valid_mask.shape))
coords = np.array(np.nonzero(valid_mask)).T
values = img[valid_mask]
it = interpolate.NearestNDInterpolator(coords, values)
img_paint = it(list(np.ndindex(img.shape))).reshape(img.shape)
return img_paint
def image_inpaint_pixels(img, valid_mask):
if img.shape != valid_mask.shape:
raise ImageDimensionError(
'image size %r and mask size %r should be equal' %
(img.shape, valid_mask.shape))
coords = np.array(np.nonzero(valid_mask)).T
values = img[valid_mask]
it = interpolate.NearestNDInterpolator(coords, values)
img_paint = it(list(np.ndindex(img.shape))).reshape(img.shape)
return img_paint
def construct_nearest_interpolator((slice, coordinates, values)):
"""Construct a linear interpolator for given coordinates and values.
This function is here because it needs to be pickleable for
multiprocessing.
slice is just passed through as state that we need for post
processing the multiprocessing output.
"""
return slice, interp.NearestNDInterpolator(coordinates, values)
def NND(self):
coord_value = InterpolationMethods.df2array(self.topDF,
self.properties)
ref_p_radian = np.array(
list(map(lambda x: (radians(x[0]), radians(x[1])), coord_value)))
ref_values = np.array(coord_value[:, 2])
interp_array = Interp.NearestNDInterpolator(ref_p_radian, ref_values)
result = interp_array(self.query_point)
return round(float(result), 2)
def _interpolate_missing_data(data, mask, method='cubic'):
"""
Interpolate missing data as identified by the ``mask`` keyword.
Parameters
----------
data : 2D `~numpy.ndarray`
An array containing the 2D image.
mask : 2D bool `~numpy.ndarray`
A 2D booleen mask array with the same shape as the input
``data``, where a `True` value indicates the corresponding
element of ``data`` is masked. The masked data points are
those that will be interpolated.
method : {'cubic', 'nearest'}, optional
The method of used to interpolate the missing data:
* ``'cubic'``: Masked data are interpolated using 2D cubic
splines. This is the default.
* ``'nearest'``: Masked data are interpolated using
nearest-neighbor interpolation.
Returns
-------
data_interp : 2D `~numpy.ndarray`
The interpolated 2D image.
"""
from scipy import interpolate
data_interp = np.array(data, copy=True)
if len(data_interp.shape) != 2:
raise ValueError('data must be a 2D array.')
if mask.shape != data.shape:
raise ValueError('mask and data must have the same shape.')
y, x = np.indices(data_interp.shape)
xy = np.dstack((x[~mask].ravel(), y[~mask].ravel()))[0]
z = data_interp[~mask].ravel()
if method == 'nearest':
interpol = interpolate.NearestNDInterpolator(xy, z)
elif method == 'cubic':
interpol = interpolate.CloughTocher2DInterpolator(xy, z)
else:
raise ValueError('Unsupported interpolation method.')
xy_missing = np.dstack((x[mask].ravel(), y[mask].ravel()))[0]
data_interp[mask] = interpol(xy_missing)
return data_interp
def interpolate_aod_nearest(aod):
good_mask = aod != NULL_VALUE
# build the interpolation grid
xx, yy = np.meshgrid(np.arange(aod.shape[1]), np.arange(aod.shape[0]))
xym = np.vstack((np.ravel(xx[good_mask]), np.ravel(yy[good_mask]))).T
aod = np.ravel(aod[good_mask])
interp = interpolate.NearestNDInterpolator(xym, aod)
return interp(np.ravel(xx), np.ravel(yy)).reshape(xx.shape)
def __init__(
self,
csvfilename,
delimiter=";",
lengthscale=1.0,
pressurescale=1e-6 # Pa => MPa
):
data = np.loadtxt(csvfilename, delimiter=delimiter)
self.pts = data[:, 0:3] * lengthscale
self.p = data[:, 3] * pressurescale
self.pinterp = spi.NearestNDInterpolator(self.pts, self.p)
def _nearest_neighbor_interpolator(self):
"""
Creates a nearest neighbor interpolator object for this grid, which can
then be called repeatedly.
"""
indgrid = np.arange(self.grid.shape[0])
interpolator = interp.NearestNDInterpolator(self.grid.to(u.m).value, indgrid)
return interpolator
