def calc_subregion_area_mean_and_std(dataset_array, subregions):
''' Calculate area mean and standard deviation values for a given subregions using datasets on common grid points
:param dataset_array: An array of OCW Dataset Objects
:type list:
:param subregions: list of subregions
:type subregions: :class:`numpy.ma.array`
:returns: area averaged time series for the dataset of shape (ntime, nsubregion)
'''
ndata = len(dataset_array)
dataset0 = dataset_array[0]
if dataset0.lons.ndim == 1:
lons, lats = np.meshgrid(dataset0.lons, dataset0.lats)
else:
lons = dataset0.lons
lats = dataset0.lats
subregion_array = np.zeros(lons.shape)
mask_array = dataset_array[0].values[0,:].mask
# dataset0.values.shsape[0]: length of the time dimension
# spatial average
t_series =ma.zeros([ndata, dataset0.values.shape[0], len(subregions)])
# spatial standard deviation
spatial_std =ma.zeros([ndata, dataset0.values.shape[0], len(subregions)])
for iregion, subregion in enumerate(subregions):
lat_min, lat_max, lon_min, lon_max = subregion[1]
y_index,x_index = np.where((lats >= lat_min) & (lats <= lat_max) & (lons >= lon_min) & (lons <= lon_max))
subregion_array[y_index,x_index] = iregion+1
for idata in np.arange(ndata):
t_series[idata, :, iregion] = ma.mean(dataset_array[idata].values[:,y_index, x_index], axis=1)
spatial_std[idata, :, iregion] = ma.std(dataset_array[idata].values[:,y_index, x_index], axis=1)
subregion_array = ma.array(subregion_array, mask=mask_array)
return t_series, spatial_std, subregion_array
def __call__(self, data, weight_id=0, progress_callback=None):
array, classes, w = data.toNumpyMA()
domain = data.domain
if isinstance(domain.class_var, Orange.feature.Discrete):
# Discrete class (extend the data with class indicator matrix)
nval = len(data.domain.class_var.values)
ext = ma.zeros((len(array), nval))
ext[([i for i, m in enumerate(classes.mask) if m],
[int(c) for c, m in zip(classes, classes.mask) if m])] = 1.0
elif isinstance(domain.class_var, Orange.feature.Continuous):
# Continuous class, just add the one column (what about multitarget)
nval = 1
ext = ma.zeros((len(array), nval))
ext[:,0] = classes
elif domain.class_var is None:
# No class var
nval = 0
ext = ma.zeros((len(array), nval))
else:
raise TypeError("Unsuported `class_var` %r" % domain.class_var)
array = ma.hstack((array, ext))
map = Map(self.map_shape, topology=self.topology)
if self.initialize == Map.InitializeLinear:
map.initialize_map_linear(array)
else:
map.initialize_map_random(array)
map = Solver(batch_train=self.batch_train, eps=self.eps, neighbourhood=self.neighbourhood,
radius_ini=self.radius_ini, radius_fin=self.radius_fin, learning_rate=self.learning_rate,
epoch=self.epochs)(array, map, progress_callback=progress_callback)
# Remove class columns from the vectors
for node in map:
node.vector = node.vector[:-nval]
return SOMMap(map, data)
def calcTemporalCorrelation(evaluationData, referenceData):
'''
Purpose ::
Calculate the temporal correlation.
Assumption(s) ::
The first dimension of two datasets is the time axis.
Input ::
evaluationData - model data array of any shape
referenceData- observation data array of any shape
Output::
temporalCorelation - A 2-D array of temporal correlation coefficients at each subregion
sigLev - A 2-D array of confidence levels related to temporalCorelation
REF: 277-281 in Stat methods in atmos sci by Wilks, 1995, Academic Press, 467pp.
sigLev: the correlation between model and observation is significant at sigLev * 100 %
'''
evaluationDataMask = process.create_mask_using_threshold(evaluationData, threshold = 0.75)
referenceDataMask = process.create_mask_using_threshold(referenceData, threshold = 0.75)
nregion = evaluationData.shape[0]
temporalCorrelation = ma.zeros([nregion])-100.
sigLev = ma.zeros([nregion])-100.
for iregion in np.arange(nregion):
temporalCorrelation[iregion], sigLev[iregion] = stats.pearsonr(evaluationData[iregion,:], referenceData[iregion,:])
sigLev[iregion] = 1 - sigLev[iregion]
temporalCorrelation=ma.masked_equal(temporalCorrelation.data, -100.)
sigLev=ma.masked_equal(sigLev.data, -100.)
return temporalCorrelation, sigLev
def grid_linefit(grid, timevals=None, timeslice=slice(None, None, None)):
"""A compressed spatiotemporal grid is provided. A line fit is performed
along the time axis for each spatial cell. Two grids are returned,
each of which is 2d, with the same spatial shape as the input.
The pixels of one grid contains the slope, the other contains the
r squared value of the line fit for that spatial cell.
A vector of time values may be provided. If not supplied, one
will be generated."""
if timevals == None:
timevals = ma.arange(grid.shape[0])
X = sm.add_constant(timevals, prepend=True)
outshape = (grid.shape[1],)
rsq_map = ma.zeros(outshape)
slope_map = ma.zeros(outshape)
for i in range(outshape[0]):
if (i % 1000) == 0:
print "%d of %d (%f)" % (i, outshape[0], (i * 100.0) / outshape[0])
if (type(grid) == "numpy.ma.core.MaskedArray") and grid[0, :].mask[i]:
rsq_map[i] = ma.masked
slope_map[i] = ma.masked
else:
m, rsq = linefit(grid, i, X, timeslice)
rsq_map[i] = rsq
slope_map[i] = m
return (slope_map, rsq_map)
def convert_unary_evaluation_result(evaluation_result, subregion = False):
if not subregion:
nmetric = len(evaluation_result)
nmodel = len(evaluation_result[0])
results = []
for imetric in range(nmetric):
result_shape = list(evaluation_result[imetric][0].shape)
result_shape.insert(0, nmodel)
result = ma.zeros(result_shape)
for imodel in range(nmodel):
result[imodel,:] = evaluation_result[imetric][imodel]
results.append(result)
return results
else:
nmetric = len(evaluation_result)
nsubregion = len(evaluation_result[0])
nmodel = len(evaluation_result[0][0])
results = []
for isubregion in range(nsubregion):
subregion_results = []
for imetric in range(nmetric):
result_shape = list(evaluation_result[imetric][isubregion][0].shape)
result_shape.insert(0, nmodel)
result = ma.zeros(result_shape)
for imodel in range(nmodel):
result[imodel,:] = evaluation_result[imetric][isubregion][imodel]
subregion_results.append(result)
results.append(subregion_results)
return results
def calculate(self, tracks):
dataMean = ma.zeros((len(self.lon_range) - 1,
len(self.lat_range) - 1))
dataMin = ma.zeros((len(self.lon_range) - 1,
len(self.lat_range) - 1))
dataMax = ma.zeros((len(self.lon_range) - 1,
len(self.lat_range) - 1))
dataMed = ma.zeros((len(self.lon_range) - 1,
len(self.lat_range) - 1))
log.debug("Processing %d tracks" % (len(tracks)))
for cell in self.gridCells:
vcell = np.array([])
for t in tracks:
ii = np.where(((t.Latitude >= cell.ymin) &
(t.Latitude < cell.ymax)) &
((t.Longitude >= cell.xmin) &
(t.Longitude < cell.xmax)))[0]
if len(ii) > 0:
vv = t.CentralPressure[ii].compress(t.CentralPressure[ii] < sys.maxint)
vcell = np.append(vcell, vv.compress(vv > 0.0))
if len(vcell > 0):
dataMean[cell.index[0], cell.index[1]] = np.mean(vcell)
dataMin[cell.index[0], cell.index[1]] = np.min(vcell)
dataMax[cell.index[0], cell.index[1]] = np.max(vcell)
dataMed[cell.index[0], cell.index[1]] = np.median(vcell)
dataMean = ma.masked_equal(dataMean, 0)
dataMin = ma.masked_equal(dataMin, 0)
dataMax = ma.masked_equal(dataMax, 0)
dataMed = ma.masked_equal(dataMed, 0)
return dataMean, dataMin, dataMax, dataMed
def sum(self, var, agg, lats, weights = None, calcarea = False, mask = None, numchunks = 1):
nt, nlats, nlons = var.shape
if weights is None: # weights
weights = ones((nt, nlats, nlons))
elif len(weights.shape) == 2:
weights = ma.resize(weights, (nt, nlats, nlons))
if calcarea: # area
area = self.area(lats, nlats, nlons)
else:
area = ones((nlats, nlons))
aggvals = self.__uniquevals(agg)
sz = len(aggvals)
if mask is None:
varmask = ones((nt, nlats, nlons)) # no additional mask
else:
varmask = mask
chunksize = sz / numchunks # chunk data to reduce memory usage
sumv = ma.masked_array(zeros((sz, nt)), mask = ones((sz, nt)))
maxchunksize = max(chunksize, chunksize + sz - chunksize * numchunks)
aselect = ma.zeros((maxchunksize, nlats, nlons), dtype = bool) # preallocate
vartmp = ma.zeros((maxchunksize, nlats, nlons))
cnt = 0
for i in range(numchunks):
startidx = cnt
if i != numchunks - 1:
endidx = cnt + chunksize
else:
endidx = sz
aggvalsc = aggvals[startidx : endidx] # work on subset of aggregation values
szc = len(aggvalsc)
aselect[:] = 0 # clear
for j in range(szc): aselect[j] = (agg == aggvalsc[j])
ridx, latidx, lonidx = where(aselect)
vartmp[:] = 0 # clear
vartmp.mask = ones(vartmp.shape)
for t in range(nt):
vartmp[ridx, latidx, lonidx] = var[t, latidx, lonidx] * \
varmask[t, latidx, lonidx] * \
weights[t, latidx, lonidx] * \
area[latidx, lonidx] * \
aselect[ridx, latidx, lonidx]
sumv[startidx : endidx, t] = vartmp.sum(axis = 2).sum(axis = 1)[: szc]
cnt += chunksize
return sumv
def writeBN_lola(fileName, lons, lats):
# write a binary data file that include longitude (1-d) and latitude (1-d) values
F = fortranfile.FortranFile(fileName, mode='w')
ngrdY = lons.shape[0]; ngrdX = lons.shape[1]
tmpDat = ma.zeros(ngrdX); tmpDat[:] = lons[0, :]; F.writeReals(tmpDat)
tmpDat = ma.zeros(ngrdY); tmpDat[:] = lats[:, 0]; F.writeReals(tmpDat)
# release temporary arrays
tmpDat = 0
F.close()
def GetVariableData(self, col, vtype = None):
"""
This is an important method. It returns the data for a single variable.
Empty variable returns None
Blanks after data are not returned
Variable type can be specified as string (or None), int or float
For string or None, all raw values are returned.
For int, all figures are rounded to nearest decimal place and values that
cannot be converted are masked.
For float, all values that cannot be converted to float are masked.
All values matching the variable's 'missingvalue' are masked.
It's inefficient and requires 2 passes but it should be reliable.
"""
maxRow = self.GetNumberRows()
meta = self.meta[col]
missing = meta["missingvalues"]
if col > self.GetNumberCols():
return None
maxIdx = -1
for idx in range(maxRow):
if not self.CheckBlank(self.GetCellValue(idx, col)):
maxIdx = idx
if maxIdx < 0:
return None
maxIdx = maxIdx + 1
data = []
if (vtype == None) or (vtype.lower() == "str"):
for row in range(maxIdx):
val = self.GetCellValue(row, col)
if self.CheckBlank(val):
val = ""
data.append(val)
elif vtype.lower() == "int":
data = ma.zeros(maxIdx,dtype='int')
for row in range(maxIdx):
val = self.GetCellValue(row, col)
if (val == missing) or (self.CheckBlank(val)):
data[row] = ma.masked
else:
try:
data[row] = int(round(float(val)))
except ValueError:
data[row] = ma.masked
elif vtype.lower() == "float":
data = ma.zeros(maxIdx,dtype='float')
for row in range(maxIdx):
val = self.GetCellValue(row, col)
if (val == missing) or (self.CheckBlank(val)):
data[row] = ma.masked
else:
try:
data[row] = float(val)
except ValueError:
data[row] = ma.masked
return data
def blockify(x, y):
"""Convert x and y vectors so that they can be used for line plots where
the y values are piecewise constant
x needs to be y.size + 1"""
assert x.size == y.size + 1, "x should be one value longer than y"
x_repeat, y_repeat = ma.zeros(2*y.size), ma.zeros(2*y.size)
x_repeat[::2], x_repeat[1::2] = x[:-1], x[1:]
y_repeat[::2], y_repeat[1::2] = y, y
return x_repeat, y_repeat
def test_discarded_mask():
data_in = ma.zeros((10,), dtype=[('wlen', float), ('flux', float)])
data_out = np.zeros((10,), dtype=[('wlen', float), ('flux', float)])
with pytest.raises(ValueError):
redshift(z_in=0, z_out=0, data_in=data_in, data_out=data_out)
wlen = np.arange(10)
flux = ma.zeros((10,))
with pytest.raises(ValueError):
redshift(z_in=0, z_out=0, data_out=data_out, rules=[
{'name': 'wlen', 'exponent': +1, 'array_in': wlen},
{'name': 'flux', 'exponent': -1, 'array_in': flux}])
def overlapping_chars(ann1, annotations):
"""Returns the number of chars in ann1 that overlap with the annotations."""
annotations = [ann2 for ann2 in annotations if is_overlapping(ann1, ann2)]
if len(annotations) == 0 or not isinstance(ann1, Annotation):
return 0
this_overlaps = zeros(ann1[TLEN], dtype=bool)
source_overlaps = zeros(ann1[SLEN], dtype=bool)
for ann2 in annotations:
mark_overlapping_chars(this_overlaps, ann1, ann2, TOFF, TLEN)
mark_overlapping_chars(source_overlaps, ann1, ann2, SOFF, SLEN)
return npsum(this_overlaps) + npsum(source_overlaps)
def find(a):
"""
use straightforward summing of mask criteria
"""
m1 = ma.zeros(a.matrix.shape)
m2 = ma.zeros(a.matrix.shape)
# look around it's neighbourhood
for i in [-1,0,1]:
for j in [-1,0,1]:
m1 += (a.shiftMatrix(i,j).matrix.mask==False) # finding a point not masked
m2 += (a.shiftMatrix(i,j).matrix.mask==True ) # finding a point masked
return m1*m2
def comp_p(dert__, map, rng): # compare rng-distant pixels within blob
p__ = dert__[:, :, 0]
mask = ~map # complemented blob.map is a mask of array
dy__ = ma.zeros(map.shape, dtype=int) # initialize dy__ as array masked for selective computation
dx__ = ma.zeros(map.shape, dtype=int)
dy__.mask = dx__.mask = mask # all operations on masked arrays ignore elements at mask == True.
# vertical comp:
d__ = p__[rng:] - p__[:-rng] # comparison between p at coordinates (x, y) and p at coordinates (x, y+ rng)
dy__[rng:] += d__ # bilateral accumulation on dy (x, y+ rng)
dy__[:-rng] += d__ # bilateral accumulation on dy (x, y)
# horizontal comp:
d__ = p__[:, rng:] - p__[:, :-rng] # comparison between p (x, y) and p (x + rng, y)
dx__[:, rng:] += d__ # bilateral accumulation on dx (x + rng, y)
dx__[:, :-rng] += d__ # bilateral accumulation on dx (x, y)
# diagonal comparison:
for xd in range(1, rng):
yd = rng - xd # y and x distance between comparands
hyp = hypot(xd, yd)
y_coef = yd / hyp # to decompose d into dy
x_coef = xd / hyp # to decompose d into dx
# top-left and bottom-right quadrants:
d__ = p__[yd:, xd:] - p__[:-yd, :-xd] # comparison between p (x, y) and p (x + xd, y + yd)
# decompose d to dy, dx:
temp_dy__ = d__ * y_coef # buffer for dy accumulation
temp_dx__ = d__ * x_coef # buffer for dx accumulation
# accumulate dy, dx:
dy__[yd:, xd:] += temp_dy__.astype(int) # bilateral accumulation on dy (x + xd, y + yd)
dy__[:-yd, :-xd] += temp_dy__.astype(int) # bilateral accumulation on dy (x, y)
dx__[yd:, xd:] += temp_dx__.astype(int) # bilateral accumulation on dx (x + xd, y + yd)
dx__[:-yd, :-xd] += temp_dx__.astype(int) # bilateral accumulation on dx (x, y)
# top-right and bottom-left quadrants:
d__ = p__[yd:, :-xd] - p__[:-yd, xd:] # comparison between p (x + xd, y) and p (x, y + yd)
# decompose d to dy, dx:
temp_dy__ = d__ * y_coef # buffer for dy accumulation
temp_dx__ = -(d__ * x_coef) # buffer for dx accumulation, sign inverted with comp direction
# accumulate dy, dx:
dy__[yd:, :-xd] += temp_dy__.astype(int) # bilateral accumulation on dy (x, y + yd)
dy__[:-yd, xd:] += temp_dy__.astype(int) # bilateral accumulation on dy (x + xd, y)
dx__[yd:, :-xd] += temp_dx__.astype(int) # bilateral accumulation on dx (x, y + yd)
dx__[:-yd, xd:] += temp_dx__.astype(int) # bilateral accumulation on dx (x + xd, y)
dert__[:, :, 2] += dx__ # add dx to shorter-rng-accumulated dx
dert__[:, :, 3] += dy__ # add dy to shorter-rng-accumulated dy
# ---------- comp_p() end -------------------------------------------------------------------------------------------
def writeBNdata(fileName, numOBSs, numMDLs, nT, ngrdX, ngrdY, numSubRgn, obsData, mdlData, obsRgnAvg, mdlRgnAvg):
#(fileName,maskOption,numOBSs,numMDLs,nT,ngrdX,ngrdY,numSubRgn,obsData,mdlData,obsRgnAvg,mdlRgnAvg):
# write spatially- and regionally regridded data into a binary data file
missing = -1.e26
F = fortranfile.FortranFile(fileName, mode='w')
# construct a data array to replace mask flag with a missing value (missing=-1.e12) for printing
data = ma.zeros((nT, ngrdY, ngrdX))
for m in np.arange(numOBSs):
data[:, :, :] = obsData[m, :, :, :]; msk = data.mask
for n in np.arange(nT):
for j in np.arange(ngrdY):
for i in np.arange(ngrdX):
if msk[n, j, i]: data[n, j, i] = missing
# write observed data. allowed to write only one row at a time
tmpDat = ma.zeros(ngrdX)
for n in np.arange(nT):
for j in np.arange(ngrdY):
tmpDat[:] = data[n, j, :]
F.writeReals(tmpDat)
# write model data (dep. on the number of models).
for m in np.arange(numMDLs):
data[:, :, :] = mdlData[m, :, :, :]
msk = data.mask
for n in np.arange(nT):
for j in np.arange(ngrdY):
for i in np.arange(ngrdX):
if msk[n, j, i]:
data[n, j, i] = missing
for n in np.arange(nT):
for j in np.arange(ngrdY):
tmpDat[:] = data[n, j, :]
F.writeReals(tmpDat)
data = 0 # release the array allocated for data
# write data in subregions
if numSubRgn > 0:
print 'Also included are the time series of the means over ', numSubRgn, ' areas from obs and model data'
tmpDat = ma.zeros(nT); print numSubRgn
for m in np.arange(numOBSs):
for n in np.arange(numSubRgn):
tmpDat[:] = obsRgnAvg[m, n, :]
F.writeReals(tmpDat)
for m in np.arange(numMDLs):
for n in np.arange(numSubRgn):
tmpDat[:] = mdlRgnAvg[m, n, :]
F.writeReals(tmpDat)
tmpDat = 0 # release the array allocated for tmpDat
F.close()
def shiibaLocal(a, b, windowSize=100, iRange=range(000, 881, 100),\
jRange=range(000, 921, 100), searchWindowHeight=11,\
searchWindowWidth=11, useRecursion=True, plotting=True ):
# imports
from shiiba import regression2 as regression
from shiiba import regressionCFLfree as cflfree
import numpy.ma as ma
# results =dictionary with
# {'mn': mn, 'C':C, 'Rsquared':Rsquared, 'CR2':CR2, 'timeSpent':timeSpent}
results=cflfree.regressLocalAll(a, b, windowSize, iRange, jRange, searchWindowHeight,\
searchWindowWidth, useRecursion, plotting)
mn = results['mn']
C = results['C']
Rsquared = results['Rsquared']
# constructing the prediction
a1 = dbz(name = 'shiiba prediction for %s and %s' % (a.name, b.name),
matrix = ma.zeros(a.matrix.shape))
a1.matrix.fill_value = a.matrix.fill_value
a1.mask = True
# constructing the vector field
U = ma.zeros(a.matrix.shape)
U.fill_value = a.matrix.fill_value
U.mask = True
V = ma.zeros(a.matrix.shape)
V.fill_value = a.matrix.fill_value
V.mask = True
vect = pattern.VectorField(U=U, V=V, \
title='shiiba prediction for %s and %s' % (a.name, b.name))
# filling in the local values
for i, j in mn.keys():
aa = a.getWindow(i, j, windowSize)
a1.matrix[i:i+windowSize, j:j+windowSize] = regression.getPrediction(C[(i,j)], aa)
vectLocal = regression.convert(C[(i,j)], aa)
vect.U[i:i+windowSize, j:j+windowSize] = vectLocal.U
vect.V[i:i+windowSize, j:j+windowSize] = vectLocal.V
#########
# added 15 july 2013 ; doesn't make sense to have a global vector map without adding in the mn[(i,j)]'s
# adding the shift back to the regression result
# (see pattern.py and regression2.py)
vect.U[i:i+windowSize, j:j+windowSize] += mn[(i,j)][0] # U = first (i-) component ; V = j-component
vect.V[i:i+windowSize, j:j+windowSize] += mn[(i,j)][1]
#
##########
results['prediction'] = a1
results['vect'] = vect
return results
def shiftgrid(lon0, datain, lonsin, start= True, cyclic=360.0):
"""
Purpose::
Shift global lat/lon grid east or west. This function is taken directly
from the (unreleased) basemap 1.0.7 source code as version 1.0.6 does not
currently support arrays with more than two dimensions.
https://github.com/matplotlib/basemap
Input::
lon0 - starting longitude for shifted grid (ending longitude if start=False).
lon0 must be on input grid (within the range of lonsin).
datain - original data with longitude the right-most dimension.
lonsin - original longitudes.
start - if True, lon0 represents the starting longitude of the new grid.
if False, lon0 is the ending longitude. Default True.
cyclic - width of periodic domain (default 360)
Output::
dataout - data on shifted grid
lonsout - lons on shifted grid
"""
if np.fabs(lonsin[-1]-lonsin[0]-cyclic) > 1.e-4:
# Use all data instead of raise ValueError, 'cyclic point not included'
start_idx = 0
else:
# If cyclic, remove the duplicate point
start_idx = 1
if lon0 < lonsin[0] or lon0 > lonsin[-1]:
raise ValueError('lon0 outside of range of lonsin')
i0 = np.argmin(np.fabs(lonsin-lon0))
i0_shift = len(lonsin)-i0
if ma.isMA(datain):
dataout = ma.zeros(datain.shape,datain.dtype)
else:
dataout = np.zeros(datain.shape,datain.dtype)
if ma.isMA(lonsin):
lonsout = ma.zeros(lonsin.shape,lonsin.dtype)
else:
lonsout = np.zeros(lonsin.shape,lonsin.dtype)
if start:
lonsout[0:i0_shift] = lonsin[i0:]
else:
lonsout[0:i0_shift] = lonsin[i0:]-cyclic
dataout[...,0:i0_shift] = datain[...,i0:]
if start:
lonsout[i0_shift:] = lonsin[start_idx:i0+start_idx]+cyclic
else:
lonsout[i0_shift:] = lonsin[start_idx:i0+start_idx]
dataout[...,i0_shift:] = datain[...,start_idx:i0+start_idx]
return dataout,lonsout
def handWritingClassTest(trainingDirName, testDirName, width, height, fileNameSplit, fileStrSplit):
from os import listdir
from numpy.ma import zeros
hwLabels = []
traningFileList = listdir(trainingDirName)
m = len(traningFileList)
trainingMat = zeros((m, width * height))
for i in range(m):
fileNameStr = traningFileList[i]
fileStr = fileNameStr.split(fileNameSplit)[0]
classNumStr = int(fileStr.split(fileStrSplit)[0])
hwLabels.append(classNumStr)
trainingMat[i, :] = KNN.img2vector(filename=trainingDirName + "/%s" % fileNameStr, width=width, height=height)
testFileList = listdir(testDirName)
errorCount = 0.0
mTest = len(testFileList)
for i in range(mTest):
fileNameStr = testFileList[i]
fileStr = fileNameStr.split(fileNameSplit)[0]
classNumStr = int(fileStr.split(fileStrSplit)[0])
vectorUnderTest = KNN.img2vector(filename=testDirName + "/%s" % fileNameStr, width=width, height=height)
classifierResult = KNN.classify0(inx=vectorUnderTest, dataSet=trainingMat, labels=hwLabels, k=3)
print("the classifier came back with: %d, the real answer is: %d" % (classifierResult, classNumStr))
if(classifierResult != classNumStr): errorCount += 1.0
print("\nthe total number of errors is: %d" % errorCount)
print("\nthe total error rate is: %f" % (errorCount/float(mTest)))
def temporal_rebin_with_time_index(target_dataset, nt_average):
""" Rebin a Dataset to a new temporal resolution
:param target_dataset: Dataset object that needs temporal rebinned
:type target_dataset: :class:`dataset.Dataset`
:param nt_average: Time resolution for the output datasets.
It is the same as the number of time indicies to be averaged. (length of time dimension in the rebinned dataset) = (original time dimension length/nt_average)
:type temporal_resolution: integer
:returns: A new temporally rebinned Dataset
:rtype: :class:`dataset.Dataset`
"""
nt = target_dataset.times.size
if nt % nt_average !=0:
print 'Warning: length of time dimension must be a multiple of nt_average'
# nt2 is the length of time dimension in the rebinned dataset
nt2 = nt/nt_average
binned_dates = target_dataset.times[np.arange(nt2)*nt_average]
binned_values = ma.zeros(np.insert(target_dataset.values.shape[1:],0,nt2))
for it in np.arange(nt2):
binned_values[it,:] = ma.average(target_dataset.values[nt_average*it:nt_average*it+nt_average,:], axis=0)
new_dataset = ds.Dataset(target_dataset.lats,
target_dataset.lons,
binned_dates,
binned_values,
variable=target_dataset.variable,
units=target_dataset.units,
name=target_dataset.name,
origin=target_dataset.origin)
return new_dataset
def __init__(self, data=None, copy=True) :
"""Can either be initialized with a raw data array or with None"""
# Dictionary that holds all data other than .data. This is safe to
# be accessed and updated by the user.
self.field = {}
# Dictionary with the same keys as field but holds the axes over which
# a parameter varies. For instance, the LST variable varies over the
# 'time' axis. axes['LST'] should thus be ('time') and
# shape(field['LST']) should be (ntimes, ).
self.field_axes = {}
# To write data to fits you need a fits format for each field.
self.field_formats = {}
# Dictionary that holds the history of this data. It's keys are
# history entries for hte data. They must be strings starting with a
# three digit integer ennumerating the histories. The corresponding
# values give additional details, held in a tuple of strings. The
# intension is that when merging data, histories must be identical, but
# details can be merged.
self.history = History()
if data is None :
self.data = ma.zeros(tuple(sp.zeros(len(self.axes))), float)
self.data_set = False
else :
self.set_data(data, copy=copy)
def mask_seafloor(input, kmt, axis=0):
""" def mask_seafloor(input, kmt):
default is that input has depth (zt) as first dimension.
axis is axis of depth dimension.
Change axis=1 if time is first and depth is second.
returns input as masked array
"""
tmp = ma.zeros(input.shape)
ndepth = input.shape[axis]
if axis==1:
# need to tile kmt
kmtt = np.tile(kmt,(input.shape[0],1,1))
for lii in np.arange(0,ndepth): # loop through each depth layer
# mask out levels below sea floor
if axis==0:
tmp[lii,...] = ma.masked_where(kmt <= lii, input[lii,...])
elif axis==1:
tmp[:,lii,...] = ma.masked_where(kmtt <= lii, input[:,lii,...])
else:
print 'axis must = 1 or 0'
return tmp
def getKmeans(a, k, threshold=1, iter=40, thresh=1e-05, minit="random", missing="warn"):
"""input : a, k threshold
output : atk
"""
if minit == "matrix":
seeds, k = k, len(k)
a.k = k # initialise (could move it to __init__ but not bothered for the moment)
height, width = a.matrix.shape
pixels = a.matrix > threshold
print "width, height:", width, height # debug
print "sum of relevant pixels:", sum(sum(pixels)) # debug
dataPoints = [[(i, j) for i in range(width) if pixels[j, i]] for j in range(height)]
dataPoints = sum(dataPoints, [])
dataPoints = np.array(dataPoints)
print dataPoints[:20]
if minit == "matrix":
a.centroids = kmeans2(data=dataPoints, k=seeds, iter=iter, thresh=thresh, minit=minit, missing=missing)
else:
a.centroids = kmeans2(data=dataPoints, k=k, iter=iter, thresh=thresh, minit=minit, missing=missing)
a.data = dataPoints
resultPattern = ma.zeros((height, width))
resultPattern.mask = True
resultPattern.fill_value = -999
for i in range(len(dataPoints)):
resultPattern[dataPoints[i][1], dataPoints[i][0]] = a.centroids[1][i]
resultPattern = dbz(
name="Clustering for %s with %d clusters" % (a.name, k + 1), matrix=resultPattern, vmin=0, vmax=k
)
atk = {"centroids": a.centroids, "data": a.data, "pattern": resultPattern}
return atk
def _calculate(self, tracks):
"""
Calculate a histogram of TC genesis counts given a set of tracks.
:param tracks: Collection of :class:`Track` objects.
"""
hist = ma.zeros((len(self.lon_range) - 1,
len(self.lat_range) - 1))
positions = np.vstack([self.X.ravel(), self.Y.ravel()])
x = []
y = []
for track in tracks:
if len(track.Longitude) == 0:
pass
elif len(track.Longitude) == 1:
x.append(track.Longitude)
y.append(track.Latitude)
else:
x.append(track.Longitude[0])
y.append(track.Latitude[0])
values = np.vstack([np.array(x), np.array(y)])
kernel = gaussian_kde(values, bw_method=.01)
Z = np.reshape(kernel(positions).T, self.X.shape)
return Z.T
def average(tsl):
# find fastest dt:
dt_common = 1e12
for ts in tsl:
if ts.dt < dt_common:
newtime = ts.timearray()
dt_common = ts.dt
# interpolate all series to new dt:
tslip = [ts.interpolate(newtime, dt_common) for ts in tsl]
if len(tslip[0].data.shape)==1:
ave = np.empty((len(tslip), len(tslip[0].data)))
else:
ave = np.empty((len(tslip), tslip[0].data.shape[0], tslip[0].data.shape[1]))
for its, ts in enumerate(tslip):
if len(ts.data.shape)==1:
ave[its] = ts.data
else:
ave[its,:,:] = ts.data[:,:]
if len(ts.data.shape)==1:
return Timeseries(ma.mean(ave, axis=0), dt_common)
else:
avef = ma.zeros((tslip[0].data.shape[0], tslip[0].data.shape[1]))
for nrow, row in enumerate(avef):
avef[nrow,:] = ma.mean(ave[:,nrow,:], axis=0)
return Timeseries(avef, dt_common)
def any_of(self, *states, **kwargs):
'''
Return a boolean array containing True where the value of the
MappedArray equals any state in states.
:param states: List of states.
:type states: [str]
:param ignore_missing: If this is False, raise an exception if any of
the states are not in the values mapping.
:type ignore_missing: bool
:returns: Boolean array.
:rtype: np.ma.array(bool)
'''
ignore_missing = kwargs.get('ignore_missing', False)
valid_states = self.values_mapping.values()
array = zeros(len(self), dtype=bool_)
for state in states:
if state not in valid_states:
if ignore_missing:
# do not check array as invalid states cause
# exception level log messages.
continue
else:
raise ValueError(
"State '%s' is not valid. Valid states: '%s'." %
(state, valid_states))
array |= self == state
return array
def __init__(self, MetricTable):
# Create empty ratio table
nprobs = MetricTable.nprobs
nsolvs = MetricTable.nsolvs
self.ratios = ma.masked_array(1.0 * ma.zeros((nprobs+1, nsolvs)))
# Compute best relative performance ratios across
# solvers for each problem
for prob in range(nprobs):
metrics = MetricTable.prob_mets(prob)
best_met = ma.minimum(metrics)
if (ma.count(metrics)==nsolvs and
ma.maximum(metrics)<=opts.minlimit):
self.ratios[prob+1,:] = 1.0;
else:
self.ratios[prob+1,:] = metrics * (1.0 / best_met)
# Sort each solvers performance ratios
for solv in range(nsolvs):
self.ratios[:,solv] = ma.sort(self.ratios[:,solv])
# Compute largest ratio and use to replace failures entries
self.maxrat = ma.maximum(self.ratios)
self.ratios = ma.filled(self.ratios, 1.01 * self.maxrat)
def __init__(self, MetricTable, opts):
epsilon = 0.0
if opts.cpu:
epsilon = 0.01
# Create empty ratio table
nprobs = MetricTable.nprobs
nsolvs = MetricTable.nsolvs
self.ratios = ma.zeros((nprobs, nsolvs), dtype=numpy.float)
# Compute best relative performance ratios across
# solvers for each problem
for prob in range(nprobs):
metrics = MetricTable.prob_mets(prob) + epsilon
best_met = ma.minimum(metrics)
self.ratios[prob,:] = metrics * (1.0 / best_met)
# Sort each solvers performance ratios
for solv in range(nsolvs):
self.ratios[:,solv] = ma.sort(self.ratios[:,solv])
# Compute largest ratio and use to replace failure entries
self.maxrat = ma.maximum(self.ratios)
self.ratios = ma.filled(self.ratios, 10 * self.maxrat)
def run(self, reference_dataset, target_dataset):
"""Calculate the temporal correlation coefficients and associated
confidence levels between two datasets, using Pearson's correlation.
.. note::
Overrides BinaryMetric.run()
:param reference_dataset: The reference dataset to use in this metric
run
:type reference_dataset: :class:`dataset.Dataset`
:param target_dataset: The target dataset to evaluate against the
reference dataset in this metric run
:type target_dataset: :class:`dataset.Dataset`
:returns: A 2D array of temporal correlation coefficients and a 2D
array of confidence levels associated with the temporal correlation
coefficients
"""
num_times, num_lats, num_lons = reference_dataset.values.shape
coefficients = ma.zeros([num_lats, num_lons])
for i in numpy.arange(num_lats):
for j in numpy.arange(num_lons):
coefficients[i, j] = calc_correlation(target_dataset.values[:, i, j], reference_dataset.values[:, i, j])
return coefficients
def get_data(self, file_middle):
params = self.params
cal_weights = params['cal_weights']
pol_weights = params['pol_weights']
n_time = self.n_time
window = params['window']
subtract_slope = params['subtract_slope']
input_fname = (params['input_root'] + file_middle +
params['input_end'])
# Read in the data.
Reader = core.fitsGBT.Reader(input_fname)
Blocks = Reader.read(params['scans'], params['IFs'],
force_tuple=True)
# On the first pass, set the channel width.
if not hasattr(self, "chan_width"):
self.chan_width = Blocks[0].field['CDELT1']
# Loop over the Blocks to select the channel polarizations and cal
# state that we want to process.
for Data in Blocks:
data = Data.data
data_selected = ma.zeros((Data.dims[0], 1, 1, Data.dims[3]),
dtype=float)
data_selected.mask = ma.getmaskarray(data_selected)
for ii in range(len(pol_weights)) :
for jj in range(len(cal_weights)) :
data_selected[:,0,0,:] += (data[:,ii,jj,:]
* pol_weights[ii]
* cal_weights[jj])
Data.set_data(data_selected)
# Convert the data to the proper format and return it.
return make_masked_time_stream(Blocks, n_time, window=window,
return_means=True,
subtract_slope=subtract_slope)
def calc_area_weighted_spatial_average(dataset, area_weight=False):
'''Calculate area weighted average of the values in OCW dataset
:param dataset: Dataset object
:type dataset: :class:`dataset.Dataset`
:returns: time series for the dataset of shape (nT)
'''
if dataset.lats.ndim ==1:
lons, lats = np.meshgrid(dataset.lons, dataset.lats)
else:
lons = dataset.lons
lats = dataset.lats
weights = np.cos(lats*np.pi/180.)
nt, ny, nx = dataset.values.shape
spatial_average = ma.zeros(nt)
for it in np.arange(nt):
if area_weight:
spatial_average[it] = ma.average(dataset.values[it,:], weights = weights)
else:
spatial_average[it] = ma.average(dataset.values[it,:])
return spatial_average
def composite_diurnal_cycle(var, season, fft=True):
"""
Compute the composite diurnal cycle for var for the given season.
Return mean + amplitudes and times-of-maximum of the first Fourier harmonic component as three transient variables.
"""
season_idx = {
"01": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"02": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"03": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"04": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
"05": [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
"06": [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
"07": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
"08": [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],
"09": [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
"10": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
"11": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
"12": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
"DJF": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
"MAM": [0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
"JJA": [0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0],
"SON": [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0],
"ANN": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
}
site = False
if var.getLatitude() is None and var.getLongitude() is None:
site = True
lat = var.lat
lon = var.lon
# Redefine time to be in the middle of the time interval
var_time = var.getTime()
if var_time is None:
# Climo cannot be run on this variable.
return var
# tbounds = var_time.getBounds()
# var_time[:] = 0.5*(tbounds[:,0]+tbounds[:,1]) #time bounds for h1-h4 are problematic
var_time_absolute = var_time.asComponentTime()
time_freq = int(24 /
(var_time_absolute[1].hour - var_time_absolute[0].hour)
) # This only valid for time interval >= 1hour
start_time = var_time_absolute[0].hour
print("start_time", var_time_absolute[0], var_time_absolute[0].hour)
print("var_time_freq={}".format(time_freq))
# Convert to masked array
v = var.asma()
# Select specified seasons:
if season == "ANNUALCYCLE": # Not supported yet!
cycle = [
"01",
"02",
"03",
"04",
"05",
"06",
"07",
"08",
"09",
"10",
"11",
"12",
]
elif season == "SEASONALCYCLE": # Not supported yet!
cycle = ["DJF", "MAM", "JJA", "SON"]
else:
cycle = [season]
ncycle = len(cycle)
# var_diurnal has shape i.e. (ncycle, ntimesteps, [lat,lon]) for lat lon data
var_diurnal = ma.zeros([ncycle] + [time_freq] + list(numpy.shape(v))[1:])
for n in range(ncycle):
# Get time index for each month/season.
idx = numpy.array(
[
season_idx[cycle[n]][var_time_absolute[i].month - 1]
for i in range(len(var_time_absolute))
],
dtype=numpy.int,
).nonzero()
var_diurnal[n, ] = ma.average( # noqa
numpy.reshape(
v[idx],
(int(v[idx].shape[0] / time_freq), time_freq) +
v[idx].shape[1:],
),
axis=0,
)
# Convert GMT to local time
if site:
nlat = 1
nlon = 1
# lat = [36.6]
# lon = [262.5]
lat = [
lat,
]
lon = [
lon,
]
else:
nlat = var.shape[1]
nlon = var.shape[2]
lat = var.getLatitude()
lon = var.getLongitude()
var_diurnal = numpy.squeeze(var_diurnal)
nt = time_freq
lst = numpy.zeros((nt, nlat, nlon))
for it, itime in enumerate(numpy.arange(0, 24, int(24 / nt))):
for ilon in range(nlon):
lst[it, :,
ilon] = (itime + start_time +
lon[ilon] / 360 * 24) % 24 # convert GMT to LST
# Compute mean, amplitude and max time of the first three Fourier components.
if not fft:
return var_diurnal, lst
else:
cycmean, maxvalue, tmax = fastAllGridFT(var_diurnal, lst)
# Save phase, amplitude, and mean for the first homonic,
amplitude = MV2.zeros((nlat, nlon))
amplitude[:, :] = maxvalue[0]
amplitude.id = "PRECT_diurnal_amplitude"
amplitude.longname = "Amplitude of diurnal cycle of PRECT"
amplitude.units = var.units
amplitude.setAxis(0, lat)
amplitude.setAxis(1, lon)
maxtime = MV2.zeros((nlat, nlon))
maxtime[:, :] = tmax[0]
maxtime.id = "PRECT_diurnal_phase"
maxtime.longname = "Phase of diurnal cycle of PRECT"
maxtime.units = "hour"
maxtime.setAxis(0, lat)
maxtime.setAxis(1, lon)
cmean = MV2.zeros((nlat, nlon))
cmean[:, :] = cycmean
cmean.id = "PRECT_diurnal_cycmean"
cmean.longname = "Mean of diurnal cycle of PRECT"
cmean.units = var.units
cmean.setAxis(0, lat)
cmean.setAxis(1, lon)
return cmean, amplitude, maxtime
def set_event_properties(self, image, time, pixel_x, pixel_y, type_tel,
tel_x, tel_y, array_direction, hillas):
"""The setter class is used to set the event properties within this
class before minimisation can take place. This simply copies a
bunch of useful properties to class members, so that we can
use them later without passing all this information around.
Parameters
----------
image: dict
Amplitude of pixels in camera images
time: dict
Time information per each pixel in camera images
pixel_x: dict
X position of pixels in nominal system
pixel_y: dict
Y position of pixels in nominal system
type_tel: dict
Type of telescope
tel_x: dict
X position of telescope in TiltedGroundFrame
tel_y: dict
Y position of telescope in TiltedGroundFrame
array_direction: SkyCoord[AltAz]
Array pointing direction in the AltAz Frame
hillas: dict
dictionary with telescope IDs as key and
HillasParametersContainer instances as values
Returns
-------
None
"""
# First store these parameters in the class so we can use them
# in minimisation For most values this is simply copying
self.image = image
self.tel_pos_x, self.tel_pos_y, self.ped = \
np.zeros(len(tel_x)), np.zeros(len(tel_x)), np.zeros(len(tel_x))
self.tel_types, self.tel_id = list(), list()
max_pix_x, max_pix_y = 0, 0
px, py, pa, pt = list(), list(), list(), list()
self.hillas_parameters = list()
# So here we must loop over the telescopes
for x, i in zip(tel_x, range(len(tel_x))):
px.append(pixel_x[x].to(u.rad).value)
if len(px[i]) > max_pix_x:
max_pix_x = len(px[i])
py.append(pixel_y[x].to(u.rad).value)
pa.append(image[x])
pt.append(time[x])
self.tel_pos_x[i] = tel_x[x].to(u.m).value
self.tel_pos_y[i] = tel_y[x].to(u.m).value
self.ped[i] = self.ped_table[type_tel[x]]
self.tel_types.append(type_tel[x])
self.tel_id.append(x)
self.hillas_parameters.append(hillas[x])
# Most interesting stuff is now copied to the class, but to remove our requirement
# for loops we must copy the pixel positions to an array with the length of the
# largest image
# First allocate everything
shape = (len(tel_x), max_pix_x)
self.pixel_x, self.pixel_y = ma.zeros(shape), ma.zeros(shape)
self.image, self.time, self.ped = ma.zeros(shape), ma.zeros(shape),\
ma.zeros(shape)
self.tel_types = np.array(self.tel_types)
# Copy everything into our masked arrays
for i in range(len(tel_x)):
array_len = len(px[i])
self.pixel_x[i][:array_len] = px[i]
self.pixel_y[i][:array_len] = py[i]
self.image[i][:array_len] = pa[i]
self.time[i][:array_len] = pt[i]
self.ped[i][:array_len] = self.ped_table[self.tel_types[i]]
# Set the image mask
mask = self.image == 0.0
self.pixel_x[mask], self.pixel_y[mask] = ma.masked, ma.masked
self.image[mask] = ma.masked
self.time[mask] = ma.masked
self.array_direction = array_direction
self.nominal_frame = NominalFrame(origin=self.array_direction)
# Finally run some functions to get ready for the event
self.get_hillas_mean()
self.initialise_templates(type_tel)
def comp_r_old(dert__, fig, root_fcr):
i__ = dert__[0] # i is ig if fig else pixel
'''
sparse aligned i__center and i__rim arrays:
'''
i__center = i__[1:-1:2, 1:-1:2].copy()
i__topleft = i__[:-2:2, :-2:2].copy()
i__top = i__[:-2:2, 1:-1:2].copy()
i__topright = i__[:-2:2, 2::2].copy()
i__right = i__[1:-1:2, 2::2].copy()
i__bottomright = i__[2::2, 2::2].copy()
i__bottom = i__[2::2, 1:-1:2].copy()
i__bottomleft = i__[2::2, :-2:2].copy()
i__left = i__[1:-1:2, :-2:2].copy()
'''
remove mask from kernels with only one masked dert
'''
mask_i = mask_SUM([
i__center.mask, i__topleft.mask, i__top.mask, i__topright.mask,
i__right.mask, i__bottomright.mask, i__bottom.mask, i__bottomleft.mask,
i__left.mask
])
i__center.mask = i__topleft.mask = i__top.mask = i__topright.mask = i__right.mask = i__bottomright.mask = \
i__bottom.mask = i__bottomleft.mask = i__left.mask = mask_i
idy__, idx__ = dert__[[1, 2]]
if root_fcr: # root fork is comp_r, accumulate derivatives:
dy__, dx__, m__ = dert__[[4, 5, 6]]
dy__ = dy__[1:-1:2, 1:-1:2].copy() # sparse to align with i__center
dx__ = dx__[1:-1:2, 1:-1:2].copy()
m__ = m__[1:-1:2, 1:-1:2].copy()
dy__.mask = dx__.mask = m__.mask = mask_i
else: # root fork is comp_g or comp_pixel, initialize sparse derivatives:
dy__ = ma.zeros(
(i__center.shape[0], i__center.shape[1])) # row, column
dx__ = ma.zeros((i__center.shape[0], i__center.shape[1]))
m__ = ma.zeros((i__center.shape[0], i__center.shape[1]))
if not fig: # compare four diametrically opposed pairs of rim pixels:
dt__ = np.stack((i__topleft - i__bottomright, i__top - i__bottom,
i__topright - i__bottomleft, i__right - i__left))
dt__.mask = mask_i # not needed?
for d__, YCOEF, XCOEF in zip(dt__, YCOEFs[:4], XCOEFs[:4]):
dy__ += d__ * YCOEF # decompose differences into dy and dx,
dx__ += d__ * XCOEF # accumulate with prior-rng dy, dx
g__ = np.hypot(dy__, dx__) # gradient
'''
inverse match = SAD, more precise measure of variation than g, direction-invariant
(all diagonal derivatives can be imported from prior 2x2 comp)
'''
m__ += (abs(i__center - i__topleft) + abs(i__center - i__top) +
abs(i__center - i__topright) + abs(i__center - i__right) +
abs(i__center - i__bottomright) + abs(i__center - i__bottom) +
abs(i__center - i__bottomleft) + abs(i__center - i__left))
else: # fig is TRUE, compare angle and then magnitude of 8 center-rim pairs
# replace float with int
i__[ma.where(i__ == 0)] = 1 # if g is int
a__ = [idy__, idx__] / i__ # sin, cos; i = ig
'''
sparse aligned a__center and a__rim arrays:
'''
a__center = a__[:, 1:-1:2, 1:-1:2].copy()
a__topleft = a__[:, :-2:2, :-2:2].copy()
a__top = a__[:, :-2:2, 1:-1:2].copy()
a__topright = a__[:, :-2:2, 2::2].copy()
a__right = a__[:, 1:-1:2, 2::2].copy()
a__bottomright = a__[:, 2::2, 2::2].copy()
a__bottom = a__[:, 2::2, 1:-1:2].copy()
a__bottomleft = a__[:, 2::2, :-2:2].copy()
a__left = a__[:, 1:-1:2, :-2:2].copy()
'''
mask only kernels with more than one masked dert
'''
mask_a = mask_SUM([
a__center.mask, a__topleft.mask, a__top.mask, a__topright.mask,
a__right.mask, a__bottomright.mask, a__bottom.mask,
a__bottomleft.mask, a__left.mask
])
a__center.mask = a__topleft.mask = a__top.mask = a__topright.mask = a__right.mask = a__bottomright.mask = \
a__bottom.mask = a__bottomleft.mask = a__left.mask = mask_a
'''
8-tuple of differences between central dert angle and rim dert angle:
'''
cos_da = ma.stack(
(((a__topleft[1] * a__center[1]) + (a__center[0] * a__topleft[0])),
((a__top[1] * a__center[1]) + (a__center[0] * a__top[0])),
((a__topright[1] * a__center[1]) +
(a__center[0] * a__topright[0])),
((a__right[1] * a__center[1]) + (a__center[0] * a__right[0])),
((a__bottomright[1] * a__center[1]) +
(a__center[0] * a__bottomright[0])),
((a__bottom[1] * a__center[1]) + (a__center[0] * a__bottom[0])),
((a__bottomleft[1] * a__center[1]) +
(a__center[0] * a__bottomleft[0])),
((a__left[1] * a__center[1]) + (a__center[0] * a__left[0]))))
'''
8-tuple of cosine matches per direction:
'''
m__ += (np.minimum(i__center, i__topleft) * cos_da[0] +
np.minimum(i__center, i__top) * cos_da[1] +
np.minimum(i__center, i__topright) * cos_da[2] +
np.minimum(i__center, i__right) * cos_da[3] +
np.minimum(i__center, i__bottomright) * cos_da[4] +
np.minimum(i__center, i__bottom) * cos_da[5] +
np.minimum(i__center, i__bottomleft) * cos_da[6] +
np.minimum(i__center, i__left) * cos_da[7])
'''
8-tuple of cosine differences per direction:
'''
dt__ = np.stack(((i__center - i__topleft * cos_da[0]),
(i__center - i__top * cos_da[1]),
(i__center - i__topright * cos_da[2]),
(i__center - i__right * cos_da[3]),
(i__center - i__bottomright * cos_da[4]),
(i__center - i__bottom * cos_da[5]),
(i__center - i__bottomleft * cos_da[6]),
(i__center - i__left * cos_da[7])))
for d__, YCOEF, XCOEF in zip(dt__, YCOEFs, XCOEFs):
dy__ += d__ * YCOEF # decompose differences into dy and dx,
dx__ += d__ * XCOEF # accumulate with prior-rng dy, dx
'''
accumulate in prior-range dy, dx: 3x3 -> 5x5 -> 9x9
'''
g__ = np.hypot(dy__, dx__)
idy__ = idy__[1:-1:2, 1:-1:2].copy() # i__center.shape, add .copy()?
idx__ = idx__[1:-1:2, 1:-1:2].copy() # i__center.shape
idy__.mask = idx__.mask = i__center.mask # align shifted masks
'''
next comp_r will use full dert
next comp_g will use g__, dy__, dx__
'''
return ma.stack((i__center, idy__, idx__, g__, dy__, dx__, m__))
def comp_a(dert__, rng):
"""
Compute and compare a over predetermined range.
"""
# Unpack dert__:
if len(dert__) in (5, 12): # idert or full dert with m.
i__, g__, m__, dy__, dx__ = dert__[:5]
else: # idert or full dert without m.
i__, g__, dy__, dx__ = dert__[:4]
if len(dert__) > 10: # if ra+:
a__ = dert__[
-7:-5] # Computed angle (use reverse indexing to avoid m check).
day__ = dert__[-4:-2] # Accumulated day__.
dax__ = dert__[-2:] # Accumulated day__.
else: # if fa:
# Compute angles:
a__ = ma.stack((dy__, dx__), axis=0) / g__
a__.mask = g__.mask
# Initialize dax, day:
day__, dax__ = [ma.zeros((2, ) + i__.shape) for _ in range(2)]
# Compute angle differences:
da__ = translated_operation(a__, rng, angle_diff)
comp_field = central_slice(rng)
# Decompose and add to corresponding day and dax:
day__[comp_field] = (da__ * Y_COEFFS[rng]).mean(axis=-1)
dax__[comp_field] = (da__ * X_COEFFS[rng]).mean(axis=-1)
# Apply mask:
msq = np.ones(a__.shape, dtype=int) # Rim mask.
msq[comp_field] = a__.mask[comp_field] + da__.mask.sum(
axis=-1) # Summed d mask.
imsq = msq.nonzero()
day__[imsq] = dax__[imsq] = ma.masked # Apply mask.
# Compute ga:
ga__ = ma.hypot(ma.arctan2(*day__), ma.arctan2(*dax__))[np.newaxis,
...] * SCALER_ga
try: # dert with m is more common:
return ma.concatenate( # Concatenate on the first dimension.
(
ma.stack((i__, g__, m__, dy__, dx__), axis=0),
a__,
ga__,
day__,
dax__,
),
axis=0,
)
except NameError: # m doesn't exist:
return ma.concatenate( # Concatenate on the first dimension.
(
ma.stack((i__, g__, dy__, dx__), axis=0),
a__,
ga__,
day__,
dax__,
),
axis=0,
)
def voronoi_binning(self, target_snr=10.0, write_fits=False, outfile=None, overwrite=False, plot=False,
flag_threshold=0.5, **kwargs):
"""
Applies Voronoi binning to the data cube, using Cappellari's Python implementation.
Parameters
----------
target_snr : float
Desired signal to noise ratio of the binned pixels
write_fits : boolean
Writes a FITS image with the output of the binning.
plot: bool
Plots the binning results.
outfile : string
Name of the output FITS file. If 'None' then the name of
the original FITS file containing the data cube will be used
as a root name, with '.bin' appended to it.
overwrite : boolean
Overwrites files with the same name given in 'outfile'.
flag_threshold : float
Bins with less than this fraction of unflagged pixels will be flagged.
**kwargs: dict
Arguments passed to voronoi_2d_binning.
Returns
-------
Nothing.
Notes
-----
The output file contains two tables which outline the tesselation process. These are
stored in the extensions 'VOR' and 'VORPLUS'.
"""
try:
from vorbin.voronoi_2d_binning import voronoi_2d_binning
except ImportError:
raise ImportError('Could not find the voronoi_2d_binning module. Please add it to your PYTHONPATH.')
if self.noise is None:
raise RuntimeError('This function requires prior execution of the snr_eval method.')
# Initializing the binned arrays as zeros.
assert hasattr(self, 'data'), 'Could not access the data attribute of the Cube object.'
b_data = ma.zeros(self.data.shape)
b_data.mask = self.flags.astype(bool)
assert hasattr(self, 'variance'), 'Could not access the variance attribute of the Cube object.'
b_variance = ma.zeros(self.variance.shape)
b_variance.mask = self.flags.astype(bool)
assert hasattr(self, 'flags'), 'Could not access the variance attribute of the Cube object.'
b_flags = np.zeros_like(self.flags, dtype=int)
valid_spaxels = np.ravel(~np.isnan(self.signal) & ~np.isnan(self.noise) & ~self.spatial_mask)
x = np.ravel(np.indices(np.shape(self.signal))[1])[valid_spaxels]
y = np.ravel(np.indices(np.shape(self.signal))[0])[valid_spaxels]
s, n = deepcopy(self.signal), deepcopy(self.noise)
s[s <= 0] = np.average(self.signal[self.signal > 0])
n[n <= 0] = np.average(self.signal[self.signal > 0]) * .5
signal, noise = np.ravel(s)[valid_spaxels], np.ravel(n)[valid_spaxels]
bin_num, x_node, y_node, x_bar, y_bar, sn, n_pixels, scale = \
voronoi_2d_binning(x, y, signal, noise, target_snr, plot=plot, quiet=0, **kwargs)
v = np.column_stack([y, x, bin_num])
# For every nan in the original cube, fill with nan the binned cubes.
nan_idx = (Ellipsis,
np.ravel(np.indices(np.shape(self.signal))[0])[~valid_spaxels],
np.ravel(np.indices(np.shape(self.signal))[1])[~valid_spaxels])
b_data[nan_idx] = np.nan
b_variance[nan_idx] = np.nan
b_flags[nan_idx] = 1
for i in np.arange(bin_num.max() + 1):
same_bin = v[:, 2] == i
same_bin_coordinates = v[same_bin, :2]
for k in same_bin_coordinates:
binned_idx = (Ellipsis, k[0], k[1])
unbinned_idx = (Ellipsis, same_bin_coordinates[:, 0], same_bin_coordinates[:, 1])
b_data[binned_idx] = ma.mean(self.data[unbinned_idx], axis=1)
b_variance[binned_idx] = ma.mean(self.variance[unbinned_idx], axis=1)
b_flags[binned_idx] = (np.mean(self.flags[unbinned_idx], axis=1) >= flag_threshold).astype(int)
b_data = b_data.data
b_variance = b_variance.data
if write_fits:
h = fits.HDUList()
hdu = fits.PrimaryHDU(header=self.header)
hdu.name = 'PRIMARY'
hdu.header['VORBIN'] = (True, 'Processed by Voronoi binning?')
hdu.header['VORTSNR'] = (target_snr, 'Target SNR for Voronoi binning.')
h.append(hdu)
hdr = self.header_data
# noinspection PyTypeChecker
hdu = fits.ImageHDU(data=b_data, header=hdr)
hdu.name = 'SCI'
h.append(hdu)
# noinspection PyTypeChecker
hdu = fits.ImageHDU(data=b_variance, header=hdr)
hdu.name = 'VAR'
h.append(hdu)
# noinspection PyTypeChecker
hdu = fits.ImageHDU(data=b_flags, header=hdr)
hdu.name = 'FLAGS'
h.append(hdu)
tbhdu = fits.BinTableHDU.from_columns(
[
fits.Column(name='xcoords', format='i8', array=x),
fits.Column(name='ycoords', format='i8', array=y),
fits.Column(name='binNum', format='i8', array=bin_num),
], name='VOR')
tbhdu_plus = fits.BinTableHDU.from_columns(
[
fits.Column(name='ubin', format='i8', array=np.unique(bin_num)),
fits.Column(name='xNode', format='F16.8', array=x_node),
fits.Column(name='yNode', format='F16.8', array=y_node),
fits.Column(name='xBar', format='F16.8', array=x_bar),
fits.Column(name='yBar', format='F16.8', array=y_bar),
fits.Column(name='sn', format='F16.8', array=sn),
fits.Column(name='nPixels', format='i8', array=n_pixels),
], name='VORPLUS')
h.append(tbhdu)
h.append(tbhdu_plus)
if outfile is None:
outfile = self.fitsfile.replace('.fits', '_vor.fits')
h.writeto(outfile, overwrite=overwrite)
self.binned_cube = b_data
def addIntegratedIntensity(full_stack, outDir):
'''
calculate the integrated line flux and the associated noise
Input:
full_stack: astropy table with one row containing the stacks for
each bin.
outDir: directory for diagnostics plots
Output:
stack with added columns for the integrated intensity.
Date Programmer Description of Changes
----------------------------------------------------------------------
5/13/2021 A.A. Kepley Original Code
'''
# get number of stacks
nstack = len(full_stack)
# initialize output arrays
int_intensity_fit = {}
int_intensity_fit_err = {}
int_intensity_fit_uplim = {}
int_intensity_sum = {}
int_intensity_sum_err = {}
int_intensity_sum_uplim = {}
fwhm_fit = np.zeros(nstack) * full_stack['spectral_axis'].unit
fwhm_sum = np.zeros(nstack) * full_stack['spectral_axis'].unit
for line in ['CO', 'HCN', 'HCOp', '13CO', 'C18O']:
int_intensity_fit[line] = ma.zeros(nstack) * full_stack[
'stack_profile_CO'].unit * full_stack['spectral_axis'].unit
int_intensity_fit_err[line] = ma.zeros(nstack) * full_stack[
'stack_profile_CO'].unit * full_stack['spectral_axis'].unit
int_intensity_fit_uplim[line] = np.full(nstack, False)
int_intensity_sum[line] = ma.zeros(nstack) * full_stack[
'stack_profile_CO'].unit * full_stack['spectral_axis'].unit
int_intensity_sum_err[line] = ma.zeros(nstack) * full_stack[
'stack_profile_CO'].unit * full_stack['spectral_axis'].unit
int_intensity_sum_uplim[line] = np.full(nstack, False)
# calculate the integrated intensity from a fit and from a simple sum.
for i in range(nstack):
line = 'CO'
(stack_int, stack_int_err, fwhm,
uplim) = fitIntegratedIntensity(full_stack[i], line, outDir)
int_intensity_fit['CO'][i] = stack_int
int_intensity_fit_err['CO'][i] = stack_int_err
int_intensity_fit_uplim[i] = uplim
fwhm_fit[i] = fwhm
for line in ['HCN', 'HCOp', '13CO', 'C18O']:
if ((full_stack[i]['galaxy'] == 'NGC6946') & ((line == '13CO') |
(line == 'C18O'))):
int_intensity_fit[line][i] = np.nan
int_intensity_fit_err[line][i] = np.nan
int_intensity_fit_uplim[line][i] = True
else:
(stack_int, stack_int_err, fwhm,
uplim) = fitIntegratedIntensity(full_stack[i],
line,
outDir,
fwhm=fwhm)
int_intensity_fit[line][i] = stack_int
int_intensity_fit_err[line][i] = stack_int_err
int_intensity_fit_uplim[line][i] = uplim
# straight sum
(stack_sum, stack_sum_err, fwhm,
uplim) = sumIntegratedIntensity(full_stack[i]['spectral_axis'],
full_stack[i]['stack_profile_CO'])
int_intensity_sum['CO'][i] = stack_sum
int_intensity_sum_err['CO'][i] = stack_sum_err
int_intensity_sum_uplim[i] = uplim
fwhm_sum[i] = fwhm
for line in ['HCN', 'HCOp', '13CO', 'C18O']:
if ((full_stack[i]['galaxy'] == 'NGC6946') & ((line == '13CO') |
(line == 'C18O'))):
int_intensity_sum[line][i] = np.nan
int_intensity_sum_err[line][i] = np.nan
int_intensity_sum_uplim[line][i] = True
else:
(stack_sum, stack_sum_err, fwhm,
uplim) = sumIntegratedIntensity(
full_stack[i]['spectral_axis'],
full_stack[i]['stack_profile_' + line],
fwhm=fwhm)
int_intensity_sum[line][i] = stack_sum
int_intensity_sum_err[line][i] = stack_sum_err
int_intensity_sum_uplim[line][i] = uplim
# add results to data able.
for line in ['CO', 'HCN', 'HCOp', '13CO', 'C18O']:
full_stack.add_column(
Column(int_intensity_fit[line], name='int_intensity_fit_' + line))
full_stack.add_column(
Column(int_intensity_fit_err[line],
name='int_intensity_fit_err_' + line))
full_stack.add_column(
Column(int_intensity_fit_uplim[line],
name='int_intensity_fit_uplim_' + line))
full_stack.add_column(
Column(int_intensity_sum[line], name='int_intensity_sum_' + line))
full_stack.add_column(
Column(int_intensity_sum_err[line],
name='int_intensity_sum_err_' + line))
full_stack.add_column(
Column(int_intensity_sum_uplim[line],
name='int_intensity_sum_uplim_' + line))
full_stack.add_column(Column(fwhm_fit), name='FWHM_fit')
full_stack.add_column(Column(fwhm_sum), name='FWHM_sum')
return full_stack
def get_likelihood(self,
source_x,
source_y,
core_x,
core_y,
energy,
x_max_scale,
goodness_of_fit=False):
"""Get the likelihood that the image predicted at the given test
position matches the camera image.
Parameters
----------
source_x: float
Source position of shower in the nominal system (in deg)
source_y: float
Source position of shower in the nominal system (in deg)
core_x: float
Core position of shower in tilted telescope system (in m)
core_y: float
Core position of shower in tilted telescope system (in m)
energy: float
Shower energy (in TeV)
x_max_scale: float
Scaling factor applied to geometrically calculated Xmax
goodness_of_fit: boolean
Determines whether expected likelihood should be subtracted from result
Returns
-------
float: Likelihood the model represents the camera image at this position
"""
# First we add units back onto everything. Currently not
# handled very well, maybe in future we could just put
# everything in the correct units when loading in the class
# and ignore them from then on
zenith = (np.pi / 2) - self.array_direction.alt.to(u.rad).value
azimuth = self.array_direction.az
# Geometrically calculate the depth of maximum given this test position
x_max = self.get_shower_max(source_x, source_y, core_x, core_y, zenith)
x_max *= x_max_scale
# Calculate expected Xmax given this energy
x_max_exp = guess_shower_depth(energy) # / np.cos(20*u.deg)
# Convert to binning of Xmax
x_max_bin = x_max - x_max_exp
# Check for range
if x_max_bin > 200:
x_max_bin = 200
if x_max_bin < -100:
x_max_bin = -100
# Calculate impact distance for all telescopes
impact = np.sqrt(
np.power(self.tel_pos_x - core_x, 2) +
np.power(self.tel_pos_y - core_y, 2))
# And the expected rotation angle
phi = np.arctan2((self.tel_pos_x - core_x),
(self.tel_pos_y - core_y)) * u.rad
# Rotate and translate all pixels such that they match the
# template orientation
pix_y_rot, pix_x_rot = self.rotate_translate(self.pixel_x,
self.pixel_y, source_x,
source_y, phi)
# In the interpolator class we can gain speed advantages by using masked arrays
# so we need to make sure here everything is masked
prediction = ma.zeros(self.image.shape)
prediction.mask = ma.getmask(self.image)
time_gradients = np.zeros((self.image.shape[0], 2))
# Loop over all telescope types and get prediction
for tel_type in np.unique(self.tel_types).tolist():
type_mask = self.tel_types == tel_type
prediction[type_mask] = \
self.image_prediction(tel_type, energy *
np.ones_like(impact[type_mask]),
impact[type_mask], x_max_bin *
np.ones_like(impact[type_mask]),
pix_x_rot[type_mask] * (180 / math.pi) * -1,
pix_y_rot[type_mask] * (180 / math.pi))
if self.use_time_gradient:
time_gradients[type_mask] = \
self.predict_time(tel_type,
energy * np.ones_like(impact[type_mask]),
impact[type_mask],
x_max_bin * np.ones_like(impact[type_mask]))
if self.use_time_gradient:
time_mask = np.logical_and(np.invert(ma.getmask(self.image)),
self.time > 0)
weight = np.sqrt(self.image) * time_mask
rv = norm()
sx = pix_x_rot * weight
sxx = pix_x_rot * pix_x_rot * weight
sy = self.time * weight
sxy = self.time * pix_x_rot * weight
d = weight.sum(axis=1) * sxx.sum(axis=1) - sx.sum(axis=1) * sx.sum(
axis=1)
time_fit = (weight.sum(axis=1) * sxy.sum(axis=1) -
sx.sum(axis=1) * sy.sum(axis=1)) / d
time_fit /= -1 * (180 / math.pi)
chi2 = -2 * np.log(
rv.pdf((time_fit - time_gradients.T[0]) / time_gradients.T[1]))
# Likelihood function will break if we find a NaN or a 0
prediction[np.isnan(prediction)] = 1e-8
prediction[prediction < 1e-8] = 1e-8
prediction *= self.template_scale
# Get likelihood that the prediction matched the camera image
like = poisson_likelihood_gaussian(self.image, prediction, self.spe,
self.ped)
like[np.isnan(like)] = 1e9
like *= np.invert(ma.getmask(self.image))
like = ma.MaskedArray(like, mask=ma.getmask(self.image))
array_like = like
if goodness_of_fit:
return np.sum(like - mean_poisson_likelihood_gaussian(
prediction, self.spe, self.ped))
prior_pen = 0
# Add prior penalities if we have them
array_like += 1e-8
if "energy" in self.priors:
prior_pen += energy_prior(energy, index=-1)
if "xmax" in self.priors:
prior_pen += xmax_prior(energy, x_max)
array_like += prior_pen / float(len(array_like))
if self.array_return:
array_like = array_like.ravel()
return array_like[np.invert(ma.getmask(array_like))]
final_sum = array_like.sum()
if self.use_time_gradient:
final_sum += chi2.sum() #* np.sum(ma.getmask(self.image))
return final_sum
def __init__(self,
in_file,
vg_files=[1],
data_type=1,
projection='cyl',
color_map='jet',
time_zone=0,
plot_contours=False,
plot_center='t',
plot_meridians=True,
plot_parallels=True,
plot_terminator=True,
resolution='c',
points_of_interest=[],
save_file='',
run_quietly=False,
dpi=150,
parent=None):
self.run_quietly = run_quietly
self.dpi = float(dpi)
plot_parameters = VOAFile((in_file + '.voa'))
plot_parameters.parse_file()
if (plot_parameters.get_projection() != 'cyl'):
print _("Error: Only lat/lon (type 1) input files are supported")
sys.exit(1)
grid = plot_parameters.get_gridsize()
self.image_defs = VOAAreaPlot.IMG_TYPE_DICT[int(data_type)]
# TODO This needs a little more work... what if the pcenter card is not specified
if plot_center == 'p':
plot_centre_location = plot_parameters.get_location(
plot_parameters.P_CENTRE)
else:
plot_centre_location = plot_parameters.get_location(
plot_parameters.TX_SITE)
self.points_of_interest = [plot_centre_location]
if len(points_of_interest) > 0:
self.points_of_interest.extend(points_of_interest)
imageBuf = P.zeros([grid, grid], float)
area_rect = plot_parameters.get_area_rect()
points = P.zeros([grid, grid], float)
lons = P.zeros(grid * grid, float)
lats = P.zeros(grid * grid, float)
lons = P.arange(area_rect.get_sw_lon(),
area_rect.get_ne_lon() + 0.001,
(area_rect.get_ne_lon() - area_rect.get_sw_lon()) /
float(grid - 1))
lats = P.arange(area_rect.get_sw_lat(),
area_rect.get_ne_lat() + 0.001,
(area_rect.get_ne_lat() - area_rect.get_sw_lat()) /
float(grid - 1))
colString = 'P.cm.' + color_map
colMap = eval(colString)
self.subplots = []
matplotlib.rcParams['axes.edgecolor'] = 'gray'
matplotlib.rcParams['axes.facecolor'] = 'white'
matplotlib.rcParams['figure.facecolor'] = 'white'
#matplotlib.rcParams['figure.figsize'] = (6, 10)
# matplotlib.rcParams['figure.subplot.hspace'] = 0.45
# matplotlib.rcParams['figure.subplot.wspace'] = 0.35
# matplotlib.rcParams['figure.subplot.right'] = 0.85
colorbar_fontsize = 8
self.num_rows = 1
self.main_title_fontsize = 18
matplotlib.rcParams['legend.fontsize'] = 12
matplotlib.rcParams['axes.labelsize'] = 10
matplotlib.rcParams['axes.titlesize'] = 10
matplotlib.rcParams['xtick.labelsize'] = 8
matplotlib.rcParams['ytick.labelsize'] = 8
# matplotlib.rcParams['figure.subplot.top'] = 0.8 # single figure plots have a larger title so require more space at the top.
self.fig = Figure(figsize=(9, 5))
# self.main_title_label = self.fig.suptitle(unicode(self.image_defs['title'],'utf-8'), fontsize=self.main_title_fontsize)
ax = self.fig.add_subplot(111, axisbg='white')
self.subplots.append(ax)
ax.label_outer()
#print "opening: ",(in_file+'.vg'+str(vg_files[plot_ctr]))
plot_ctr = 0
# dir_name = os.path.dirname(in_file)
# if not os.path.exists(dir_name):
# os.makedirs(dir_name)
vgFile = open(in_file + '.vg' + str(vg_files[plot_ctr]))
pattern = re.compile(r"[a-z]+")
for line in vgFile:
match = pattern.search(line)
if not match:
value = float(line[int(self.image_defs['first_char']
):int(self.image_defs['last_char'])])
# TODO Does this need to be normalised here if it's also being done in the plot?
value = max(self.image_defs['min'], value)
value = min(self.image_defs['max'], value)
#if value < self.image_defs[2] : value = self.image_defs[2]
#if value > self.image_defs[3] : value = self.image_defs[3]
points[int(line[3:6]) - 1][int(line[0:3]) - 1] = value
vgFile.close()
map = Basemap(\
llcrnrlon=area_rect.get_sw_lon(), llcrnrlat=area_rect.get_sw_lat(),\
urcrnrlon=area_rect.get_ne_lon(), urcrnrlat=area_rect.get_ne_lat(),\
projection=projection,\
lat_0=plot_centre_location.get_latitude(),\
lon_0=plot_centre_location.get_longitude(),\
resolution=resolution,
ax=ax)
map.drawcoastlines(color='black')
map.drawcountries(color='grey')
map.drawmapboundary(color='black', linewidth=1.0)
warped = ma.zeros((grid, grid), float)
warped, warped_lon, warped_lat = map.transform_scalar(
points,
lons,
lats,
grid,
grid,
returnxy=True,
checkbounds=False,
masked=True)
warped = warped.filled(self.image_defs['min'] - 1.0)
colMap.set_under(color='k', alpha=0.0)
im = map.imshow(warped,
cmap=colMap,
extent=(-180, 180, -90, 90),
origin='lower',
norm=P.Normalize(clip=False,
vmin=self.image_defs['min'],
vmax=self.image_defs['max']))
#######################
# Plot greyline
#######################
if plot_terminator:
the_sun = Sun()
the_month = plot_parameters.get_month(vg_files[plot_ctr] - 1)
the_day = plot_parameters.get_day(vg_files[plot_ctr] - 1)
the_hour = plot_parameters.get_utc(vg_files[plot_ctr] - 1)
if (the_day == 0):
the_day = 15
the_year = datetime.date.today().year
num_days_since_2k = the_sun.daysSince2000Jan0(
the_year, the_month, the_day)
res = the_sun.sunRADec(num_days_since_2k)
declination = res[1]
if (declination == 0.0):
declination = -0.001
tau = the_sun.computeGHA(the_day, the_month, the_year, the_hour)
if declination > 0:
terminator_end_lat = area_rect.get_sw_lat()
else:
terminator_end_lat = area_rect.get_ne_lat()
terminator_lat = [terminator_end_lat]
terminator_lon = [area_rect.get_sw_lon()]
for i in range(int(area_rect.get_sw_lon()),
int(area_rect.get_ne_lon()),
1) + [int(area_rect.get_ne_lon())]:
longitude = i + tau
tan_lat = -the_sun.cosd(longitude) / the_sun.tand(declination)
latitude = the_sun.atand(tan_lat)
latitude = max(latitude, area_rect.get_sw_lat())
latitude = min(latitude, area_rect.get_ne_lat())
xpt, ypt = map(i, latitude)
terminator_lon.append(xpt)
terminator_lat.append(ypt)
terminator_lon.append(area_rect.get_ne_lon())
terminator_lat.append(terminator_end_lat)
#This is a little simplistic and doesn't work for ortho plots....
ax.plot(terminator_lon, terminator_lat, color='grey', alpha=0.75)
ax.fill(terminator_lon,
terminator_lat,
facecolor='grey',
alpha=0.5)
tau = -tau
if (tau > 180.0):
tau = tau - 360.0
if (tau < -180.0):
tau = tau + 360.0
#Plot the position of the sun (if it's in the coverage area)
if area_rect.contains(declination, tau):
xpt, ypt = map(tau, declination)
#sbplt_ax.plot([xpt],[ypt],'yh')
ax.plot([xpt], [ypt], 'yh')
##########################
# Points of interest
##########################
for location in self.points_of_interest:
if area_rect.contains(location.get_latitude(),
location.get_longitude()):
xpt, ypt = map(location.get_longitude(),
location.get_latitude())
ax.plot([xpt], [ypt], 'ro')
ax.text(xpt + 100000, ypt + 100000, location.get_name())
if plot_meridians:
if (area_rect.get_lon_delta() <= 90.0):
meridians = P.arange(-180, 190.0, 10.0)
elif (area_rect.get_lon_delta() <= 180.0):
meridians = P.arange(-180.0, 210.0, 30.0)
else:
meridians = P.arange(-180, 240.0, 60.0)
if ((projection == 'ortho') or (projection == 'vandg')):
map.drawmeridians(meridians)
else:
map.drawmeridians(meridians, labels=[1, 1, 0, 1])
if plot_parallels:
if (area_rect.get_lat_delta() <= 90.0):
parallels = P.arange(-90.0, 120.0, 60.0)
else:
parallels = P.arange(-90.0, 120.0, 30.0)
if ((projection == 'ortho') or (projection == 'vandg')):
map.drawparallels(parallels)
else:
map.drawparallels(parallels, labels=[1, 0, 0, 1])
if plot_contours:
map.contour(warped_lon,
warped_lat,
warped,
self.image_defs['y_labels'],
linewidths=1.0,
colors='k',
alpha=0.5)
#add a title
title_str = plot_parameters.get_plot_description_string(
vg_files[plot_ctr] - 1, self.image_defs['plot_type'], time_zone)
title_str = title_str + "\n" + plot_parameters.get_detailed_plot_description_string(
vg_files[plot_ctr] - 1)
self.subplot_title_label = ax.set_title(title_str)
# Add a colorbar on the right hand side, aligned with the
# top of the uppermost plot and the bottom of the lowest
# plot.
pos = [0.91, 0.19, 0.02, 0.62]
self.cb_ax = self.fig.add_axes(pos)
cb = self.fig.colorbar(im,
cax=self.cb_ax,
orientation='vertical',
ticks=self.image_defs['y_labels'],
format=P.FuncFormatter(
eval('self.' +
self.image_defs['formatter'])))
cb.set_label(unicode(self.image_defs['title'], 'utf-8'))
#print self.image_defs['y_labels']
for t in self.cb_ax.get_yticklabels():
t.set_fontsize(colorbar_fontsize)
canvas = FigureCanvasAgg(self.fig)
if save_file:
self.save_plot(canvas, save_file)
class trainCaffeNet:
niter = 6000
test_interval = 100
display = 100
idx = int(ceil(niter * 1.0 / display))
train_loss = zeros(idx)
cpu_u = zeros(idx)
gpu_u = zeros(idx)
gpu_m_u = zeros(idx)
test_loss = zeros(idx)
test_acc = zeros(idx)
def __init__(self):
self.time_s = datetime.datetime.now()
caffe.set_device(0)
caffe.set_mode_gpu()
def train(self):
solver = caffe.get_solver('./model/solver.prototxt')
_train_loss = 0
_test_loss = 0
_accuracy = 0
_cpu = 0
_gpu = 0
_gpu_m = 0
print 'Running solvers for %d iterations...' % self.niter
for it in range(self.niter):
solver.step(1)
_train_loss += solver.net.blobs['loss'].data
_cpu += psutil.cpu_percent()
_gpu += GPU.readl(9)
_gpu_m += GPU.readl(10)
if it % 100 == 0:
print(str(it) + " CPU : " + str(psutil.cpu_percent())) + '%'
print str(it) + " GPU : " + str(GPU.readl(9)) + '%'
print str(it) + " GPU_M : " + str(GPU.readl(10)) + '%'
# print(psutil.virtual_memory()) #
print str(it) + " LOSS : " + str(_train_loss / 100)
self.train_loss[it / self.test_interval] = _train_loss / 100
self.cpu_u[it / self.test_interval] = _cpu / 10000
self.gpu_u[it / self.test_interval] = _gpu / 10000
self.gpu_m_u[it / self.test_interval] = _gpu_m / 10000
_train_loss = 0
_cpu = 0
_gpu = 0
_gpu_m = 0
if it % self.test_interval == 0:
for test_it in range(100):
solver.test_nets[0].forward()
_test_loss += solver.test_nets[0].blobs['loss'].data
_accuracy += solver.test_nets[0].blobs['acc'].data
self.test_loss[it / self.test_interval] = _test_loss / 100
self.test_acc[it / self.test_interval] = _accuracy / 100
print str(it) + " VAL_LOSS : " + str(_test_loss / 100)
print str(it) + " VAL_ACC : " + str(_accuracy / 100)
_test_loss = 0
_accuracy = 0
log.out_log(self.time_s, self.cpu_u, self.gpu_u, self.gpu_m_u,
self.train_loss, self.test_loss, self.test_acc)
# if it % display == 0:
print 'Done.'
self.plot(self.cpu_u, self.gpu_u, self.gpu_m_u, self.train_loss,
self.test_loss, self.test_acc)
print 'Out.'
def plot(self, cpu_u, gpu_u, gpu_m_u, train_loss, test_loss, test_acc):
_, ax1 = plt.subplots(2, sharex=True)
ax2 = ax1[0].twinx()
ax1[0].plot(self.display * arange(len(train_loss)), train_loss, 'g')
ax1[0].plot(self.test_interval * arange(len(test_loss)), test_loss,
'y')
ax2.plot(self.test_interval * arange(len(test_acc)), test_acc, 'r')
ax1[0].set_xlabel('iteration')
ax1[0].set_ylabel('loss')
ax2.set_ylabel('accuracy')
ax1[1].plot(self.display * arange(len(cpu_u)), cpu_u, 'g')
ax1[1].plot(self.display * arange(len(gpu_u)), gpu_u, 'y')
ax1[1].plot(self.display * arange(len(gpu_m_u)), gpu_m_u, 'r')
ax1[1].set_xlabel('iteration')
ax1[1].set_ylabel('precentage')
plt.show()
# -*- coding: utf-8 -*-
from PIL import Image
import PIL
from numpy.ma import array, zeros
import matplotlib.pyplot
from scipy.misc import imresize
import graphcut
# 读取一幅图像，并从图像的两个矩形区域估算出类概率,然后创建一个图:
im = array(Image.open('empire.jpg'))
im = imresize(im, 0.07, interp='bilinear')
size = im.shape[:2]
# 添加两个矩形训练区域
labels = zeros(size)
labels[3:18, 3:18] = -1
labels[-18:-3, -18:-3] = 1
# 创建图
g = graphcut.build_bayes_graph(im, labels, kappa=1)
# 对图进行分割
res = graphcut.cut_graph(g, size)
figure()
graphcut.show_labeling(im, labels)
figure()
imshow(res)
gray()
axis('off')
show()
def wct_significance(a1, a2, significance_level=0.95, mc_count=300,
verbose=False, **kwargs):
"""
Calculates wavelet coherence significance using Monte Carlo
simulations with 95% confidence.
PARAMETERS
a1, a2 (float) :
Lag-1 autoregressive coeficients of both time series.
significance_level (float, optional) :
Significance level to use. Default is 0.95.
count (integer, optional) :
Number of Monte Carlo simulations. Default is 300.
verbose (boolean, optional) :
If set to true, does not print anything on screen.
kwargs (dictionary) :
List of parameters like dt, dj, s0, J=-1 and wavelet.
Please refer to the wavelet.cwt function documentation for
further details.
RETURNS
"""
# Load cache if previously calculated. It is assumed that wavelet analysis
# is performed using the wavelet's default parameters.
aa = round(arctanh(array([a1, a2]) * 4))
aa = abs(aa) + 0.5 * (aa < 0)
cache = 'cache_%0.5f_%0.5f_%0.5f_%0.5f_%d_%s' % (aa[0], aa[1], kwargs['dj'],
kwargs['s0']/kwargs['dt'], kwargs['J'], kwargs['wavelet'].name)
cached = '%s/.klib/wavelet' % (expanduser("~"))
try:
dat = loadtxt('%s/%s.gz' % (cached, cache), unpack=True)
return dat[:, 0], dat[:, 1]
except:
pass
# Some output to the screen
if not verbose:
vS = 'Calculating wavelet coherence significance'
vs = '%s...' % (vS)
stdout.write(vs)
stdout.flush()
# Choose N so that largest scale has at least some part outside the COI
ms = kwargs['s0'] * (2 ** (kwargs['J'] * kwargs['dj'])) / kwargs['dt']
N = ceil(ms * 6)
noise1 = rednoise(N, a1, 1)
nW1 = cwt(noise1, **kwargs)
#
period = ones([1, N]) / nW1['freqs'][:, None]
coi = ones([kwargs['J']+1, 1]) * nW1['coi'][None, :]
outsidecoi = (period <= coi)
scales = ones([1, N]) * nW1['sj'][:, None]
#
sig95 = zeros(kwargs['J'] + 1)
maxscale = find(outsidecoi.any(axis=1))[-1]
sig95[outsidecoi.any(axis=1)] = nan
#
nbins = 1000
wlc = ma.zeros([kwargs['J']+1, nbins])
t1 = time()
for i in range(mc_count):
t2 = time()
# Generates two red-noise signals with lag-1 autoregressive
# coefficients given by a1 and a2
noise1 = rednoise(N, a1, 1)
noise2 = rednoise(N, a2, 1)
# Calculate the cross wavelet transform of both red-noise signals
nW1 = cwt(noise1, **kwargs)
nW2 = cwt(noise2, **kwargs)
nW12 = nW1['W'] * nW2['W'].conj()
# Smooth wavelet wavelet transforms and calculate wavelet coherence
# between both signals.
S1 = kwargs['wavelet'].smooth(abs(nW1['W']) ** 2 / scales,
kwargs['dt'], nW1['dj'], nW1['sj'])
S2 = kwargs['wavelet'].smooth(abs(nW2['W']) ** 2 / scales,
kwargs['dt'], nW2['dj'], nW2['sj'])
S12 = kwargs['wavelet'].smooth(nW12 / scales, kwargs['dt'], nW1['dj'],
nW1['sj'])
R2 = ma.array(abs(S12) ** 2 / (S1 * S2), mask=~outsidecoi)
# Walks through each scale outside the cone of influence and builds a
# coherence coefficient counter.
for s in range(maxscale):
cd = floor(R2[s, :] * nbins)
for j, t in enumerate(cd[~cd.mask]):
wlc[s, t] += 1
# Outputs some text to screen if desired
if not verbose:
stdout.write(len(vs) * '\b')
vs = '%s... %s ' % (vS, profiler(mc_count, i + 1, 0, t1, t2))
stdout.write(vs)
stdout.flush()
# After many, many, many Monte Carlo simulations, determine the
# significance using the coherence coefficient counter percentile.
wlc.mask = (wlc.data == 0.)
R2y = (arange(nbins) + 0.5) / nbins
for s in range(maxscale):
sel = ~wlc[s, :].mask
P = wlc[s, sel].data.cumsum()
P = (P - 0.5) / P[-1]
sig95[s] = interp(significance_level, P, R2y[s:, sel])
# Save the results on cache to avoid to many computations in the future
try:
makedirs(cached)
except:
pass
savetxt('%s/%s.gz' % (cached, cache), [sig95, nW1['sj']])
# And returns the results
return sig95, nW1['sj']
mod_yc_data_nodp = ycdata[yc_cols]
mod_yc_data = mod_yc_data_nodp.dropna(axis=0)
#limit to 1983 and on to not include Volker period
dates = pa.date_range("1/1/1983", "10/1/2009", freq="MS").to_pydatetime()
#dates = pa.date_range("6/1/1979", "12/1/2012", freq="MS").to_pydatetime()
# limit dates of yields and take logs
mod_yc_data = np.log(mod_yc_data.ix[dates])
# Maturities
mats = [12, 24, 36, 60, 84, 120]
# construct model matrices up
datatype = np.complex_
lam_0_e = ma.zeros([latent, 1], dtype=datatype)
lam_1_e = ma.zeros([latent, latent], dtype=datatype)
delta_0_e = ma.zeros([1, 1], dtype=datatype)
delta_1_e = ma.zeros([latent, 1], dtype=datatype)
mu_e = ma.zeros([latent, 1], dtype=datatype)
phi_e = ma.zeros([latent, latent], dtype=datatype)
sigma_e = ma.zeros([latent, latent], dtype=datatype)
#mask values to be estimated
lam_0_e[:, 0] = ma.masked
lam_1_e[:, :] = ma.masked
delta_0_e[:, :] = ma.masked
delta_1_e[:, :] = ma.masked
delta_1_e[:, :] = ma.nomask
delta_1_e[:, :] = 1
def test_testAverage2(self):
# More tests of average.
w1 = [0, 1, 1, 1, 1, 0]
w2 = [[0, 1, 1, 1, 1, 0], [1, 0, 0, 0, 0, 1]]
x = arange(6)
assert_(allclose(average(x, axis=0), 2.5))
assert_(allclose(average(x, axis=0, weights=w1), 2.5))
y = array([arange(6), 2.0 * arange(6)])
assert_(
allclose(average(y, None),
np.add.reduce(np.arange(6)) * 3. / 12.))
assert_(allclose(average(y, axis=0), np.arange(6) * 3. / 2.))
assert_(
allclose(average(y, axis=1),
[average(x, axis=0),
average(x, axis=0) * 2.0]))
assert_(allclose(average(y, None, weights=w2), 20. / 6.))
assert_(
allclose(average(y, axis=0, weights=w2),
[0., 1., 2., 3., 4., 10.]))
assert_(
allclose(average(y, axis=1),
[average(x, axis=0),
average(x, axis=0) * 2.0]))
m1 = zeros(6)
m2 = [0, 0, 1, 1, 0, 0]
m3 = [[0, 0, 1, 1, 0, 0], [0, 1, 1, 1, 1, 0]]
m4 = ones(6)
m5 = [0, 1, 1, 1, 1, 1]
assert_(allclose(average(masked_array(x, m1), axis=0), 2.5))
assert_(allclose(average(masked_array(x, m2), axis=0), 2.5))
assert_(average(masked_array(x, m4), axis=0) is masked)
assert_equal(average(masked_array(x, m5), axis=0), 0.0)
assert_equal(count(average(masked_array(x, m4), axis=0)), 0)
z = masked_array(y, m3)
assert_(allclose(average(z, None), 20. / 6.))
assert_(allclose(average(z, axis=0), [0., 1., 99., 99., 4.0, 7.5]))
assert_(allclose(average(z, axis=1), [2.5, 5.0]))
assert_(
allclose(average(z, axis=0, weights=w2),
[0., 1., 99., 99., 4.0, 10.0]))
a = arange(6)
b = arange(6) * 3
r1, w1 = average([[a, b], [b, a]], axis=1, returned=True)
assert_equal(shape(r1), shape(w1))
assert_equal(r1.shape, w1.shape)
r2, w2 = average(ones((2, 2, 3)),
axis=0,
weights=[3, 1],
returned=True)
assert_equal(shape(w2), shape(r2))
r2, w2 = average(ones((2, 2, 3)), returned=True)
assert_equal(shape(w2), shape(r2))
r2, w2 = average(ones((2, 2, 3)),
weights=ones((2, 2, 3)),
returned=True)
assert_(shape(w2) == shape(r2))
a2d = array([[1, 2], [0, 4]], float)
a2dm = masked_array(a2d, [[0, 0], [1, 0]])
a2da = average(a2d, axis=0)
assert_(eq(a2da, [0.5, 3.0]))
a2dma = average(a2dm, axis=0)
assert_(eq(a2dma, [1.0, 3.0]))
a2dma = average(a2dm, axis=None)
assert_(eq(a2dma, 7. / 3.))
a2dma = average(a2dm, axis=1)
assert_(eq(a2dma, [1.5, 4.0]))
def test_testOddFeatures(self):
# Test of other odd features
x = arange(20)
x = x.reshape(4, 5)
x.flat[5] = 12
assert_(x[1, 0] == 12)
z = x + 10j * x
assert_(eq(z.real, x))
assert_(eq(z.imag, 10 * x))
assert_(eq((z * conjugate(z)).real, 101 * x * x))
z.imag[...] = 0.0
x = arange(10)
x[3] = masked
assert_(str(x[3]) == str(masked))
c = x >= 8
assert_(count(where(c, masked, masked)) == 0)
assert_(shape(where(c, masked, masked)) == c.shape)
z = where(c, x, masked)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is masked)
assert_(z[7] is masked)
assert_(z[8] is not masked)
assert_(z[9] is not masked)
assert_(eq(x, z))
z = where(c, masked, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
z = masked_where(c, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
assert_(eq(x, z))
x = array([1., 2., 3., 4., 5.])
c = array([1, 1, 1, 0, 0])
x[2] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
c[0] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
assert_(
eq(masked_where(greater_equal(x, 2), x),
masked_greater_equal(x, 2)))
assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
assert_(
eq(
masked_inside(array(list(range(5)), mask=[1, 0, 0, 0, 0]), 1,
3).mask, [1, 1, 1, 1, 0]))
assert_(
eq(
masked_outside(array(list(range(5)), mask=[0, 1, 0, 0, 0]), 1,
3).mask, [1, 1, 0, 0, 1]))
assert_(
eq(
masked_equal(array(list(range(5)), mask=[1, 0, 0, 0, 0]),
2).mask, [1, 0, 1, 0, 0]))
assert_(
eq(
masked_not_equal(array([2, 2, 1, 2, 1], mask=[1, 0, 0, 0, 0]),
2).mask, [1, 0, 1, 0, 1]))
assert_(
eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
[99, 99, 3, 4, 5]))
atest = ones((10, 10, 10), dtype=np.float32)
btest = zeros(atest.shape, MaskType)
ctest = masked_where(btest, atest)
assert_(eq(atest, ctest))
z = choose(c, (-x, x))
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
x = arange(6)
x[5] = masked
y = arange(6) * 10
y[2] = masked
c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
cm = c.filled(1)
z = where(c, x, y)
zm = where(cm, x, y)
assert_(eq(z, zm))
assert_(getmask(zm) is nomask)
assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
z = where(c, masked, 1)
assert_(eq(z, [99, 99, 99, 1, 1, 1]))
z = where(c, 1, masked)
assert_(eq(z, [99, 1, 1, 99, 99, 99]))
def mimic(doPrint=False,
doPlot=False,
RossHS=False,
RecipFlag=False,
thisSza=None):
'''
A test method to reproduce the results in Wanner et al. 1995.
There are no parameters and a single option:
doPrint=True : print results to stdout (default doPrint=False)
The method returns:
VZA,SZA,RAA,RossThick,RossThin,LiSparse,LiDense,Roujean,LiTransit
where all are numy arrays of dimensions 3 x nSamples
so:
VZA[0,:],RossThick[0,:] are the results for sza = 0.0
VZA[1,:],RossThick[1,:] are the results for sza = 30.0
VZA[2,:],RossThick[2,:] are the results for sza = 60.0
'''
# set up the angles
r = 89 # do results for +/- r degrees)
if thisSza == None:
SZAS = ma.array([0.0, -30.0, -60.0]) # sza
else:
SZAS = ma.array(thisSza)
vza = numpy.array(range(2 * r + 1)) * 1.0 - r
# set up storage info
RossThick = ma.zeros([3, len(vza)])
RossThin = ma.zeros([3, len(vza)])
LiSparse = ma.zeros([3, len(vza)])
LiDense = ma.zeros([3, len(vza)])
Roujean = ma.zeros([3, len(vza)])
LiTransit = ma.zeros([3, len(vza)])
SZA = ma.zeros([3, len(vza)])
VZA = ma.zeros([3, len(vza)])
RAA = ma.zeros([3, len(vza)])
# fill the angle info
RossHS = RossHS
for i in range(len(SZAS)):
SZA[i, :] = SZAS[i]
VZA[i, :] = vza[:]
RAA[i, :] = 0.0
# do the kernels
kk = Kernels(VZA[i, :],
SZA[i, :],
RAA[i, :],
RossHS=RossHS,
MODISSPARSE=True,
RecipFlag=RecipFlag,
normalise=1,
doIntegrals=False,
LiType='Dense',
RossType='Thick')
RossThick[i, :] = kk.Ross[:]
LiDense[i, :] = kk.Li[:]
if doPrint == True:
kk.printKernels(file='RossThickLiDense.' + str(SZAS[i]) + '.dat')
kk.printer('')
kk = Kernels(VZA[i, :],
SZA[i, :],
RAA[i, :],
RossHS=RossHS,
MODISSPARSE=True,
RecipFlag=RecipFlag,
normalise=1,
doIntegrals=False,
LiType='Sparse',
RossType='Thin')
RossThin[i, :] = kk.Ross[:]
LiSparse[i, :] = kk.Li[:]
if doPrint == True:
kk.printKernels(file='RossThinLiSparse.' + str(SZAS[i]) + '.dat')
kk.printer('')
kk = Kernels(VZA[i, :],
SZA[i, :],
RAA[i, :],
RossHS=RossHS,
MODISSPARSE=True,
RecipFlag=RecipFlag,
normalise=1,
doIntegrals=False,
LiType='Roujean',
RossType='Thin')
Roujean[i, :] = kk.Li[:]
if doPrint == True:
kk.printKernels(file='RossThinRoujean.' + str(SZAS[i]) + '.dat')
kk.printer('')
kk = Kernels(VZA[i, :],
SZA[i, :],
RAA[i, :],
RossHS=RossHS,
MODISSPARSE=True,
RecipFlag=RecipFlag,
normalise=1,
doIntegrals=False,
LiType='Transit',
RossType='Thin')
LiTransit[i, :] = kk.Li[:]
if doPrint == True:
kk.printKernels(file='RossThinLiTransit.' + str(SZAS[i]) + '.dat')
kk.printer('')
if (doPlot == True):
import pylab
x = [-90.0, 90.0]
y = [0.0, 0.0]
for i in range(len(SZAS)):
sza = SZAS[i]
pylab.clf()
pylab.xlabel('View Zenith Angle')
pylab.ylabel('Kernel Value')
pylab.title('Solar Zenith Angle ' + str(sza) + ' Degrees')
pylab.plot(x, y)
pylab.plot(kk.vzaDegrees, RossThick[i, :], label='RThick')
pylab.plot(kk.vzaDegrees, RossThin[i, :], label='RThin')
pylab.plot(kk.vzaDegrees, LiSparse[i, :], label='LiSp')
pylab.plot(kk.vzaDegrees, LiDense[i, :], label='LiDen')
pylab.plot(kk.vzaDegrees, Roujean[i, :], label='Roujean')
pylab.plot(kk.vzaDegrees, LiTransit[i, :], label='LiTrans')
pylab.axis([-90.0, 90.0, -3.0, 3.0])
pylab.legend(loc=0)
pylab.show()
return VZA, SZA, RAA, RossThick, RossThin, LiSparse, LiDense, Roujean, LiTransit
dataset = local.load_file(
'AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc', 'nonabsorbing_ave')
''' Subset the data for East Asia'''
Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150)
dataset = dsp.subset(dataset, Bounds)
'''The original dataset includes nonabsorbing AOD values between March 2000 and February 2015.
dsp.temporal_subset will extract data in September-October-November.'''
dataset_SON = dsp.temporal_subset(dataset,
month_start=9,
month_end=11,
average_each_year=True)
ny, nx = dataset_SON.values.shape[1:]
# multi-year mean aod
clim_aod = ma.zeros([3, ny, nx])
clim_aod[0, :] = ma.mean(dataset_SON.values, axis=0) # 16-year mean
clim_aod[1, :] = ma.mean(dataset_SON.values[-5:, :],
axis=0) # the last 5-year mean
clim_aod[2, :] = dataset_SON.values[-1, :] # the last year's value
# plot clim_aod (3 subplots)
plotter.draw_contour_map(
clim_aod,
dataset_SON.lats,
dataset_SON.lons,
fname='nonabsorbing_AOD_clim_East_Asia_Sep-Nov',
gridshape=[1, 3],
subtitles=['2000-2015: 16 years', '2011-2015: 5 years', '2015: 1 year'],
clevs=np.arange(21) * 0.02)
def rebin(Data, n_bins_combined):
"""The function that acctually does the rebinning on a Data Block."""
nt = Data.data.shape[0]
new_nt = nt // n_bins_combined
new_shape = (new_nt, ) + Data.data.shape[1:]
unmask = sp.logical_not(ma.getmaskarray(Data.data))
data = Data.data.filled(0)
# Allowcate memeory for the rebinned data.
new_data = ma.zeros(new_shape, dtype=data.dtype)
counts = sp.zeros(new_shape, dtype=int)
# Add up the bins to be combined.
for ii in range(n_bins_combined):
new_data += data[ii:new_nt * n_bins_combined:n_bins_combined, ...]
counts += unmask[ii:new_nt * n_bins_combined:n_bins_combined, ...]
new_data[counts == 0] = ma.masked
counts[counts == 0] = 1
new_data /= counts
Data.set_data(new_data)
# Now deal with all the other records that aren't the main data.
for field_name in Data.field.iterkeys():
# DATE-OBS is a string field so we have to write special code for it.
if field_name == "DATE-OBS":
time_field = Data.field[field_name]
new_field = sp.empty(new_nt, dtype=Data.field[field_name].dtype)
# Convert to float, average, then convert back to a string.
time_float = utils.time2float(time_field)
for ii in range(new_nt):
tmp_time = sp.mean(time_float[n_bins_combined *
ii:n_bins_combined * (ii + 1)])
new_field[ii] = utils.float2time(tmp_time)
Data.set_field(field_name,
new_field,
axis_names=Data.field_axes[field_name],
format=Data.field_formats[field_name])
continue
# Only change fields that have a 'time' axis.
try:
time_axis = list(Data.field_axes[field_name]).index('time')
except ValueError:
continue
# For now, the time axis has to be the first axis.
if time_axis != 0:
msg = "Expected time to be the first axis for all fields."
raise NotImplementedError(msg)
field_data = Data.field[field_name]
if not field_data.dtype.name == "float64":
msg = "Field data type is not float. Handle explicitly."
raise NotImplementedError(msg)
new_field = sp.empty(field_data.shape[:time_axis] + (new_nt, ) +
field_data.shape[time_axis + 1:],
dtype=field_data.dtype)
for ii in range(new_nt):
tmp_data = sp.sum(
field_data[n_bins_combined * ii:n_bins_combined * (ii + 1),
...], 0)
tmp_data /= n_bins_combined
new_field[ii, ...] = tmp_data
Data.set_field(field_name,
new_field,
axis_names=Data.field_axes[field_name],
format=Data.field_formats[field_name])
def comp_g(
dert__
): # cross-comp of g in 2x2 kernels, between derts in ma.stack dert__
# initialize return variable
new_dert__ = ma.zeros(
(dert__.shape[0], dert__.shape[1] - 1, dert__.shape[2] - 1))
new_dert__.mask = True # initialize mask
ig__, idy__, idx__, gg__, dgy__, dgx__, mg__ = new_dert__ # assign 'views'. Use [:] to update views
# Unpack relevant params
g__, dy__, dx__ = dert__[[3, 4, 5]] # g, dy, dx -> local i, idy, idx
g__.data[np.where(
g__.data == 0
)] = 1 # replace 0 values with 1 to avoid error, not needed in high-g blobs?
'''
for all operations below: only mask kernels with more than one masked dert
'''
majority_mask = (
g__[:-1, :-1].mask.astype(int) + g__[:-1, 1:].mask.astype(int) +
g__[1:, 1:].mask.astype(int) + g__[1:, :-1].mask.astype(int)) > 1
g__.mask = dy__.mask = dx__.mask = majority_mask
g0__, dy0__, dx0__ = g__[:-1, :
-1].data, dy__[:-1, :
-1].data, dx__[:-1, :
-1].data # top left
g1__, dy1__, dx1__ = g__[:-1,
1:].data, dy__[:-1,
1:].data, dx__[:-1,
1:].data # top right
g2__, dy2__, dx2__ = g__[1:, 1:].data, dy__[1:, 1:].data, dx__[
1:, 1:].data # bottom right
g3__, dy3__, dx3__ = g__[1:, :-1].data, dy__[1:, :-1].data, dx__[
1:, :-1].data # bottom left
sin0__ = dy0__ / g0__
cos0__ = dx0__ / g0__
sin1__ = dy1__ / g1__
cos1__ = dx1__ / g1__
sin2__ = dy2__ / g2__
cos2__ = dx2__ / g2__
sin3__ = dy3__ / g3__
cos3__ = dx3__ / g3__
'''
cosine of difference between diagonally opposite angles, in vector representation
print(cos_da1__.shape, type(cos_da1__))
'''
cos_da0__ = (cos2__ * cos0__) + (sin2__ * sin0__
) # top left to bottom right
cos_da1__ = (cos3__ * cos1__) + (sin3__ * sin1__
) # top right to bottom left
dgy__[:] = ((g3__ + g2__) - (g0__ * cos_da0__ + g1__ * cos_da0__))
# y-decomposed cosine difference between gs
dgx__[:] = ((g1__ + g2__) - (g0__ * cos_da0__ + g3__ * cos_da1__))
# x-decomposed cosine difference between gs
gg__[:] = ma.hypot(dgy__, dgx__) # gradient of gradient
mg0__ = ma.minimum(g0__, g2__) * (cos_da1__ + 1) # +1 to make all positive
mg1__ = ma.minimum(g1__, g3__) * (cos_da1__ + 1)
mg__[:] = mg0__ + mg1__ # match of gradient
ig__[:] = g__[:-1, :
-1] # remove last row and column to align with derived params
idy__[:] = dy__[:-1, :-1]
idx__[:] = dx__[:-1, :-1] # -> idy, idx to compute cos for comp rg
# unnecessary?:
gg__.mask = mg__.mask = dgy__.mask = dgx__.mask = majority_mask
'''
next comp_rg will use g, dy, dx
next comp_gg will use gg, dgy, dgx
'''
return new_dert__ # new_dert__ has been updated along with 'view' arrays: ig__, idy__, idx__, gg__, dgy__, dgx__, mg__
upper = 2100
# CMIP6 models
cmip6_models = [
'ACCESS-ESM1-5', 'BCC-CSM2-MR', 'CanESM5', 'CNRM-ESM2-1', 'IPSL-CM6A-LR',
'MIROC-ES2L', 'UKESM1-0-LL'
]
n_models = len(cmip6_models)
model_shapes = ['o', '^', '+', 's', '*', 'd', 'x']
# SSP scenarios
ssp_options = ['ssp126', 'ssp245', 'ssp585']
ssp_options_length = len(ssp_options)
# defining empty numpy array to save values
x_array = ma.zeros((len(ssp_options), len(cmip6_models)))
y_array = ma.zeros((len(ssp_options), len(cmip6_models)))
#%%
# Loop through each ssp run being considered
for ssp_option in range(0, ssp_options_length):
ssp = ssp_options[ssp_option] # selecting the ssp scenarios
# for loop for each CMIP5 model
for model_i in range(0, n_models):
model = cmip6_models[model_i] # seleting the models
print(ssp, model)
#%% historical soil turnover time
def estimate_shift2D(self,
reference='current',
correlation_threshold=None,
chunk_size=30,
roi=None,
normalize_corr=False,
sobel=True,
medfilter=True,
hanning=True,
plot=False,
dtype='float',
show_progressbar=None,
sub_pixel_factor=1):
"""Estimate the shifts in an image using phase correlation.
This method can only estimate the shift by comparing
bi-dimensional features that should not change position
between frames. To decrease the memory usage, the time of
computation and the accuracy of the results it is convenient
to select a region of interest by setting the ``roi`` argument.
Parameters
----------
reference : {'current', 'cascade' ,'stat'}
If 'current' (default) the image at the current
coordinates is taken as reference. If 'cascade' each image
is aligned with the previous one. If 'stat' the translation
of every image with all the rest is estimated and by
performing statistical analysis on the result the
translation is estimated.
correlation_threshold : {None, 'auto', float}
This parameter is only relevant when reference='stat'.
If float, the shift estimations with a maximum correlation
value lower than the given value are not used to compute
the estimated shifts. If 'auto' the threshold is calculated
automatically as the minimum maximum correlation value
of the automatically selected reference image.
chunk_size : {None, int}
If int and reference='stat' the number of images used
as reference are limited to the given value.
roi : tuple of ints or floats (left, right, top, bottom)
Define the region of interest. If int(float) the position
is given axis index(value). Note that ROIs can be used
in place of a tuple.
normalize_corr : bool, default False
If True, use phase correlation to align the images, otherwise
use cross correlation.
sobel : bool, default True
Apply a Sobel filter for edge enhancement
medfilter : bool, default True
Apply a median filter for noise reduction
hanning : bool, default True
Apply a 2D hanning filter
plot : bool or 'reuse'
If True plots the images after applying the filters and
the phase correlation. If 'reuse', it will also plot the images,
but it will only use one figure, and continuously update the images
in that figure as it progresses through the stack.
dtype : str or dtype
Typecode or data-type in which the calculations must be
performed.
%s
sub_pixel_factor : float
Estimate shifts with a sub-pixel accuracy of 1/sub_pixel_factor
parts of a pixel. Default is 1, i.e. no sub-pixel accuracy.
Returns
-------
shifts : list of array
List of estimated shifts
Notes
-----
The statistical analysis approach to the translation estimation
when using ``reference='stat'`` roughly follows [Schaffer2004]_.
If you use it please cite their article.
References
----------
.. [Schaffer2004] Schaffer, Bernhard, Werner Grogger, and Gerald Kothleitner.
“Automated Spatial Drift Correction for EFTEM Image Series.”
Ultramicroscopy 102, no. 1 (December 2004): 27–36.
See Also
--------
* :py:meth:`~._signals.signal2d.Signal2D.align2D`
"""
if show_progressbar is None:
show_progressbar = preferences.General.show_progressbar
self._check_signal_dimension_equals_two()
if roi is not None:
# Get the indices of the roi
yaxis = self.axes_manager.signal_axes[1]
xaxis = self.axes_manager.signal_axes[0]
roi = tuple([xaxis._get_index(i) for i in roi[2:]] +
[yaxis._get_index(i) for i in roi[:2]])
ref = None if reference == 'cascade' else \
self.__call__().copy()
shifts = []
nrows = None
images_number = self.axes_manager._max_index + 1
if plot == 'reuse':
# Reuse figure for plots
plot = plt.figure()
if reference == 'stat':
nrows = images_number if chunk_size is None else \
min(images_number, chunk_size)
pcarray = ma.zeros((nrows, self.axes_manager._max_index + 1,
),
dtype=np.dtype([('max_value', np.float),
('shift', np.int32,
(2,))]))
nshift, max_value = estimate_image_shift(
self(),
self(),
roi=roi,
sobel=sobel,
medfilter=medfilter,
hanning=hanning,
normalize_corr=normalize_corr,
plot=plot,
dtype=dtype,
sub_pixel_factor=sub_pixel_factor)
np.fill_diagonal(pcarray['max_value'], max_value)
pbar_max = nrows * images_number
else:
pbar_max = images_number
# Main iteration loop. Fills the rows of pcarray when reference
# is stat
with progressbar(total=pbar_max,
disable=not show_progressbar,
leave=True) as pbar:
for i1, im in enumerate(self._iterate_signal()):
if reference in ['current', 'cascade']:
if ref is None:
ref = im.copy()
shift = np.array([0, 0])
nshift, max_val = estimate_image_shift(
ref, im, roi=roi, sobel=sobel, medfilter=medfilter,
hanning=hanning, plot=plot,
normalize_corr=normalize_corr, dtype=dtype,
sub_pixel_factor=sub_pixel_factor)
if reference == 'cascade':
shift += nshift
ref = im.copy()
else:
shift = nshift
shifts.append(shift.copy())
pbar.update(1)
elif reference == 'stat':
if i1 == nrows:
break
# Iterate to fill the columns of pcarray
for i2, im2 in enumerate(
self._iterate_signal()):
if i2 > i1:
nshift, max_value = estimate_image_shift(
im,
im2,
roi=roi,
sobel=sobel,
medfilter=medfilter,
hanning=hanning,
normalize_corr=normalize_corr,
plot=plot,
dtype=dtype,
sub_pixel_factor=sub_pixel_factor)
pcarray[i1, i2] = max_value, nshift
del im2
pbar.update(1)
del im
if reference == 'stat':
# Select the reference image as the one that has the
# higher max_value in the row
sqpcarr = pcarray[:, :nrows]
sqpcarr['max_value'][:] = symmetrize(sqpcarr['max_value'])
sqpcarr['shift'][:] = antisymmetrize(sqpcarr['shift'])
ref_index = np.argmax(pcarray['max_value'].min(1))
self.ref_index = ref_index
shifts = (pcarray['shift'] +
pcarray['shift'][ref_index, :nrows][:, np.newaxis])
if correlation_threshold is not None:
if correlation_threshold == 'auto':
correlation_threshold = \
(pcarray['max_value'].min(0)).max()
_logger.info("Correlation threshold = %1.2f",
correlation_threshold)
shifts[pcarray['max_value'] <
correlation_threshold] = ma.masked
shifts.mask[ref_index, :] = False
shifts = shifts.mean(0)
else:
shifts = np.array(shifts)
del ref
return shifts
def project(cube, target_proj, nx=None, ny=None):
"""
Nearest neighbour regrid to a specified target projection.
Return a new cube that is the result of projecting a cube with 1 or 2
dimensional latitude-longitude coordinates from its coordinate system into
a specified projection e.g. Robinson or Polar Stereographic.
This function is intended to be used in cases where the cube's coordinates
prevent one from directly visualising the data, e.g. when the longitude
and latitude are two dimensional and do not make up a regular grid.
Args:
* cube
An instance of :class:`iris.cube.Cube`.
* target_proj
An instance of the Cartopy Projection class, or an instance of
:class:`iris.coord_systems.CoordSystem` from which a projection
will be obtained.
Kwargs:
* nx
Desired number of sample points in the x direction for a domain
covering the globe.
* ny
Desired number of sample points in the y direction for a domain
covering the globe.
Returns:
An instance of :class:`iris.cube.Cube` and a list describing the
extent of the projection.
.. note::
This function assumes global data and will if necessary extrapolate
beyond the geographical extent of the source cube using a nearest
neighbour approach. nx and ny then include those points which are
outside of the target projection.
.. note::
Masked arrays are handled by passing their masked status to the
resulting nearest neighbour values. If masked, the value in the
resulting cube is set to 0.
.. warning::
This function uses a nearest neighbour approach rather than any form
of linear/non-linear interpolation to determine the data value of each
cell in the resulting cube. Consequently it may have an adverse effect
on the statistics of the data e.g. the mean and standard deviation
will not be preserved.
"""
try:
lon_coord, lat_coord = _get_lon_lat_coords(cube)
except IndexError:
raise ValueError('Cannot get latitude/longitude '
'coordinates from cube {!r}.'.format(cube.name()))
if lat_coord.coord_system != lon_coord.coord_system:
raise ValueError('latitude and longitude coords appear to have '
'different coordinates systems.')
if lon_coord.units != 'degrees':
lon_coord = lon_coord.copy()
lon_coord.convert_units('degrees')
if lat_coord.units != 'degrees':
lat_coord = lat_coord.copy()
lat_coord.convert_units('degrees')
# Determine source coordinate system
if lat_coord.coord_system is None:
# Assume WGS84 latlon if unspecified
warnings.warn('Coordinate system of latitude and longitude '
'coordinates is not specified. Assuming WGS84 Geodetic.')
orig_cs = iris.coord_systems.GeogCS(semi_major_axis=6378137.0,
inverse_flattening=298.257223563)
else:
orig_cs = lat_coord.coord_system
# Convert to cartopy crs
source_cs = orig_cs.as_cartopy_crs()
# Obtain coordinate arrays (ignoring bounds) and convert to 2d
# if not already.
source_x = lon_coord.points
source_y = lat_coord.points
if source_x.ndim != 2 or source_y.ndim != 2:
source_x, source_y = _meshgrid(source_x, source_y)
# Calculate target grid
target_cs = None
if isinstance(target_proj, iris.coord_systems.CoordSystem):
target_cs = target_proj
target_proj = target_proj.as_cartopy_projection()
# Resolution of new grid
if nx is None:
nx = source_x.shape[1]
if ny is None:
ny = source_x.shape[0]
target_x, target_y, extent = cartopy.img_transform.mesh_projection(
target_proj, nx, ny)
# Determine dimension mappings - expect either 1d or 2d
if lat_coord.ndim != lon_coord.ndim:
raise ValueError("The latitude and longitude coordinates have "
"different dimensionality.")
latlon_ndim = lat_coord.ndim
lon_dims = cube.coord_dims(lon_coord)
lat_dims = cube.coord_dims(lat_coord)
if latlon_ndim == 1:
xdim = lon_dims[0]
ydim = lat_dims[0]
elif latlon_ndim == 2:
if lon_dims != lat_dims:
raise ValueError("The 2d latitude and longitude coordinates "
"correspond to different dimensions.")
# If coords are 2d assume that grid is ordered such that x corresponds
# to the last dimension (shortest stride).
xdim = lon_dims[1]
ydim = lon_dims[0]
else:
raise ValueError('Expected the latitude and longitude coordinates '
'to have 1 or 2 dimensions, got {} and '
'{}.'.format(lat_coord.ndim, lon_coord.ndim))
# Create array to store regridded data
new_shape = list(cube.shape)
new_shape[xdim] = nx
new_shape[ydim] = ny
new_data = ma.zeros(new_shape, cube.data.dtype)
# Create iterators to step through cube data in lat long slices
new_shape[xdim] = 1
new_shape[ydim] = 1
index_it = np.ndindex(*new_shape)
if lat_coord.ndim == 1 and lon_coord.ndim == 1:
slice_it = cube.slices([lat_coord, lon_coord])
elif lat_coord.ndim == 2 and lon_coord.ndim == 2:
slice_it = cube.slices(lat_coord)
else:
raise ValueError('Expected the latitude and longitude coordinates '
'to have 1 or 2 dimensions, got {} and '
'{}.'.format(lat_coord.ndim, lon_coord.ndim))
# # Mask out points outside of extent in source_cs - disabled until
# # a way to specify global/limited extent is agreed upon and code
# # is generalised to handle -180 to +180, 0 to 360 and >360 longitudes.
# source_desired_xy = source_cs.transform_points(target_proj,
# target_x.flatten(),
# target_y.flatten())
# if np.any(source_x < 0.0) and np.any(source_x > 180.0):
# raise ValueError('Unable to handle range of longitude.')
# # This does not work in all cases e.g. lon > 360
# if np.any(source_x > 180.0):
# source_desired_x = (source_desired_xy[:, 0].reshape(ny, nx) +
# 360.0) % 360.0
# else:
# source_desired_x = source_desired_xy[:, 0].reshape(ny, nx)
# source_desired_y = source_desired_xy[:, 1].reshape(ny, nx)
# outof_extent_points = ((source_desired_x < source_x.min()) |
# (source_desired_x > source_x.max()) |
# (source_desired_y < source_y.min()) |
# (source_desired_y > source_y.max()))
# # Make array a mask by default (rather than a single bool) to allow mask
# # to be assigned to slices.
# new_data.mask = np.zeros(new_shape)
# Step through cube data, regrid onto desired projection and insert results
# in new_data array
for index, ll_slice in zip(index_it, slice_it):
# Regrid source data onto target grid
index = list(index)
index[xdim] = slice(None, None)
index[ydim] = slice(None, None)
index = tuple(index) # Numpy>=1.16 : index with tuple, *not* list.
new_data[index] = cartopy.img_transform.regrid(ll_slice.data, source_x,
source_y, source_cs,
target_proj, target_x,
target_y)
# # Mask out points beyond extent
# new_data[index].mask[outof_extent_points] = True
# Remove mask if it is unnecessary
if not np.any(new_data.mask):
new_data = new_data.data
# Create new cube
new_cube = iris.cube.Cube(new_data)
# Add new grid coords
x_coord = iris.coords.DimCoord(target_x[0, :],
'projection_x_coordinate',
units='m',
coord_system=copy.copy(target_cs))
y_coord = iris.coords.DimCoord(target_y[:, 0],
'projection_y_coordinate',
units='m',
coord_system=copy.copy(target_cs))
new_cube.add_dim_coord(x_coord, xdim)
new_cube.add_dim_coord(y_coord, ydim)
# Add resampled lat/lon in original coord system
source_desired_xy = source_cs.transform_points(target_proj,
target_x.flatten(),
target_y.flatten())
new_lon_points = source_desired_xy[:, 0].reshape(ny, nx)
new_lat_points = source_desired_xy[:, 1].reshape(ny, nx)
new_lon_coord = iris.coords.AuxCoord(new_lon_points,
standard_name='longitude',
units='degrees',
coord_system=orig_cs)
new_lat_coord = iris.coords.AuxCoord(new_lat_points,
standard_name='latitude',
units='degrees',
coord_system=orig_cs)
new_cube.add_aux_coord(new_lon_coord, [ydim, xdim])
new_cube.add_aux_coord(new_lat_coord, [ydim, xdim])
coords_to_ignore = set()
coords_to_ignore.update(cube.coords(contains_dimension=xdim))
coords_to_ignore.update(cube.coords(contains_dimension=ydim))
for coord in cube.dim_coords:
if coord not in coords_to_ignore:
new_cube.add_dim_coord(coord.copy(), cube.coord_dims(coord))
for coord in cube.aux_coords:
if coord not in coords_to_ignore:
new_cube.add_aux_coord(coord.copy(), cube.coord_dims(coord))
discarded_coords = coords_to_ignore.difference([lat_coord, lon_coord])
if discarded_coords:
warnings.warn('Discarding coordinates that share dimensions with '
'{} and {}: {}'.format(
lat_coord.name(), lon_coord.name(),
[coord.name() for coord in discarded_coords]))
# TODO handle derived coords/aux_factories
# Copy metadata across
new_cube.metadata = cube.metadata
return new_cube, extent
def create_random_synthetic_ifgs(volcanoes,
defo_sources,
n_ifgs,
n_pix=224,
outputs=['uuu'],
intermediate_figure=False,
coh_scale=5000,
coh_threshold=0.7,
coh_interpolation_threshold=5e3,
min_deformation=0.05,
max_deformation=0.25,
snr_threshold=2.0,
turb_aps_mean=0.02,
turb_aps_length=5000,
turb_aps_interpolation_threshold=5e3,
topo_aps_mean=56.0,
topo_aps_var=2.0):
"""
A function to generate n random synthetic interferograms at subaerial volcanoes in the Smithsonian database at SRTM3 resolution (ie.e. ~90m). Different deformation
sources are supported (no deformatin, point (Mogi), sill or dyke), topographically correlated and turbulent atmopsheric phase screens (APS) are added,
and areas of incoherence are synthesisd. The outputs are as rank 4 arrays with channels last (ie n_ifgs x ny x nx x 3 ), and can be in a variety of
styles (e.g. unwrapped across 3 channels, of unwrapped in channels 1 and 2 and the dem in 3). The paper Gaddes et al. (in prep) details this
in more detail.
General structure:
open_dem - these are required for making coastline and a topo correlated APS
coherence_mask - synthesise areas of incoherence.
atmosphere_turb - generates the spatially correlated noise which is used to create areas of incoherence.
create_random_defo_m - creates the random source_kwargs (e.g. depth/opening) and checks signals are of correct magnitude
deformation_wrapper - prepare grids in meters etc. and project 3D deformation to satellite LOS
deformation_Mogi - if deformation is Mogi, take source_wargs and make 3d surfaced deformation
deformation_eq_dyke_sill - if an Okada dislocation, take source_wargs and make 3d surfaced deformation
def_and_dem_translate - try random locations of the deformation signals and see if on land an in a coherent area.
atmosphere_turb - generate a turbulent APS
atmosphere_topo - generate a topo correlated APS
check_def_visible - check that signal to noise ratio (SNR) is acceptable and deformation pattern hasn't dissapeared.
combine_signals - combine signals and return in different 3 channel formats (ie for use with AlexNet etc. )
Inputs:
volcanoes | list of dicts | each volcano is a dictionary in the list, and contains various keys and values.
'dem': the dem 'lons_mg' : longitude of each pixel (ie a meshgrid) 'lats_mg' : latitude of each pixel
defo_sources | list | defo sources to be synthesised. e.g. ['no_def', 'dyke', 'sill', 'mogi']
n_ifgs | int | the number of interferogram to generate.
n_pix | int | Interferograms are square, with side length of this many pixels. Note that we use SRTM3 pixels, so squares of ~90m side length.
intermediate_figure | boolean | If True, a figure showing the search for a viable deformatin location and SNR is shown.
coh_scale | float | sets spatial scale of incoherent areas
coh_threshold | float | coherence is in range of 0-1, values above this are classed as incoherent
coh_interpolation_threshold | int | If n_pix is larger than this, interpolation will be used to generate the extra resolution (as the spatially correlated noise function used here is very slow for large images). Similar to the setting turb_aps_interpolation_threshold
min_deformation | float | Deformation must be above this size (in metres), even before checking the SNR agains the deformation and the atmosphere.
max_deformation | float | Deformation must be below this size (in metres), even before checking the SNR agains the deformation and the atmosphere.
snr_threshold | float | SNR of the deformation vs (topographically correlated APS + turbulent APS) must be above this for the signals to be considered as visible.
turb_aps_mean | float | mean strength of turbulent atmospheres, in metres. Note the the atmosphere_turb funtion takes cmm, and the value provided in m is converted first
turb_aps_length | float | Length scale of spatial correlatin, in metres. e.g. 5000m
turb_aps_interpolation_threshold | int | If n_pix is larger than this, interpolation will be used to generate the extra resolution (as the spatially correlated noise function used here is very slow for large images). Similar to the setting coh_interpolation_threshold
topo_aps_mean | float | rad/km of delay for the topographically correlated APS
topo_aps_var | float | rad/km. Sets the strength difference between topographically correlated APSs
Returns:
X_all | dict of masked arrays | keys are formats (e.g. uuu), then rank 4 masked array
Y_class | rank 2 array | class labels, n x 1 (ie not one hot encoding)
Y_loc | rank 2 array | location of deformaiton, nx4 (xy location, xy width)
Y_source_kwargs | list of dicts | stores the source_kwargs that were generated randomly to create each interferogram. Also contains the source names (ie the same as Y_class, but as a string).
History:
2020/10/19 | MEG | Written from various scripts.
2020/10/26 | MEG | Add funtion to record the source_kwargs. Intended for use if these are to be the label of interest (e.g. trianing a CNN to determine strike etc. )
"""
import numpy as np
import numpy.ma as ma
import matplotlib.pyplot as plt
from syinterferopy_functions import coherence_mask, atmosphere_turb, atmosphere_topo
from auxiliary_functions import truncate_colormap # needed to plot the DEM with nice (terrain) colours
# hard coded variables:
count_max = 8 # the number of times the function searches for acceptable deformation positions and SNR
# begin to generate the data for this output file
succesful_generate = 0 # count how many ifgs succesful made so we can stop when we get to n_ifgs
attempt_generate = 0 # only succesfully generated ifgs are counted above, but also useful to count all
X_all = {} # the data will be stored in a dictionary, X_all
for output in outputs:
X_all[output] = ma.zeros(
(n_ifgs, n_pix, n_pix, 3)
) # populate the dictionary with the required outputs and empty arrays.
Y_class = np.zeros(
(n_ifgs, 1)) # initate for labels showing type of deformation
Y_loc = np.zeros(
(n_ifgs, 4)) # initate for labels showing location of deformation
Y_source_kwargs = [
] # initate an empty list for storing source kwargs (parameters like opening dip etc.)
while succesful_generate < n_ifgs:
volcano_n = np.random.randint(
0, len(volcanoes)) # choose a volcano at random
defo_source = defo_sources[np.random.randint(0, len(
defo_sources))] # random choice of which deformation source to use
print(f"Volcano: {volcanoes[volcano_n]['name']} ", end='')
# 0: generate incoherence mask, choose dem choose if ascending or descending.
dem_large = volcanoes[volcano_n]['dem'] # open a dem
dem_ll_extent = [
(volcanoes[volcano_n]['lons_mg'][0, 0],
volcanoes[volcano_n]['lats_mg'][0, 0]
), # get lon lat of lower left corner
(volcanoes[volcano_n]['lons_mg'][-1, -1],
volcanoes[volcano_n]['lats_mg'][-1, -1])
] # and upper right corner
mask_coherence = coherence_mask(
volcanoes[volcano_n]['lons_mg'][:n_pix, :n_pix],
volcanoes[volcano_n]['lats_mg']
[:n_pix, :
n_pix], # generate coherence mask, but at the number of pixels required for the ouput, and not hte size of the large dem
coh_scale,
coh_threshold,
coh_interpolation_threshold
) # if threshold is 0, all of the pixels are incoherent , and if 1, none are.
print(f"| Coherence mask generated ", end='')
if np.random.rand() < 0.5:
asc_or_desc = 'asc'
heading = 348
else:
asc_or_desc = 'desc'
heading = 192
print(f"| Deformation source: {defo_source} ", end='')
if intermediate_figure:
f, axes = plt.subplots(3, count_max, figsize=(16, 8))
f.suptitle(
f"{attempt_generate}: Volcano: {volcanoes[volcano_n]['name']} | Deformation label: {defo_source}"
)
f.canvas.set_window_title(
f"{attempt_generate}_Volcano:{volcanoes[volcano_n]['name']}")
axes[0, 0].set_ylabel(
'Location search \n (hatch = water or incoherent)')
axes[1, 0].set_ylabel('SNR search \n (defo + APS_turb + APT_topo')
for axe_n, axe in enumerate(axes[0, :]):
axe.set_title(f"Attempt # {axe_n}")
cmap = plt.get_cmap('terrain') # makes sense for DEMs
cmap = truncate_colormap(
cmap, 0.2, 1
) # but by deafult it starts at blue, so crop (truncate) that part off.
axes[2, 0].imshow(dem_large, cmap=cmap)
axes[2, 0].set_xlabel('Full DEM')
axes[2, 1].imshow(mask_coherence)
axes[2, 1].set_xlabel('Coherence Mask')
axes[2, 1].yaxis.tick_right()
for axe in np.concatenate((axes[0, 1:], axes[1, :], axes[2, 2:]),
axis=0):
axe.set_axis_off()
# 1: If no deformation, just generate topo. correlated and turbulent APS
if defo_source == 'no_def':
viable_location = viable_snr = True # in the no deformation case, these are always True
source_kwargs = {'source': 'no_def'}
defo_m = np.ones(
dem_large.shape
) # in the no deformation case, make a deformaiton that is just zeros.
defo_m, dem, viable_location, loc_list, masks = def_and_dem_translate(
dem_large,
defo_m,
mask_coherence,
threshold=0.3,
n_pixs=n_pix,
defo_fraction=0.8
) # doesn't matter if this returns false. Note that masks is a dictionary of deformation, coherence and water, and water
dem = ma.array(dem, mask=masks['coh_water']
) # mask the DEM for water and incoherence
APS_turb_m = atmosphere_turb(
1,
volcanoes[volcano_n]['lons_mg'][:n_pix, :n_pix],
volcanoes[volcano_n]['lats_mg']
[:n_pix, :
n_pix], # generate a turbulent APS, but for speed not at the size of the original DEM, and instead at the correct n_pixs
None,
turb_aps_length,
False,
False,
turb_aps_interpolation_threshold,
turb_aps_mean)
APS_turb_m = APS_turb_m[0, ] # remove the 1st dimension
APS_topo_m = atmosphere_topo(
dem, topo_aps_mean, topo_aps_var, difference=True
) # generate a topographically correlated APS # helps to split up and clarify terminal output.
if intermediate_figure:
axes[0, 0].imshow(np.zeros((n_pix, n_pix)))
axes[0, 0].set_xlabel(f"No deformation")
axes[0, 0].set_axis_on()
temp_combined = APS_topo_m + APS_turb_m
axes[1, 0].imshow(temp_combined)
axes[1, 0].set_xlabel(
f"[{np.round(np.min(temp_combined), 2)}, {np.round(np.max(temp_combined),2 )}] m"
)
axes[1, 0].set_axis_on()
plt.pause(4)
# 2: Or, if we do have deformation, generate it and topo correlated and turbulent APS
else:
viable_location = False
count = 0
# prepare for while statement that will search for a viable deformation location.
# 2a: Try to make a deformation signal of the correct magnitude, and then place it on land.
while viable_location is False and count < count_max: # random translations of dem and deformation, but deformation must remain visible.
defo_m, source_kwargs = create_random_defo_m(
dem_large,
volcanoes[volcano_n]['lons_mg'],
volcanoes[volcano_n]['lats_mg'],
volcanoes[volcano_n]['centre'],
defo_source, # make a deformation signal with a size within the bounds set by min and max.
min_deformation,
max_deformation,
asc_or_desc
) # Note that is is made at the size of the large DEM (dem_large)
source_kwargs[
'source'] = defo_source # add name of source to dict of source_kwargs (e.g. depth/opening etc. )
defo_m, dem, viable_location, loc_list, masks = def_and_dem_translate(
dem_large,
defo_m,
mask_coherence,
threshold=
0.3, # do the random crop of the dem and the defo pattern, so reducing the size to that desired.
n_pixs=n_pix,
defo_fraction=0.8
) # and check that the majority of the deformation pattern isn't in an incoheret area, or in water.
dem = ma.array(
dem, mask=masks['coh_water']
) # mask the DEM (for water and areas of incoherence)
if intermediate_figure:
axes[0, count].imshow(ma.array(defo_m,
mask=masks['coh_water']),
vmin=np.min(defo_m),
vmax=np.max(defo_m))
axes[0,
count].imshow(ma.array(defo_m,
mask=1 - masks['coh_water']),
vmin=np.min(defo_m),
vmax=np.max(defo_m))
axes[0, count].contourf(masks['coh_water'],
1,
hatches=['', '//'],
alpha=0)
axes[0, count].set_xlabel(
f"Viable Location: {viable_location}")
axes[0, count].set_axis_on()
plt.pause(1)
if viable_location == False:
count += 1
if viable_location: #
# 2b: If we have a viable size and location, try to make the atmospheric signals and check that deforamtion is still visible (ie acceptable signal to noise ratio)
print(f"| Viable location ", end='')
viable_snr = False
count = 0 # make dem and ph_def and check that def is visible
while viable_snr is False and count < count_max:
APS_turb_m = atmosphere_turb(
1,
volcanoes[volcano_n]['lons_mg'][:n_pix, :n_pix],
volcanoes[volcano_n]['lats_mg']
[:n_pix, :
n_pix], # generate a turbulent APS, but for speed not at the size of the original DEM, and instead at the correct n_pixs
None,
turb_aps_length,
False,
False,
turb_aps_interpolation_threshold,
turb_aps_mean)
APS_turb_m = APS_turb_m[0, ] # remove the 1st dimension
APS_topo_m = atmosphere_topo(
dem, topo_aps_mean, topo_aps_var, difference=True
) # generate a topographically correlated APS using the DEM
viable_snr, snr = check_def_visible(
defo_m, masks['def'], APS_topo_m, APS_turb_m,
snr_threshold
) # check that the deformation is visible over the ph_topo and ph_trub (SNR has to be above snr_threshold)
if intermediate_figure:
temp_combined = defo_m + APS_topo_m + APS_turb_m
axes[1, count].imshow(temp_combined)
axes[1, count].set_xlabel(
f"[{np.round(np.min(temp_combined), 2)}, {np.round(np.max(temp_combined),2 )}] m \n"
f"SNR: {np.round(snr, 2)}")
axes[1, count].set_axis_on()
plt.pause(1)
if viable_snr == False:
count += 1
if viable_snr:
print(f"| Viable SNR ", end='')
else:
print('| SNR is too low. \n')
plt.pause(2)
else:
print(f"| No viable location found. \n")
# 3: If succesful, append to X (data) and Y (labels) arrays. # still in the main while loop, but out of the deformation / no deformation else statement.
if (viable_location and viable_snr) or (defo_source == 'no_def'):
X_all, Y_class, Y_loc, succesful = combine_signals(
X_all, Y_class, Y_loc, defo_m, APS_topo_m, APS_turb_m, heading,
dem, defo_source, defo_sources, loc_list, outputs,
succesful_generate
) # we have a succesful flag as sometimes this can fail due to Nans etc.
if succesful:
Y_source_kwargs.append(source_kwargs)
succesful_generate += 1 # updat the countery of how many ifgs have been succesfuly made
print(f"| Succesful write. \n")
else:
print(f"| Failed write. \n")
attempt_generate += 1 # update the counter of how many have been made in total (successes and failures)
plt.close()
return X_all, Y_class, Y_loc, Y_source_kwargs
def estimShifts(self, dataOneSubjTB, thetas, variances, ageOneSubj1array,
clustProbBC, prevSubShift, prevSubShiftAvg, fixSpeed):
'''
do not use dot product because when NaNs are involved the weights will not sum to 1.
use np.ma.average(.., weights) instead, as the weights will be re-normalised accordingly
'''
clustProbBCColNorm = clustProbBC / np.sum(clustProbBC, 0)[None, :]
nrBiomk, nrClust = clustProbBC.shape
nrTimepts = dataOneSubjTB.shape[0]
dataOneSubjBT = dataOneSubjTB.T
# declare it as masked array, compute it for every cluster with ma.average
dataOneSubjWeightedCT = ma.zeros((nrClust, nrTimepts), float)
for c in range(nrClust):
dataOneSubjWeightedCT[c, :] = ma.average(
dataOneSubjBT, axis=0, weights=clustProbBCColNorm[:, c])
# convert back to np array for speed, do the calculation manually.
dataOneSubjWeiManMaskCT = np.array(dataOneSubjWeightedCT)
dataOneSubjWeiManMaskCT[dataOneSubjWeightedCT.mask] = np.nan
if fixSpeed: # fixes parameter alpha to 1
composeShift = lambda beta: [prevSubShiftAvg[0], beta]
initSubShift = prevSubShift[1]
# objFuncLambda2 = lambda beta: self.objFunShift(composeShift(beta), dataOneSubjWeightedCT, thetas,
# variances, ageOneSubj1array, clustProbBC)
objFuncLambda = lambda beta: self.objFunShiftMaskedManual(
composeShift(beta), dataOneSubjWeiManMaskCT, thetas, variances,
ageOneSubj1array, clustProbBC)
prevSubShiftAvgCurr = prevSubShiftAvg[1].reshape(1, -1)
else:
composeShift = lambda shift: shift
initSubShift = prevSubShift
# objFuncLambda2 = lambda shift: self.objFunShift(shift, dataOneSubjWeightedCT, thetas,
# variances, ageOneSubj1array, clustProbBC)
objFuncLambda = lambda beta: self.objFunShiftMaskedManual(
composeShift(beta), dataOneSubjWeiManMaskCT, thetas, variances,
ageOneSubj1array, clustProbBC)
prevSubShiftAvgCurr = prevSubShiftAvg
# assert objFuncLambda(initSubShift) == objFuncLambda2(initSubShift)
# print(adsa)
# print('objFuncLambda(initSubShift)', objFuncLambda(initSubShift))
res = scipy.optimize.minimize(objFuncLambda,
initSubShift,
method='Nelder-Mead',
options={
'xatol': 1e-2,
'disp': False
})
bestShift = res.x
nrStartPoints = 2
nrParams = prevSubShiftAvgCurr.shape[0]
pertSize = 1
minSSD = res.fun
success = False
for i in range(nrStartPoints):
perturbShift = prevSubShiftAvgCurr * (
np.ones(nrParams) + pertSize * np.random.multivariate_normal(
np.zeros(nrParams), np.eye(nrParams)))
res = scipy.optimize.minimize(objFuncLambda,
perturbShift,
method='Nelder-Mead',
options={
'xtol': 1e-8,
'disp': False,
'maxiter': 100
})
currShift = res.x
currSSD = res.fun
# print('currSSD', currSSD, objFuncLambda(currShift))
if currSSD < minSSD:
# if we found a better solution then we decrease the step size
minSSD = currSSD
bestShift = currShift
pertSize /= 1.2
success = res.success
else:
# if we didn't find a solution then we increase the step size
pertSize *= 1.2
print('bestShift', bestShift)
return composeShift(bestShift)
]
n_models = len(cmip6_models)
model_shapes = ['o', '^', '+', 's', '*', 'd', 'x']
# SSP scenarios
ssp_options = ['ssp126', 'ssp245', 'ssp585']
ssp_options_length = len(ssp_options)
# Global mean temperature change
temperature_change_options = [1, 2, 3]
temperature_change_options_length = len(temperature_change_options)
#%%
# defining array to save values
x_array = ma.zeros(
[len(temperature_change_options),
len(ssp_options),
len(cmip6_models)])
y_array = ma.zeros(
[len(temperature_change_options),
len(ssp_options),
len(cmip6_models)])
obs_array = ma.zeros(
[len(temperature_change_options),
len(ssp_options) * len(cmip6_models)])
#%%
# loop through each global mean temperature change
for temp_option in range(0, temperature_change_options_length):
min_temperature = temperature_change_options[
temp_option] # selecting the temperature change
def deriv(*args, **kwargs):
"""Calculate the derivative along a single dimension.
Calling Sequence:
Result = deriv([x_in,] y_in, missing=1e+20, algorithm='default')
Positional Input Arguments:
* x_in: Abscissa values of y_in to take with respect to. If
not set, the derivative of y_in is take with respect to unit
abscissa intervals. Numeric array of same shape and size as
y_in. Must be monotonic and with no duplicate values.
Optional. First positional argument out of two, if present.
* y_in: Ordinate values, to take the derivative with respect
to. Numeric array vector of rank 1. Required. Second posi-
tional argument, if x_in is present; only positional argument
if x_in is absent.
Keyword Input Arguments:
* missing: If y_in and/or x_in has missing values, this is the
missing value value. Scalar. Default is 1e+20.
* algorithm: Name of the algorithm to use. String scalar.
Default is 'default'. Possible values include:
+ 'default': Default method (currently set to 'order1').
+ 'order1': First-order finite-differencing (backward and
forward differencing used at the endpoints, and centered
differencing used everywhere else). If abscissa intervals
are irregular, differencing will be correspondingly asym-
metric.
Output Result:
* Derivative of y_in with respect to x_in (or unit interval
abscissa, if x_in is not given). Numeric array of same shape
and size as y_in. If there are missing values, those elements
in the output are set to the value in |missing|. For instance,
if y_in is only one element, a one-element vector is returned
as the derivative with the value of |missing|. If there are
missing values in the output due to math errors and |missing|
is set to None, output will fill those missing values with the
MA default value of 1e+20.
References:
* Press, W. H., et al. (1992): Numerical Recipes in Fortran
77: The Art of Scientific Computing. New York, NY: Cambridge
University Press, pp. 180-184.
* Wang, Y. (1999): "Numerical Differentiation," Introduction to
MHD Numerical Simulation in Space, ESS265: Instrumentation,
Data Processing and Data Analysis in Space Physics (UCLA).
URL: http://www-ssc.igpp.ucla.edu/personnel/russell/ESS265/
Ch10/ylwang/node21.html.
Example with one argument, no missing values, using the default
method:
>>> from deriv import deriv
>>> import Numeric as N
>>> y = N.sin(N.arange(8))
>>> dydx = deriv(y)
>>> ['%.7g' % dydx[i] for i in range(4)]
['0.841471', '0.4546487', '-0.3501755', '-0.83305']
>>> true = N.cos(N.arange(8)) #- Compare with exact solution
>>> ['%.7g' % true[i] for i in range(4)]
['1', '0.5403023', '-0.4161468', '-0.9899925']
Example with two arguments with missing values, using first-
order differencing:
>>> x = N.arange(8)/(2.*N.pi)
>>> y = N.sin(x)
>>> y[3] = 1e20 #- Set an element to missing value
>>> dydx = deriv(x, y, missing=1e20, algorithm='order1')
>>> ['%.7g' % dydx[i] for i in range(5)]
['0.9957836', '0.9831985', '1e+20', '0.8844179', '1e+20']
>>> true = N.cos(x) #- Compare with exact solution
>>> ['%.7g' % true[i] for i in range(5)]
['1', '0.9873616', '0.9497657', '0.8881628', '0.8041098']
"""
#- Establish y_in and x_in from *args:
if len(args) == 1:
y_in = args[0]
x_in = np.arange(len(y_in), dtype=y_in.dtype)
elif len(args) == 2:
x_in = args[0]
y_in = args[1]
else:
raise ValueError("deriv: Bad inputs")
#- Establish missing and algorithm from *kwargs:
if ('missing' in kwargs) == 1:
missing = kwargs['missing']
else:
missing = 1e+20
if ('algorithm' in kwargs) == 1:
algorithm = kwargs['algorithm']
else:
algorithm = 'default'
#- Check positional and keyword inputs for possible errors:
if (len(y_in.shape) != 1) or (len(x_in.shape) != 1):
raise ValueError("deriv: Inputs not a vector")
if type(algorithm) != type(''):
raise ValueError("deriv: algorithm not str")
#- Set algorithm_to_use variable, based on the algorithm keyword.
# The algorithm_to_use tells which algorithm below to actually
# use (so here is where we set what algorithm to use for default):
if algorithm == 'default':
algorithm_to_use = 'order1'
else:
algorithm_to_use = algorithm
#- Change input to MA: just set to input value unless there are
# missing values, in which case add mask:
if missing == None:
x = ma.masked_array(x_in)
y = ma.masked_array(y_in)
else:
x = ma.masked_values(x_in, missing, copy=0)
y = ma.masked_values(y_in, missing, copy=0)
#- Calculate and return derivative:
# * Create working arrays that are consistent with a 3-point
# stencil in the interior and 2-point stencil on the ends:
# *im1 means the point before index i, *ip1 means the point
# after index i, and the i index array is just plain x or
# y; the endpadded arrays replicate the ends of x and y.
# I use an MA array filled approach instead of concatentation
# because the MA concatenation routine doesn't work right
# when the endpoint element is a missing value:
x_endpadded = ma.zeros(x.size + 2, dtype=x.dtype)
x_endpadded[0] = x[0]
x_endpadded[1:-1] = x
x_endpadded[-1] = x[-1]
y_endpadded = ma.zeros(y.size + 2, dtype=y.dtype)
y_endpadded[0] = y[0]
y_endpadded[1:-1] = y
y_endpadded[-1] = y[-1]
y_im1 = y_endpadded[:-2]
y_ip1 = y_endpadded[2:]
x_im1 = x_endpadded[:-2]
x_ip1 = x_endpadded[2:]
# * Option 1: First-order differencing (interior points use
# centered differencing, and end points use forward or back-
# ward differencing, as applicable):
if algorithm_to_use == 'order1':
dydx = (y_ip1 - y_im1) / (x_ip1 - x_im1)
# * Option 2: Bad algorithm specified:
else:
raise ValueError("deriv: bad algorithm")
#- Return derivative as Numeric array:
return ma.filled(dydx, missing)
def cascadeSpec(self,
epsGeV,
gmin=None,
gsteps=25,
Esteps=25,
epssteps=25,
epsmin=1e-10,
epsmax=1e1,
intSpectrum=True):
"""
Calculate cascade spectrum.
Parameters
----------
epsGeV: n-dim array, energy of upscattered CMB photons, in GeV
kwargs
------
gmin: n-dim array with minimum gamma-factors.
If none, calculated from gammaMin function (default = None)
gsteps: int, number of steps for gamma integration (default = 20)
Esteps: int, number of steps for energy integrations
epsmin: float, minimum energy of CMB integration, in eV (default = 1e-10)
epsmax: float, maximum energy of CMB integration, in eV (default = 1e1)
intSpectrum: bool,
if True, provided spectrum assumed to be the intrinsic spectrum,
otherwise, assumed to be observed spectrum
Returns
-------
n-dim array with cascade spectrum in 1/cm^2/s/eV
Notes
-----
"""
if gmin == None:
logGmin = log(self.gammaMin(epsGeV))
else:
logGmin = log(gmin)
# maximum gamma factor
# corresponding to max prim. energy
#logGmax = log(self.EmaxTeV * 1e12 / (2. * M_E_EV)) * 0.999
logGmax = log(1e8)
# check if maximum gamma factor is larger than minimum gamma factor
if np.all(logGmin >= logGmax):
wString = '*** all minimum gamma values are larger than the maximum gamma value:'
logging.warning(wString)
logging.warning('*** gmax: {0:.3e}'.format(exp(logGmax)))
logging.warning('*** max(gammaMin): {0:.3e}'.format(
np.max(self.gammaMin(epsGeV))))
logging.warning('*** max(gammaEngine): {0:.3e}'.format(
np.max(self.gammaEngine(epsGeV))))
logging.warning('*** max(gammaDeflect): {0:.3e}'.format(
np.max(self.gammaDeflect(epsGeV))))
logging.warning('*** max(gammaCMB): {0:.3e}'.format(
np.max(self.gammaCMB(epsGeV))))
return np.zeros(epsGeV.shape)
# mask out regions that where gmin > gmax
epsGeV = ma.array(epsGeV, mask=logGmin >= logGmax)
logGmin = ma.array(logGmin, mask=epsGeV.mask)
# define the energy arrays for integration
# outer integral runs over log gamma (one log gamma for each cascade energy)
# inner integral runs over primary energy spectrum
# second inner integral runs over energy of CMB spectrum
logG = []
logEprim = []
logEpsCMB = []
tauPrim = []
logG = ma.zeros((epsGeV.shape[0], gsteps))
logG.mask = np.zeros(logG.shape, dtype=bool)
for ilg, logg in enumerate(
logGmin): # first loop over minimum gamma factors
logG.mask[ilg] = logGmin.mask[ilg] * np.ones(gsteps, dtype=bool)
if not logGmin.mask[ilg]:
# check if corresponding minimum energy is larger than max energy
if logg + log(2. * M_E_EV * 1e-12) > log(self.EmaxTeV):
logG.mask[ilg] = np.ones(gsteps, dtype=bool)
else:
logG[ilg] = np.linspace(logg, logGmax, gsteps)
# calculate the lower bound for the first inner integral for the injected spectrum
# in TeV
logEmin = log(self.EminTeV) * np.ones(logG.shape)
logEprimMinTeV = logG + log(2. * M_E_EV * 1e-12)
logEprimMinTeV = logEprimMinTeV * (logEprimMinTeV > logEmin) + \
logEmin * (logEprimMinTeV <= logEmin)
# generate three dim arrays for the intergral over the primary spectrum
# also generate the optical depths for the injected energies
logEprim = ma.zeros((epsGeV.shape[0], gsteps, Esteps))
logEprim.mask = np.zeros(logEprim.shape, dtype=bool)
tauPrim = ma.zeros((epsGeV.shape[0], gsteps, Esteps))
tauPrim.mask = np.zeros(tauPrim.shape, dtype=bool)
for ilE, logE in enumerate(logEprimMinTeV):
for jlE, logEE in enumerate(logE):
logEprim.mask[ilE,jlE] = logEprimMinTeV.mask[ilE,jlE] * \
np.ones(Esteps, dtype = bool)
if not (logEprimMinTeV.mask[ilE, jlE]
or logEE > log(self.EmaxTeV)):
logEprim[ilE, jlE] = np.linspace(logEE, log(self.EmaxTeV),
Esteps)
tauPrim[ilE,
jlE] = self.tau.opt_depth(self.zSource,
exp(logEprim[ilE, jlE]))
# if all masks are true (gmin > gmax) || (ETeVmin > ETeVmax)
# return zero
if np.all(logEprim.mask):
logging.warning(
"All (gmin > gmax) || (ETeVmin > ETeVmax), returning 0.")
return np.zeros(epsGeV.shape)
tauPrim.mask = logEprim.mask
# generate three dim arrays for the intergral over the CMB spectrum
logEpsCMB = ma.zeros((epsGeV.shape[0], gsteps, epssteps))
logEpsCMB.mask = np.zeros(logEpsCMB.shape, dtype=bool)
x = ma.zeros((epsGeV.shape[0], gsteps, epssteps))
for ilg, lg in enumerate(logG):
for jlg, lgg in enumerate(lg):
logEpsCMB[ilg, jlg] = np.linspace(log(epsmin), log(epsmax),
epssteps)
x[ilg,jlg] = epsGeV[ilg] * 1e9 / 4. / exp(logEpsCMB[ilg,jlg]) / \
exp(2. * logG[ilg,jlg])
x.mask = logEpsCMB.mask | (x >= 1)
# calculate kernel for CMB integral ---------------------- #
# the rollaxis calls do cast the arrays into the right shapes to
# be multiplied
# log(x) will return warning for masked entries
kernelCMB = self._F_IC_T(x) * 4. * \
nphotCMBarray(exp(logEpsCMB)) / exp(logEpsCMB)
kernelCMB *= exp(logEpsCMB)
# rollaxis needed, since logG and kernelCMB have different dimensions
kernelCMB = np.rollaxis(
np.rollaxis(np.rollaxis(kernelCMB, 2) * exp(2. * logG), 2), 2)
# Do the integration over CMB energy, with rollaxis this is axis 0 at the moment
kernelGamma = simps(kernelCMB, logEpsCMB, axis=2)
# calculate kernel for injected source integral ------------------- #
kernelInjSpec = self.intSpec(exp(logEprim), **self.intSpecPar)
if intSpectrum: # provided spectrum is the intrinsic one
kernelInjSpec *= 1. - exp(-tauPrim)
else: # provided spectrum is the observed one
kernelInjSpec *= exp(tauPrim) - 1.
kernelInjSpec *= exp(logEprim) # account for log integration
kernelGamma *= simps(kernelInjSpec, logEprim, axis=2) / exp(logG * 5.)
mask = np.isnan(kernelGamma)
kernelGamma = ma.array(kernelGamma, mask=mask)
logG = ma.array(logG, mask=mask | logG.mask)
#logging.info('new {0}'.format(kernelGamma))
#logging.info('new {0}'.format(kernelGamma.shape))
#logging.info('new {0}'.format(logG))
#logging.info('new {0}'.format(logG.shape))
result = np.zeros(epsGeV.shape)
for ikG, kG in enumerate(kernelGamma):
if np.all(kG.mask): continue
result[ikG] = simps(kG.compressed(), logG[ikG].compressed())
result *= M_E_EV / U_CMB * 9. / 64.
return result
filenames1 = glob.glob(
'C:/Users/admin/Desktop/Master_thesis/Javier_ERAI_winds_icedrift_OSISAF/SortedData_OSISAF_IceWind/WindSpeed_10m/'
+ PEP + '/' + B + '/*.nc')
filenames2 = glob.glob(
'C:/Users/admin/Desktop/Master_thesis/Javier_ERAI_winds_icedrift_OSISAF/SortedData_OSISAF_IceWind/SeaIceDriftSpeed/'
+ PEP + '/' + B + '/*.nc')
Wu = np.zeros(
[21063, len(filenames1)], dtype=np.float
) # 2D array to save LOCAL (X) WIND VELOCITY values as they're extracted from each .nc file (colunmns)
Wv = np.zeros(
[21063, len(filenames1)], dtype=np.float
) # 2D array to save MERIDIONAL (Y) WIND VELOCITY values as they're extracted from each .nc file (colunmns)
Iu = ma.zeros(
[21063, len(filenames2)], dtype=np.float
) # 2D array to save LOCAL (X) ICE VELOCITY values as they're extracted from each .vec file (colunmns)
Iv = ma.zeros(
[21063, len(filenames2)], dtype=np.float
) # 2D array to save MERIDIONAL (X) ICE VELOCITY values as they're extracted from each .vec file (colunmns)
n = -1 # initial value of counter for iceDrift files
nn = -1 # initial value of counter for wind files
for filename in filenames1: # extracts data from each .nc file one by one
datum1 = Dataset(filename, mode='r')
print('filename: ', filename)
lat1 = datum1.variables['lat1'][:]
lon1 = datum1.variables['lon1'][:]
def Read(self, **args):
# self.dsetname = self.runConfig.get( 'input', 'Dataset' )
# cfgDir = os.path.dirname( self.runConfigFile )
# datasetConfigFile = os.path.expanduser( os.path.join( cfgDir, 'datasets.cfg' ) )
# self.datasetConfig = ConfigParser.RawConfigParser()
# self.datasetConfig.read( datasetConfigFile )
#
# inputDatasetDir = self.datasetConfig.get( self.dsetname, 'DataDir' )
# self.latVarName = self.datasetConfig.get( self.dsetname, 'LatVarName' )
# self.lonVarName = self.datasetConfig.get( self.dsetname, 'LonVarName' )
# self.vertVarName = self.datasetConfig.get( self.dsetname, 'VertVarName' )
# self.timeVarName = self.datasetConfig.get( self.dsetname, 'TimeVarName' )
# inputFileName = self.runConfig.get( 'input', 'Datafile' )
# inputFilePath = os.path.join(inputDatasetDir,inputFileName)
self.zscale = args.pop("zscale", self.zscale)
if self.varName == None:
self.varName = self.GetDefaultVarName()
try:
var = self.variableList[self.varName]
except:
print >> sys.stderr, "\n\n Error reading var %s from variable list: %s\n" % (
self.varName, str(self.variableList.keys()))
return
# self.missingValue = var.fmissing_value
inputDataBounds = self.GetDataBounds()
shapeT = np.array(var.shape)
self.shape = np.ndarray([len(shapeT)], dtype=int)
self.shape[::1] = shapeT[::-1]
self.nTimeFrames = 1 if len(self.shape) < 4 else self.shape[3]
self.vertvar = self.variableList[self.vertVarName]
self.ROIExtent = np.array(
[0, self.shape[0], 0, self.shape[1], 0, self.shape[2]])
self.dataRange = np.array([
inputDataBounds[1] - inputDataBounds[0],
inputDataBounds[3] - inputDataBounds[2],
inputDataBounds[5] - inputDataBounds[4]
])
self.dataOffset = np.array(
[inputDataBounds[0], inputDataBounds[2], inputDataBounds[4]])
offset = [180, 90]
if self.ROI != None:
for iD in range(4):
self.ROI[iD] = self.ROI[iD] + offset[iD / 2]
for iD in range(0, 6, 2):
if self.ROI[iD] < inputDataBounds[iD]:
self.ROI[iD] = inputDataBounds[iD]
for iD1 in range(1, 6, 2):
if self.ROI[iD1] > inputDataBounds[iD1]:
self.ROI[iD1] = inputDataBounds[iD1]
for i in range(6):
i0 = i / 2
ROIFraction = (self.ROI[i] -
self.dataOffset[i0]) / self.dataRange[i0]
self.ROIExtent[i] = int(ROIFraction * self.shape[i0])
roishapeT = list(var.shape)
roishapeT[-1] = self.ROIExtent[1] - self.ROIExtent[0]
roishapeT[-2] = self.ROIExtent[3] - self.ROIExtent[2]
roishapeT[-3] = self.ROIExtent[5] - self.ROIExtent[4]
self.roishape = np.ndarray([len(roishapeT)], dtype=int)
self.roishape = roishapeT[::-1]
# tempvar = np.zeros( roishapeT, dtype=var.dtype )
subvar = var[..., self.ROIExtent[4]:self.ROIExtent[5],
self.ROIExtent[2]:self.ROIExtent[3],
self.ROIExtent[0]:self.ROIExtent[1]]
# tempvar = subvar
# tempvar = ma.masked_array( subvar )
# undef_value_mask = tempvar == self.missingValue
# tempvar.mask = undef_value_mask
tempvar = ma.masked_values(subvar, self.undef)
# undef_value_mask = tempvar == self.missingValue
# tempvar.mask = undef_value_mask
tempvar = tempvar.transpose()
newvar = tempvar
if self.reductionFactor > 1:
newshape = np.array(self.roishape)
for i in range(-2, 0):
self.shape[i] /= self.reductionFactor
newshape[i] /= self.reductionFactor
newvar = ma.zeros(newshape, dtype=var.dtype)
bounds = newshape * self.reductionFactor
for ix in range(0, self.reductionFactor):
for iy in range(0, self.reductionFactor):
subarray = tempvar[ix:bounds[0]:self.reductionFactor,
iy:bounds[1]:self.reductionFactor, ...]
newvar[...] += subarray
newvar /= (self.reductionFactor * self.reductionFactor)
print "ReadNetCDFVariable '" + self.varName + "', roi shape = " + str(
self.roishape) + ", data var shape = " + str(
var.shape) + ", new var shape = " + str(
newvar.shape) + ", ROI Extent = " + str(
self.ROIExtent) + ", reductionFactor = " + str(
self.reductionFactor) + ", InvertZ = " + str(
self.invertZ)
if self.invertZ:
self._dataArray = newvar.copy()
self._dataArray = newvar[..., ::-1, :]
del newvar
else:
self._dataArray = newvar
range_min = self._dataArray.min()
range_max = self._dataArray.max()
self.timeseries_range = [range_min, range_max]
self.grid_origin = self.GetDataCornerPosition()
self.grid_spacing = self.GetDataRangeScaling(self.zscale)
self.hasData = True
import numpy.ma as ma
import os.path
hits = np.array([[0, 0, 0, 0, 0], [0, 1, 1, 1, 0], [0, 1, 1, 2, 0],
[0, 2, 1, 3, 0], [0, 0, 0, 0, 1]])
visits = np.array([[0, 0, 0, 0, 0], [0, 1, 1, 1, 0], [0, 1, 9, 3, 0],
[0, 2, 2, 4, 0], [0, 0, 0, 0, 8]])
undef_mask = (visits == 0)
alpha = ma.masked_array(hits, dtype=np.float)
alpha[undef_mask] = ma.masked
means = ma.divide(alpha, visits)
means_ds = ma.zeros(means.shape)
means_ds[undef_mask] = DiSt.UNDEFINED.value
means_ds[~undef_mask] = ma.masked
worldmap_extent = [150.4, 183.0, 0, 24.5]
test_ds_list = [DiSt.UNDEFINED, DiSt.UNIFORM, DiSt.BIMODAL]
test_v_min = 0
test_v_max = 1
test_occ = True
# Create Colorizer Object
mean_colorizer = MapColorizer()
mean_colorizer.set_wm_extent(worldmap_extent)
mean_colorizer.set_disc_state_list(test_ds_list)
def comp_r(dert__, fig, root_fcr):
'''
Cross-comparison of input param (dert[0]) over rng passed from intra_blob.
This fork is selective for blobs with below-average gradient,
where input intensity didn't vary much in shorter-range cross-comparison.
Such input is predictable enough for selective sampling: skipping current
rim derts as kernel-central derts in following comparison kernels.
Skipping forms increasingly sparse output dert__ for greater-range cross-comp, hence
rng (distance between centers of compared derts) increases as 2^n, starting at 0:
rng = 1: 3x3 kernel,
rng = 2: 5x5 kernel,
rng = 4: 9x9 kernel,
...
Due to skipping, configuration of input derts in next-rng kernel will always be 3x3, see:
https://github.com/boris-kz/CogAlg/blob/master/frame_2D_alg/Illustrations/intra_comp_diagrams.png
'''
# initialize new dert structure
new_dert__ = ma.zeros((dert__.shape[0], (dert__.shape[1] - 1) // 2,
(dert__.shape[2] - 1) // 2),
dtype=dert__.dtype)
new_dert__.mask = True
# extract new_dert__ 'views', use [:] to 'update' views and new_dert__ at the same time
i__center, idy__, idx__, g__, dy__, dx__, m__ = new_dert__
i__ = dert__[0] # i is ig if fig else pixel
'''
sparse aligned i__center and i__rim arrays:
'''
i__center[:] = i__[1:-1:2, 1:-1:2] # also assignment to new_dert__[0]
i__topleft = i__[:-2:2, :-2:2]
i__top = i__[:-2:2, 1:-1:2]
i__topright = i__[:-2:2, 2::2]
i__right = i__[1:-1:2, 2::2]
i__bottomright = i__[2::2, 2::2]
i__bottom = i__[2::2, 1:-1:2]
i__bottomleft = i__[2::2, :-2:2]
i__left = i__[1:-1:2, :-2:2]
'''
unmask all derts in kernels with only one masked dert (can be set to any number of masked derts),
to avoid extreme blob shrinking and loss of info in other derts of partially masked kernels
unmasked derts were computed due to extend_dert() in intra_blob
'''
majority_mask = (i__[:, 1:-1:2, 1:-1:2].mask.astype(int) +
i__[:, :-2:2, :-2:2].mask.astype(int) +
i__[:, :-2:2, 1:-1:2].mask.astype(int) +
i__[:, :-2:2, 2::2].mask.astype(int) +
i__[:, 1:-1:2, 2::2].mask.astype(int) +
i__[:, 2::2, 2::2].mask.astype(int) +
i__[:, 2::2, 1:-1:2].mask.astype(int) +
i__[:, 2::2, :-2:2].mask.astype(int) +
i__[:, 1:-1:2, :-2:2].mask.astype(int)) > 1
i__center.mask = i__topleft.mask = i__top.mask = i__topright.mask = i__right.mask = i__bottomright.mask = \
i__bottom.mask = i__bottomleft.mask = i__left.mask = majority_mask # not only i__center
idy__[:], idx__[:] = dert__[[1, 2], 1:-1:2, 1:-1:2]
if root_fcr: # root fork is comp_r, accumulate derivatives:
dy__[:] = dert__[4, 1:-1:2, 1:-1:2] # sparse to align with i__center
dx__[:] = dert__[5, 1:-1:2, 1:-1:2]
m__[:] = dert__[6, 1:-1:2, 1:-1:2]
dy__.mask = dx__.mask = m__.mask = majority_mask
if not fig: # compare four diametrically opposed pairs of rim pixels:
d_tl_br = i__topleft.data - i__bottomright.data
d_t_b = i__top.data - i__bottom.data
d_tr_bl = i__topright.data - i__bottomleft.data
d_r_l = i__right.data - i__left.data
dy__ += (d_tl_br * YCOEFs[0] + d_t_b * YCOEFs[1] +
d_tr_bl * YCOEFs[2] + d_r_l * YCOEFs[3])
dx__ += (d_tl_br * XCOEFs[0] + d_t_b * XCOEFs[1] +
d_tr_bl * XCOEFs[2] + d_r_l * XCOEFs[3])
g__[:] = ma.hypot(dy__, dx__) # gradient
'''
inverse match = SAD, direction-invariant and more precise measure of variation than g
(all diagonal derivatives can be imported from prior 2x2 comp)
'''
m__ += (abs(i__center.data - i__topleft.data) +
abs(i__center.data - i__top.data) +
abs(i__center.data - i__topright.data) +
abs(i__center.data - i__right.data) +
abs(i__center.data - i__bottomright.data) +
abs(i__center.data - i__bottom.data) +
abs(i__center.data - i__bottomleft.data) +
abs(i__center.data - i__left.data))
else: # fig is TRUE, compare angle and then magnitude of 8 center-rim pairs
i__[ma.where(i__ == 0)] = 1 # to avoid / 0
a__ = dert__[[1, 2]] / i__ # sin = idy / i, cos = idx / i, i = ig
'''
sparse aligned a__center and a__rim arrays:
'''
a__center = a__[:, 1:-1:2, 1:-1:2]
a__topleft = a__[:, :-2:2, :-2:2]
a__top = a__[:, :-2:2, 1:-1:2]
a__topright = a__[:, :-2:2, 2::2]
a__right = a__[:, 1:-1:2, 2::2]
a__bottomright = a__[:, 2::2, 2::2]
a__bottom = a__[:, 2::2, 1:-1:2]
a__bottomleft = a__[:, 2::2, :-2:2]
a__left = a__[:, 1:-1:2, :-2:2]
'''
only mask kernels with more than one masked dert, for all operations below:
'''
majority_mask_a = (a__[:, 1:-1:2, 1:-1:2].mask.astype(int) +
a__[:, :-2:2, :-2:2].mask.astype(int) +
a__[:, :-2:2, 1:-1:2].mask.astype(int) +
a__[:, :-2:2, 2::2].mask.astype(int) +
a__[:, 1:-1:2, 2::2].mask.astype(int) +
a__[:, 2::2, 2::2].mask.astype(int) +
a__[:, 2::2, 1:-1:2].mask.astype(int) +
a__[:, 2::2, :-2:2].mask.astype(int) +
a__[:, 1:-1:2, :-2:2].mask.astype(int)) > 1
a__center.mask = a__topleft.mask = a__top.mask = a__topright.mask = a__right.mask = a__bottomright.mask = \
a__bottom.mask = a__bottomleft.mask = a__left.mask = majority_mask_a
assert (majority_mask_a[0] == majority_mask_a[1]).all()
# what does that do?
dy__.mask = dx__.mask = m__.mask = majority_mask_a[0]
'''
8-tuple of differences between central dert angle and rim dert angle:
'''
cos_da = [((a__topleft[1].data * a__center[1].data) +
(a__center[0].data * a__topleft[0].data)),
((a__top[1].data * a__center[1].data) +
(a__center[0].data * a__top[0].data)),
((a__topright[1].data * a__center[1].data) +
(a__center[0].data * a__topright[0].data)),
((a__right[1].data * a__center[1].data) +
(a__center[0].data * a__right[0].data)),
((a__bottomright[1].data * a__center[1].data) +
(a__center[0].data * a__bottomright[0].data)),
((a__bottom[1].data * a__center[1].data) +
(a__center[0].data * a__bottom[0].data)),
((a__bottomleft[1].data * a__center[1].data) +
(a__center[0].data * a__bottomleft[0].data)),
((a__left[1].data * a__center[1].data) +
(a__center[0].data * a__left[0].data))]
'''
8-tuple of cosine matches per direction:
'''
m__ += (ma.minimum(i__center.data, i__topleft.data) * cos_da[0] +
ma.minimum(i__center.data, i__top.data) * cos_da[1] +
ma.minimum(i__center.data, i__topright.data) * cos_da[2] +
ma.minimum(i__center.data, i__right.data) * cos_da[3] +
ma.minimum(i__center.data, i__bottomright.data) * cos_da[4] +
ma.minimum(i__center.data, i__bottom.data) * cos_da[5] +
ma.minimum(i__center.data, i__bottomleft.data) * cos_da[6] +
ma.minimum(i__center.data, i__left.data) * cos_da[7])
'''
8-tuple of cosine differences per direction:
'''
dt__ = [(i__center.data - i__topleft.data * cos_da[0]),
(i__center.data - i__top.data * cos_da[1]),
(i__center.data - i__topright.data * cos_da[2]),
(i__center.data - i__right.data * cos_da[3]),
(i__center.data - i__bottomright.data * cos_da[4]),
(i__center.data - i__bottom.data * cos_da[5]),
(i__center.data - i__bottomleft.data * cos_da[6]),
(i__center.data - i__left.data * cos_da[7])]
for d__, YCOEF, XCOEF in zip(dt__, YCOEFs, XCOEFs):
dy__ += d__ * YCOEF # decompose differences into dy and dx,
dx__ += d__ * XCOEF # accumulate with prior-rng dy, dx
'''
accumulate in prior-range dy, dx: 3x3 -> 5x5 -> 9x9
'''
g__[:] = ma.hypot(dy__, dx__)
'''
next comp_r will use full dert
next comp_g will use g__, dy__, dx__
'''
return new_dert__ # new_dert__ has been updated along with 'view' arrays: i__center, idy__, idx__, g__, dy__, dx__, m__
