def _get_max_true_block_length(up_down):
"""
Calculate the maximum length of True blocks in an array.
The True blocks in the array are defined by up/down changes from False
to True and vice-versa.
Args:
* up_down (tuple of numpy.arrays):
information about the up/down changes
Returns:
numpy.array of same dimension of up_down[0] holding the number
of maximum of the True block lengths in the array
"""
out_shape = up_down[0].shape[:-1]
ret = ma.zeros(out_shape) - 1
ret.fill_value = -1
ret.mask = True
for index in np.ndindex(out_shape):
if not up_down[0][index].mask.any():
start_idxs = ma.where(up_down[0][index])
stop_idxs = ma.where(up_down[1][index])
try:
ret[index] = np.max(stop_idxs[-1] - start_idxs[-1] + 1)
except ValueError:
ret[index] = 0
return ret
def __getitem__(self, index):
data = self.arguments[0] if is_constant(
self.arguments[0]) else self.arguments[0][index]
above_val = self.keywords['above']
below_val = self.keywords['below']
if above_val is None and below_val is None:
return data
above_str = ''
if above_val is not None:
above_ind = where(data > above_val)
if len(above_ind) > 0:
data[above_ind] = above_val
above_str = ', above={}'.format(above_val)
below_str = ''
if below_val is not None:
below_ind = where(data < below_val)
if len(below_ind) > 0:
data[below_ind] = below_val
below_str = ', below={}'.format(below_val)
new_name = 'limit({}{}{})'.format(data.name, above_str, below_str)
return PhysArray(data, name=new_name)
def _get_len_true_block_length(up_down, length):
"""
Calculate the len of True blocks in an array that succeed the given length.
The True blocks in the array are defined by up/down changes from False
to True and vice-versa.
Args:
* up_down (tuple of numpy.arrays):
information about the up/down changes
* length (int or float):
threshold for the length of blocks to be accounted for
Returns:
numpy.array of same dimension of up_down[0] holding the number
of block lengths succeeding the given length
"""
out_shape = up_down[0].shape[:-1]
ret = ma.zeros(out_shape) - 1
ret.fill_value = -1
ret.mask = True
for index in np.ndindex(out_shape):
if not up_down[0][index].mask.any():
start_idxs = ma.where(up_down[0][index])
stop_idxs = ma.where(up_down[1][index])
try:
dry_blocks = stop_idxs[-1] - start_idxs[-1] + 1
ret[index] = np.where(dry_blocks > length)[0].shape[0]
except ValueError:
ret[index] = 0
return ret
def main_loop_lag_an(j, corr_matrix, area, clims, lat2):
print j
for i in range(nx):
corr_m = corr_matrix[j, i, :, :] / np.max(corr_matrix[j, i, :, :])
if np.max(corr_m):
if clims[j, i] == 1:
jinds0 = []
while len(jinds0) < 9:
jinds0, iinds0 = ma.where(
corr_m * abs(corr_m) >= clims[j, i])
clims[j, i] = clims[j, i] - 0.01
if clims[j, i] < 0.5:
break
else:
jinds0, iinds0 = ma.where(
corr_m * abs(corr_m) >= clims[j, i])
#jinds1,iinds1=ma.where(corr_m*abs(corr_m)<=0.8)
#if len(jinds0)==0:
# jinds0=jinds1; iinds0=iinds1
ind0 = ma.where(corr_matrix2[j, i, jinds0, iinds0] > 0)[0]
jinds0 = jinds0[ind0]
iinds0 = iinds0[ind0]
#sum over all the grid cells
area[j, i] = np.sum(
(6371E3 * 0.25 * np.pi / 180.) *
(6371E3 * np.cos(lat2[jinds0, iinds0] * np.pi / 180.) *
0.25 * np.pi / 180.))
def _get_max_true_block_length(up_down):
"""
Calculate the maximum length of True blocks in an array.
The True blocks in the array are defined by up/down changes from False
to True and vice-versa.
Args:
* up_down (tuple of numpy.arrays):
information about the up/down changes
Returns:
numpy.array of same dimension of up_down[0] holding the number
of maximum of the True block lengths in the array
"""
out_shape = up_down[0].shape[:-1]
ret = ma.zeros(out_shape) - 1
ret.fill_value = -1
ret.mask = True
for index in np.ndindex(out_shape):
if not up_down[0][index].mask.any():
start_idxs = ma.where(up_down[0][index])
stop_idxs = ma.where(up_down[1][index])
try:
ret[index] = np.max(stop_idxs[-1] - start_idxs[-1] + 1)
except ValueError:
ret[index] = 0
return ret
def _get_len_true_block_length(up_down, length):
"""
Calculate the len of True blocks in an array that succeed the given length.
The True blocks in the array are defined by up/down changes from False
to True and vice-versa.
Args:
* up_down (tuple of numpy.arrays):
information about the up/down changes
* length (int or float):
threshold for the length of blocks to be accounted for
Returns:
numpy.array of same dimension of up_down[0] holding the number
of block lengths succeeding the given length
"""
out_shape = up_down[0].shape[:-1]
ret = ma.zeros(out_shape) - 1
ret.fill_value = -1
ret.mask = True
for index in np.ndindex(out_shape):
if not up_down[0][index].mask.any():
start_idxs = ma.where(up_down[0][index])
stop_idxs = ma.where(up_down[1][index])
try:
dry_blocks = stop_idxs[-1] - start_idxs[-1] + 1
ret[index] = np.where(dry_blocks > length)[0].shape[0]
except ValueError:
ret[index] = 0
return ret
def check_mask_vs_fitorder(firstNan, order):
"""
checks what columns can be fitted with what order of polynomial
"""
# needs to be list as they could have different lengths
fitColumnIndex = []
# 3D mask that contains mask for each highest order possible
mask = np.ones((order + 1, firstNan.shape[0]), dtype=np.int8)
# make firstNan masked array so we can disregard used ones
firstNan = ma.array(firstNan, mask=np.zeros(firstNan.shape))
# are there any series where number of data points is
# not sufficient for order of polynom?
for i in range(order, 0, -1):
# can do full order
tmp = np.int32(ma.where(firstNan > i)[0])
fitColumnIndex.append(tmp)
# mask the used values
firstNan.mask[tmp] = 1
# for the overall mask we want 0 for the values we can use
# for this order
mask[order - i, tmp] = 0
# remaining columns have one or less data points
tmp = np.int32(ma.where(firstNan <= i)[0])
fitColumnIndex.append(tmp)
# for the overall mask we want 0 for the values we can use
# for this order
mask[-1, tmp] = 0
return fitColumnIndex, mask
def generic_interp_pres(p, pres, field):
'''
Generic interpolation routine
Parameters
----------
p : number, numpy array
Pressure (hPa) of the level for which the field variable is desired
pres : numpy array
The array of pressure
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given pressure : number, numpy array
'''
if field.count() == 0 or pres.count() == 0:
return ma.masked
if ma.isMaskedArray(pres):
not_masked1 = ~pres.mask * np.ones(pres.shape, dtype=bool)
else:
not_masked1 = np.ones(pres.shape, dtype=bool)
not_masked1[:] = True
if ma.isMaskedArray(field):
not_masked2 = ~field.mask * np.ones(pres.shape, dtype=bool)
else:
not_masked2 = np.ones(field.shape, dtype=bool)
not_masked2[:] = True
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(p,
pres[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
if hasattr(p, 'shape') and p.shape == tuple():
p = p[()]
if type(p) != type(ma.masked) and np.all(~np.isnan(p)):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = ma.where(np.isclose(p, pres[not_masked][0]),
field[not_masked][0], field_intrp)
field_intrp = ma.where(np.isclose(p, pres[not_masked][-1]),
field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
return field_intrp
def log_linear_vinterp(T,P,levs):
'''
# Author Charles Doutriaux
# Version 1.1
# Expect 2D field here so there''s no reorder which I suspect to do a memory leak
# email: [email protected]
# Converts a field from sigma levels to pressure levels
# Log linear interpolation
# Input
# T : temperature on sigma levels
# P : pressure field from TOP (level 0) to BOTTOM (last level)
# levs : pressure levels to interplate to (same units as P)
# Output
# t : temperature on pressure levels (levs)
# External: Numeric'''
import numpy.ma as MA
## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16
sh=P.shape
nsigma=sh[0] # Number of sigma levels
try:
nlev=len(levs) # Number of pressure levels
except:
nlev=1 # if only one level len(levs) would breaks
t=[]
for ilv in range(nlev): # loop through pressure levels
try:
lev=levs[ilv] # get value for the level
except:
lev=levs # only 1 level passed
# print ' ......... level:',lev
Pabv=MA.ones(P[0].shape,Numeric.Float)
Tabv=-Pabv # Temperature on sigma level Above
Tbel=-Pabv # Temperature on sigma level Below
Pbel=-Pabv # Pressure on sigma level Below
Pabv=-Pabv # Pressure on sigma level Above
for isg in range(1,nsigma): # loop from second sigma level to last one
## print 'Sigma level #',isg
a = MA.greater(P[isg], lev) # Where is the pressure greater than lev
b = MA.less(P[isg-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Tabv, Tbel
Pabv=MA.where(MA.logical_and(a,b),P[isg],Pabv) # Pressure on sigma level Above
Tabv=MA.where(MA.logical_and(a,b),T[isg],Tabv) # Temperature on sigma level Above
Pbel=MA.where(MA.logical_and(a,b),P[isg-1],Pbel) # Pressure on sigma level Below
Tbel=MA.where(MA.logical_and(a,b),T[isg-1],Tbel) # Temperature on sigma level Below
# end of for isg in range(1,nsigma)
# val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is
tl=MA.masked_where(MA.equal(Pbel,-1.),MA.log(lev/MA.absolute(Pbel))/MA.log(Pabv/Pbel)*(Tabv-Tbel)+Tbel) # Interpolation
t.append(tl) # add a level to the output
# end of for ilv in range(nlev)
return asMA(t).astype(Numeric.Float32) # convert t to an array
def get_month_concSN(datapath,
year,
month,
alg=0,
pole='AA',
mask=1,
maxConc=0,
lowerConc=0):
""" Get monthly ice concentration from the NSIDC"""
if (alg == 0):
team = 'NASA_TEAM'
team_s = 'nt'
header = 300
datatype = 'uint8'
scale_factor = 250.
if (alg == 1):
team = 'BOOTSTRAP'
team_s = 'bt'
header = 0
datatype = '<i2'
scale_factor = 1000.
if (pole == 'AA'):
poleStr = 'ANTARCTIC'
rows = 332
cols = 316
if (pole == 'A'):
poleStr = 'ARCTIC'
rows = 448
cols = 304
month_str = '%02d' % (month + 1)
year_str = str(year)
files = glob(datapath + '/ICE_CONC/' + team + '/' + poleStr + '/monthly/' +
team_s + '_' + year_str + month_str + '*.bin')
fd = open(files[-1], 'r')
data = fromfile(file=fd, dtype=datatype)
data = data[header:]
#FIRST 300 FILES ARE HEADER INFO
ice_conc = reshape(data, [rows, cols])
#divide by 250 to express in concentration
ice_conc = ma.masked_where(ice_conc > 250., ice_conc)
ice_conc = ice_conc / scale_factor
#GREATER THAN 250 is mask/land etc
if (mask == 1):
ice_conc = ma.masked_where(ice_conc > 1., ice_conc)
if (maxConc == 1):
ice_conc = ma.where(ice_conc > 1., 0, ice_conc)
if (lowerConc == 1):
ice_conc = ma.where(ice_conc < 0.15, 0, ice_conc)
return ice_conc
def getdata(imgname, chans=[], zeromask=False):
"""Return all good data from a CASA image as a masked numpy array.
The tb.open() access method, although faster, didn't seem to carry
the mask (or at least in an easy way). ma.masked_invalid(data)
still returned all pixels ok.
NOTE: since this routine grabs all data in a single numpy
array, this routine should only be used for 2D images
or small 3D cubes, things with little impact on memory
Parameters
----------
imagename : str
The (absolute) CASA image filename
Returns
-------
array
data in a masked numpy array
"""
taskinit.ia.open(imgname)
if len(chans) == 0:
d = taskinit.ia.getchunk(blc=[0, 0, 0, 0],
trc=[-1, -1, -1, 0],
getmask=False).squeeze()
m = taskinit.ia.getchunk(blc=[0, 0, 0, 0],
trc=[-1, -1, -1, 0],
getmask=True).squeeze()
else:
d = taskinit.ia.getchunk(blc=[0, 0, chans[0], 0],
trc=[-1, -1, chans[1], 0],
getmask=False).squeeze()
m = taskinit.ia.getchunk(blc=[0, 0, chans[0], 0],
trc=[-1, -1, chans[1], 0],
getmask=True).squeeze()
taskinit.ia.close()
# note CASA and MA have their mask logic reversed
# casa: true means a good point
# ma: true means a masked/bad point
if zeromask:
shape = d.shape
ndim = len(d.shape)
if ndim == 2:
(x, y) = ma.where(~m)
d[x, y] = 0.0
elif ndim == 3:
(x, y, z) = ma.where(~m)
d[x, y, z] = 0.0
else:
raise Exception, "getdata: cannot handle data of dimension %d" % ndim
dm = ma.masked_array(d, mask=~m)
return dm
def test_testMinMax2(self):
# Test of minimum, maximum.
assert_(eq(minimum([1, 2, 3], [4, 0, 9]), [1, 0, 3]))
assert_(eq(maximum([1, 2, 3], [4, 0, 9]), [4, 2, 9]))
x = arange(5)
y = arange(5) - 2
x[3] = masked
y[0] = masked
assert_(eq(minimum(x, y), where(less(x, y), x, y)))
assert_(eq(maximum(x, y), where(greater(x, y), x, y)))
assert_(minimum.reduce(x) == 0)
assert_(maximum.reduce(x) == 4)
def getExtentAreaFromConc(iceConcMon):
iceConcMon = ma.masked_where(iceConcMon <= 0.15, iceConcMon)
iceConcMon = ma.masked_where(ice_flag >= 1.5, iceConcMon)
concHole = ma.mean(iceConcMon[(lats > pmask - 0.5) & (lats < pmask)])
iceConcMonP = ma.where((lats >= pmask), 1., iceConcMon)
iceConcMonA = ma.where((lats >= pmask), concHole, iceConcMon)
iceExtent = ma.sum(ma.where((iceConcMonP > 0.15), 1, 0) * areaF)
iceArea = ma.sum(iceConcMonA * areaF)
return iceExtent, iceArea
def test_testMinMax2(self):
# Test of minimum, maximum.
assert_(eq(minimum([1, 2, 3], [4, 0, 9]), [1, 0, 3]))
assert_(eq(maximum([1, 2, 3], [4, 0, 9]), [4, 2, 9]))
x = arange(5)
y = arange(5) - 2
x[3] = masked
y[0] = masked
assert_(eq(minimum(x, y), where(less(x, y), x, y)))
assert_(eq(maximum(x, y), where(greater(x, y), x, y)))
assert_(minimum.reduce(x) == 0)
assert_(maximum.reduce(x) == 4)
def _h_arrows(self, length):
""" length is in arrow width units """
# It might be possible to streamline the code
# and speed it up a bit by using complex (x,y)
# instead of separate arrays; but any gain would be slight.
minsh = self.minshaft * self.headlength
N = len(length)
length = length.reshape(N, 1)
# x, y: normal horizontal arrow
x = np.array([0, -self.headaxislength, -self.headlength, 0],
np.float64)
x = x + np.array([0, 1, 1, 1]) * length
y = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
y = np.repeat(y[np.newaxis, :], N, axis=0)
# x0, y0: arrow without shaft, for short vectors
x0 = np.array(
[0, minsh - self.headaxislength, minsh - self.headlength, minsh],
np.float64)
y0 = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
ii = [0, 1, 2, 3, 2, 1, 0]
X = x.take(ii, 1)
Y = y.take(ii, 1)
Y[:, 3:] *= -1
X0 = x0.take(ii)
Y0 = y0.take(ii)
Y0[3:] *= -1
shrink = length / minsh
X0 = shrink * X0[np.newaxis, :]
Y0 = shrink * Y0[np.newaxis, :]
short = np.repeat(length < minsh, 7, axis=1)
#print 'short', length < minsh
# Now select X0, Y0 if short, otherwise X, Y
X = ma.where(short, X0, X)
Y = ma.where(short, Y0, Y)
if self.pivot[:3] == 'mid':
X -= 0.5 * X[:, 3, np.newaxis]
elif self.pivot[:3] == 'tip':
X = X - X[:, 3, np.newaxis] #numpy bug? using -= does not
# work here unless we multiply
# by a float first, as with 'mid'.
tooshort = length < self.minlength
if tooshort.any():
# Use a heptagonal dot:
th = np.arange(0, 7, 1, np.float64) * (np.pi / 3.0)
x1 = np.cos(th) * self.minlength * 0.5
y1 = np.sin(th) * self.minlength * 0.5
X1 = np.repeat(x1[np.newaxis, :], N, axis=0)
Y1 = np.repeat(y1[np.newaxis, :], N, axis=0)
tooshort = ma.repeat(tooshort, 7, 1)
X = ma.where(tooshort, X1, X)
Y = ma.where(tooshort, Y1, Y)
return X, Y
def generic_interp_pres(p, pres, field):
'''
Generic interpolation routine
Parameters
----------
p : number, numpy array
Pressure (hPa) of the level for which the field variable is desired
pres : numpy array
The array of pressure
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given pressure
'''
if ma.isMaskedArray(pres):
not_masked1 = ~pres.mask
else:
not_masked1 = np.ones(pres.shape, dtype=bool)
not_masked1[:] = True
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape, dtype=bool)
not_masked2[:] = True
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(p, pres[not_masked], field[not_masked], left=ma.masked,
right=ma.masked)
if hasattr(p, 'shape') and p.shape == tuple():
p = p[()]
if type(p) != type(ma.masked):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = ma.where(np.isclose(p, pres[not_masked][0]), field[not_masked][0], field_intrp)
field_intrp = ma.where(np.isclose(p, pres[not_masked][-1]), field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
return field_intrp
def _h_arrows(self, length):
""" length is in arrow width units """
# It might be possible to streamline the code
# and speed it up a bit by using complex (x,y)
# instead of separate arrays; but any gain would be slight.
minsh = self.minshaft * self.headlength
N = len(length)
length = length.reshape(N, 1)
# x, y: normal horizontal arrow
x = np.array([0, -self.headaxislength,
-self.headlength, 0], np.float64)
x = x + np.array([0,1,1,1]) * length
y = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
y = np.repeat(y[np.newaxis,:], N, axis=0)
# x0, y0: arrow without shaft, for short vectors
x0 = np.array([0, minsh-self.headaxislength,
minsh-self.headlength, minsh], np.float64)
y0 = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
ii = [0,1,2,3,2,1,0]
X = x.take(ii, 1)
Y = y.take(ii, 1)
Y[:, 3:] *= -1
X0 = x0.take(ii)
Y0 = y0.take(ii)
Y0[3:] *= -1
shrink = length/minsh
X0 = shrink * X0[np.newaxis,:]
Y0 = shrink * Y0[np.newaxis,:]
short = np.repeat(length < minsh, 7, axis=1)
#print 'short', length < minsh
# Now select X0, Y0 if short, otherwise X, Y
X = ma.where(short, X0, X)
Y = ma.where(short, Y0, Y)
if self.pivot[:3] == 'mid':
X -= 0.5 * X[:,3, np.newaxis]
elif self.pivot[:3] == 'tip':
X = X - X[:,3, np.newaxis] #numpy bug? using -= does not
# work here unless we multiply
# by a float first, as with 'mid'.
tooshort = length < self.minlength
if tooshort.any():
# Use a heptagonal dot:
th = np.arange(0,7,1, np.float64) * (np.pi/3.0)
x1 = np.cos(th) * self.minlength * 0.5
y1 = np.sin(th) * self.minlength * 0.5
X1 = np.repeat(x1[np.newaxis, :], N, axis=0)
Y1 = np.repeat(y1[np.newaxis, :], N, axis=0)
tooshort = ma.repeat(tooshort, 7, 1)
X = ma.where(tooshort, X1, X)
Y = ma.where(tooshort, Y1, Y)
return X, Y
def generic_interp_pres(p, pres, field):
'''
Generic interpolation routine
Parameters
----------
p : number, numpy array
Pressure (hPa) of the level for which the field variable is desired
pres : numpy array
The array of pressure
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given pressure
'''
if ma.isMaskedArray(pres):
not_masked1 = ~pres.mask
else:
not_masked1 = np.ones(pres.shape, dtype=bool)
not_masked1[:] = True
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape, dtype=bool)
not_masked2[:] = True
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(p,
pres[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
if type(p) != type(ma.masked):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = ma.where(np.isclose(p, pres[not_masked][0]),
field[not_masked][0], field_intrp)
field_intrp = ma.where(np.isclose(p, pres[not_masked][-1]),
field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
return field_intrp
def __fillMissingScenarios(self):
nv, nt, _, _ = self.data.shape
for i in range(nv):
for j in range(nt):
badscens = where(self.data[i, j].mask.all(axis=1))[0]
goodscens = where(~self.data[i, j].mask.all(axis=1))[0]
if goodscens.size:
for k in range(len(badscens)):
idx = where(goodscens <= badscens[k])[0]
idx = idx[-1] if idx.size else 0
self.data[i, j,
badscens[k]] = self.data[i, j,
goodscens[idx]]
def sort_points(self, event):
"""
Sort points starting from first point by least distance to previous point.
Parameters
----------
event : A :py:class:`traits.observation.events.TraitChangeEvent` instance
The trait change handler for sort_points_button.
"""
if len(self.points) > 0:
points = [val if val else ArrayClass() for val in self.points]
distances = np.zeros((len(points), len(points)))
for i, point1 in enumerate(points):
for j, point2 in enumerate(points):
distances[i,
j] = np.linalg.norm(point1.value - point2.value)
point_order = [0]
while len(point_order) < len(points):
masked = ma.array(distances[point_order[-1], :])
masked[point_order] = ma.masked
point_order.extend(ma.where(masked == masked.min())[0])
tmp_points = [points[i] for i in point_order]
self.points = tmp_points
def create_mask(flag_file):
flag_file = np.int64(flag_file)
flag_mask = ma.zeros(flag_file.shape)
flag_mask[ma.where((flag_file & INVALID)\
| (flag_file & LAND)\
| (flag_file & CLOUD)\
| (flag_file & CLOUD_AMBIGUOUS)\
| (flag_file & CLOUD_MARGIN)\
| (flag_file & SNOW_ICE)\
| (flag_file & SUSPECT)\
# | (flag_file & HISOLZEN)\
| (flag_file & SATURATED)\
| (flag_file & HIGHGLINT)\
| (flag_file & WHITECAPS)\
| (flag_file & AC_FAIL)\
# | (flag_file & RWNEG_O2)\
# | (flag_file & RWNEG_O3)\
# | (flag_file & RWNEG_O4)\
# | (flag_file & RWNEG_O5)\
# | (flag_file & RWNEG_O6)\
# | (flag_file & RWNEG_O7)\
# | (flag_file & RWNEG_O8)\
)]=1
flag_mask = np.int64(flag_mask)
return flag_mask
def get_pixels_sorted_by_distance(data, mask, dtype, tdef, tol):
# get pixel positions of peak of the clump
#
peak = np.unravel_index(np.argmax(data, axis=None), data.shape)
# get positions of pixels in the clump
#
clump_pixel = ma.where(mask)
num_pixels = np.shape(clump_pixel)[1]
if data[peak[0], peak[1]] < 0:
data[peak[0], peak[1]] = tdef
idx = 0
position_and_distance = []
for i in range(num_pixels):
x, y = (clump_pixel[0][i], clump_pixel[1][i])
dx = peak[0] - x
dy = peak[1] - y
d = np.sqrt(dx * dx + dy * dy)
if d < tol:
continue
position_and_distance.append((x, y, d))
idx += 1
pd_arr = np.array(position_and_distance, dtype)
return np.sort(pd_arr, order='dist')
def areas(self, var, agg, lats, weights = None, calcarea = False, mask = None):
nt, nlats, nlons = var.shape
if weights is None: # weights
weights = ones((nt, nlats, nlons))
elif len(weights.shape) == 2:
weights = ma.resize(weights, (nt, nlats, nlons))
if calcarea: # area
area = self.area(lats, nlats, nlons)
else:
area = ones((nlats, nlons))
aggvals = self.__uniquevals(agg)
sz = len(aggvals)
varmask = logical_not(var.mask) if ma.isMaskedArray(var) else ones(var.shape) # use variable mask
if not mask is None: varmask = logical_and(varmask, mask) # additional mask
areas = ma.masked_array(zeros((sz, nt)), mask = ones((sz, nt)))
vartmp = zeros((nt, nlats, nlons))
for i in range(len(aggvals)):
warea = weights * area * (agg == aggvals[i])
tidx, latidx, lonidx = ma.where(warea)
vartmp[:] = 0
vartmp[tidx, latidx, lonidx] = warea[tidx, latidx, lonidx] * \
varmask[tidx, latidx, lonidx]
areas[i] = vartmp.sum(axis = 2).sum(axis = 1)
areas = ma.masked_where(areas == 0, areas)
areas.mask = resize(areas.mask, areas.shape) # ensure mask is same size as data
return areas
def correct_liquid_top(obs: ClassData,
liquid: dict,
is_freezing: np.ndarray,
limit: float = 200) -> np.ndarray:
"""Corrects lidar detected liquid cloud top using radar data.
Args:
obs: The :class:`ClassData` instance.
liquid: Dictionary about liquid clouds including `tops` and `presence`.
is_freezing: 2-D boolean array of sub-zero temperature, derived from the model
temperature and melting layer based on radar data.
limit: The maximum correction distance (m) above liquid cloud top.
Returns:
Corrected liquid cloud array.
References:
Hogan R. and O'Connor E., 2004, https://bit.ly/2Yjz9DZ.
"""
is_liquid_corrected = np.copy(liquid["presence"])
top_above = utils.n_elements(obs.height, limit)
for prof, top in zip(*np.where(liquid["tops"])):
ind = _find_ind_above_top(is_freezing[prof, top:], top_above)
rad = obs.z[prof, top:top + ind + 1]
if not (rad.mask.all() or ~rad.mask.any()):
first_masked = ma.where(rad.mask)[0][0]
is_liquid_corrected[prof, top:top + first_masked] = True
return is_liquid_corrected
def main():
if len(sys.argv) != 3:
print("Usage: %s <scenario> <directory>" %
os.path.basename(sys.argv[0]))
sys.exit(1)
title = 'Biodiversity Projection'
oname = "%s.mp4" % sys.argv[1]
files = sorted(
filter(lambda x: re.match(r'%s-\d{4}.tif$' % sys.argv[1], x),
os.listdir(sys.argv[2])))
years = map(lambda x: re.sub(r'%s-(\d{4,4}).tif' % sys.argv[1], '\\1', x),
files)
ds = rasterio.open(os.path.join(sys.argv[2], files[0]))
data = np.empty((len(files), ds.shape[0], ds.shape[1]))
for idx, f in enumerate(files):
ds = rasterio.open(os.path.join(sys.argv[2], files[idx]))
dd = ds.read(1, masked=True)
data[idx] = np.exp(ma.where(np.isnan(dd), 1, dd))
db = 10 * np.log(data / (data[0] + 0.01))
cnorm = colors.Normalize(vmin=-3, vmax=3)
for idx, img, text in to_mp4(title,
oname,
len(years),
dd,
years[0],
10,
cnorm=cnorm):
img.set_array(db[idx])
text.set_text(years[idx])
def stats(op, infiles, band, log, area_weighted):
whats, years = get_domain(infiles)
df = pd.DataFrame(columns=whats, index=sorted(years))
for arg in infiles:
with rasterio.open(arg) as src:
data = src.read(band, masked=True)
if log:
data = ma.exp(data)
if area_weighted:
data *= rcs(data.shape[0], src.res, *src.bounds)
data.mask = np.logical_or(data.mask,
ma.where(np.isnan(data), True, False))
if op == 'sum':
op = 'total'
res = eval("%s(data)" % op)
if re.search(r'-hpd-(\d){4}.tif', arg):
print('%s: %8.4f %8.4f' %
(os.path.basename(arg), res, np.log(res + 1) / 10.02083))
else:
print('%s: %8.4f' % (os.path.basename(arg), res))
scenario, what, year = os.path.splitext(arg)[0].split('-')
df.ix[int(year), what] = res
print(df)
df.plot.bar()
plt.savefig('lu-comp.png')
plt.show()
def do_block(win, mask, index):
pdb.set_trace()
xres, yres = mask.res
affine = mask.window_transform(win)
mask_data = mask.read(1, masked=True, window=win)
out = ma.empty(mask_data.shape, dtype=np.float32)
out.mask = mask_data.mask.copy()
height, width = out.shape
startx, starty = affine * (win[1][0] - xres / 2.0, win[0][0] - yres / 2.0)
endx, endy = affine * (win[1][1] - xres / 2.0, win[0][1] - yres / 2.0)
click.echo("block %d:%d" % (win[0][0], win[0][1]))
lats = np.linspace(starty, endy, height)
lons = np.linspace(startx, endx, width)
for y in range(height):
if y > 10:
break
click.echo("block %d" % (y + win[0][0]))
lat = lats[y]
lat_min = lat - yres / 2.0
lat_max = lat + yres / 2.0
for lon in lons[ma.where(mask_data.mask[y, :] != True)]:
bbox = (lon - xres / 2.0, lat_min, lon + xres / 2.0, lat_max)
length = 0
for obj in index.intersection(bbox, objects='raw'):
length += do_intersect(bbox, obj)
# end x for loop
#out[y, x] = length
return out
def correct_liquid_top(obs, liquid, is_freezing, limit=200):
"""Corrects lidar detected liquid cloud top using radar data.
Args:
obs (ClassData): The :class:`ClassData` instance.
liquid (dict): Dictionary about liquid clouds including `tops` and
`presence`.
is_freezing (ndarray): 2-D boolean array of sub-zero temperature,
derived from the model temperature and melting layer based
on radar data.
limit (float): The maximum correction distance (m) above liquid cloud top.
Returns:
ndarray: Corrected liquid cloud array.
"""
is_liquid_corrected = np.copy(liquid['presence'])
top_above = utils.n_elements(obs.height, limit)
for prof, top in zip(*np.where(liquid['tops'])):
ind = _find_ind_above_top(is_freezing[prof, top:], top_above)
rad = obs.z[prof, top:top + ind + 1]
if not (rad.mask.all() or ~rad.mask.any()):
first_masked = ma.where(rad.mask)[0][0]
is_liquid_corrected[prof, top:top + first_masked] = True
return is_liquid_corrected
def load_files(filepath, fnames):
#load files given filepath and list of filenames#
dataout = {}
for e, filename in enumerate(fnames):
#print filename
datain = np.load(filepath + filename)
if e == 0:
variables = datain.keys()
variables.sort()
lmask = np.zeros(datain['V_global'].shape)
lmask[ma.where(datain['V_global'][:] == 0)] = 1
for var in variables:
if var in [
'Kx_global', 'Ky_global', 'R_global', 'U_global',
'V_global'
]:
if e == 0:
exec('dataout["' + var + '"]=ma.masked_array(datain["' +
var + '"][:],mask=lmask)')
elif e > 0:
exec('dataout["' + var + '"]=ma.concatenate([dataout["' +
var + '"],ma.masked_array(datain["' + var +
'"][:],mask=lmask)],axis=0)')
else:
if e == 0:
exec('dataout["' + var + '"]=datain["' + var + '"][:]')
elif e > 0:
exec('dataout["' + var + '"]=ma.concatenate([dataout["' +
var + '"],datain["' + var + '"][:]],axis=0)')
#
return dataout
def draw_bar(ax, show_first_operation, x, y, time, init_time, draw_color):
assert ax is not None
if y is None:
if show_first_operation is True:
mask_time = ma.where(time >= init_time)
mask_init_time = ma.where(init_time >= time)
ax.bar(x[mask_time],
time[mask_time],
width=bar_width,
color=draw_color)
ax.bar(x, init_time, width=bar_width, color=first_operation_color)
ax.bar(x[mask_init_time],
time[mask_init_time],
width=bar_width,
color=draw_color)
else:
ax.bar(x, time, width=bar_width, color=draw_color)
else:
if show_first_operation is True:
mask_time = ma.where(time >= init_time)
mask_init_time = ma.where(init_time >= time)
ax.bar(x[mask_time], (time - init_time)[mask_time],
y[mask_time],
bottom=init_time[mask_time],
zdir='y',
width=bar_width,
color=draw_color)
ax.bar(x[mask_init_time], (init_time - time)[mask_init_time],
y[mask_init_time],
bottom=time[mask_init_time],
zdir='y',
width=bar_width,
color=first_operation_color)
ax.bar(x[mask_init_time],
time[mask_init_time],
y[mask_init_time],
zdir='y',
width=bar_width,
color=draw_color)
ax.bar(x[mask_time],
init_time[mask_time],
y[mask_time],
zdir='y',
width=bar_width,
color=first_operation_color)
else:
ax.bar(x, time, y, zdir='y', width=bar_width, color=draw_color)
def getchanindex(self, chan):
"""
"""
if ma.max(self._chans) >= chan >= ma.min(self._chans):
return ma.where(self._chans == int(chan))[0][0]
else:
return -1
def transform_non_affine(self, a):
sign = np.sign(a)
masked = ma.masked_inside(a, -self.invlinthresh, self.invlinthresh, copy=False)
exp = sign * self.linthresh * (ma.power(self.base, (sign * (masked / self.linthresh)) - self._linscale_adj))
if masked.mask.any():
return ma.where(masked.mask, a / self._linscale_adj, exp)
else:
return exp
def transform_non_affine(self, a):
sign = np.sign(a)
masked = ma.masked_inside(a, -self.linthresh, self.linthresh, copy=False)
log = sign * self.linthresh * (self._linscale_adj + ma.log(np.abs(masked) / self.linthresh) / self._log_base)
if masked.mask.any():
return ma.where(masked.mask, a * self._linscale_adj, log)
else:
return log
def transform(self, a):
sign = np.sign(np.asarray(a))
masked = ma.masked_inside(a, -self.linthresh, self.linthresh, copy=False)
log = sign * ma.log(np.abs(masked)) / self._log_base
if masked.mask.any():
return np.asarray(ma.where(masked.mask, a * self._linadjust, log))
else:
return np.asarray(log)
def transform(self, a):
sign = np.sign(a)
masked = ma.masked_inside(a, -self.linthresh, self.linthresh, copy=False)
log = sign * self.linthresh * (1 + ma.log(np.abs(masked) / self.linthresh))
if masked.mask.any():
return ma.where(masked.mask, a, log)
else:
return log
def getchanindex(self, chan):
"""
"""
if ma.max(self._chans) >= chan >= ma.min(self._chans):
return ma.where(self._chans == int(chan))[0][0]
else:
return -1
def transform(self, a):
sign = np.sign(a)
masked = ma.masked_inside(a, -self.linthresh, self.linthresh, copy=False)
exp = sign * self.linthresh * ma.exp(sign * masked / self.linthresh - 1)
if masked.mask.any():
return ma.where(masked.mask, a, exp)
else:
return exp
def getdata(imgname, chans=[], zeromask=False):
"""Return all good data from a CASA image as a masked numpy array.
The tb.open() access method, although faster, didn't seem to carry
the mask (or at least in an easy way). ma.masked_invalid(data)
still returned all pixels ok.
NOTE: since this routine grabs all data in a single numpy
array, this routine should only be used for 2D images
or small 3D cubes, things with little impact on memory
Parameters
----------
imagename : str
The (absolute) CASA image filename
Returns
-------
array
data in a masked numpy array
"""
taskinit.ia.open(imgname)
if len(chans) == 0:
d = taskinit.ia.getchunk(blc=[0,0,0,0],trc=[-1,-1,-1,0],getmask=False).squeeze()
m = taskinit.ia.getchunk(blc=[0,0,0,0],trc=[-1,-1,-1,0],getmask=True).squeeze()
else:
d = taskinit.ia.getchunk(blc=[0,0,chans[0],0],trc=[-1,-1,chans[1],0],getmask=False).squeeze()
m = taskinit.ia.getchunk(blc=[0,0,chans[0],0],trc=[-1,-1,chans[1],0],getmask=True).squeeze()
taskinit.ia.close()
# note CASA and MA have their mask logic reversed
# casa: true means a good point
# ma: true means a masked/bad point
if zeromask:
shape = d.shape
ndim = len(d.shape)
if ndim == 2:
(x,y) = ma.where(~m)
d[x,y] = 0.0
elif ndim == 3:
(x,y,z) = ma.where(~m)
d[x,y,z] = 0.0
else:
raise Exception,"getdata: cannot handle data of dimension %d" % ndim
dm = ma.masked_array(d,mask=~m)
return dm
def get_eval_stats(outputs, labels):
silence_out = ma.array(outputs,
mask=list(
map(lambda e: True
if e == 1 else False, labels)))
voice_out = ma.array(outputs,
mask=list(
map(lambda e: True if e == 0 else False, labels)))
# labels are 0 on voiceless intervals
hits_s = ma.where(silence_out < err_threshold)[0].size
misses_s = ma.where(silence_out > err_threshold)[0].size
score_s = hits_s / misses_s
# labels are exactly 1 on voice intervals
hits_v = ma.where(voice_out > err_threshold)[0].size
misses_v = ma.where(voice_out < err_threshold)[0].size
score_v = hits_v / misses_v
return EvalStats(silence_out.count(), voice_out.count(), hits_s, misses_s,
score_s, hits_v, misses_v, score_v)
def generic_interp_hght(h, hght, field, log=False):
'''
Generic interpolation routine
Parameters
----------
h : number, numpy array
Height (m) of the level for which pressure is desired
hght : numpy array
The array of heights
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given height : number, numpy array
'''
if field.count() == 0 or hght.count() == 0:
return ma.masked_where(ma.ones(np.shape(h)), h) # JTS
if ma.isMaskedArray(hght):
# Multiplying by ones ensures that the result is an array, not a single value ... which
# happens sometimes ... >.<
not_masked1 = ~hght.mask * np.ones(hght.shape, dtype=bool)
else:
not_masked1 = np.ones(hght.shape)
if ma.isMaskedArray(field):
not_masked2 = ~field.mask * np.ones(field.shape, dtype=bool)
else:
not_masked2 = np.ones(field.shape)
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(h,
hght[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
if hasattr(h, 'shape') and h.shape == tuple():
h = h[()]
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
if log:
return 10**field_intrp
else:
return field_intrp
def transform(self, a):
a = np.asarray(a)
sign = np.sign(a)
masked = ma.masked_inside(a, -self.linthresh, self.linthresh, copy=False)
if masked.mask.any():
log = sign * (ma.log(np.abs(masked)) / self._log_base + self._linadjust)
return np.asarray(ma.where(masked.mask, a * self._linscale, log))
else:
return sign * (np.log(np.abs(a)) / self._log_base + self._linadjust)
def comp_g_old(
dert__
): # cross-comp of g in 2x2 kernels, between derts in ma.stack dert__
g__, dy__, dx__ = dert__[[3, 4, 5]] # g, dy, dx -> local i, idy, idx
g__[ma.where(
g__ == 0
)] = 1 # replace 0 values with 1 to avoid error, not needed in high-g blobs?
g0__, dy0__, dx0__ = g__[:-1, :-1], dy__[:-1, :-1], dx__[:-1, :
-1] # top left
g1__, dy1__, dx1__ = g__[:-1, 1:], dy__[:-1, 1:], dx__[:-1,
1:] # top right
g2__, dy2__, dx2__ = g__[1:, 1:], dy__[1:, 1:], dx__[1:,
1:] # bottom right
g3__, dy3__, dx3__ = g__[1:, :-1], dy__[1:, :-1], dx__[1:, :
-1] # bottom left
sin0__ = dy0__ / g0__
cos0__ = dx0__ / g0__
sin1__ = dy1__ / g1__
cos1__ = dx1__ / g1__
sin2__ = dy2__ / g2__
cos2__ = dx2__ / g2__
sin3__ = dy3__ / g3__
cos3__ = dx3__ / g3__
'''
cosine of difference between diagonally opposite angles, in vector representation
print(cos_da1__.shape, type(cos_da1__))
'''
cos_da0__ = (cos2__ * cos0__) + (sin2__ * sin0__
) # top left to bottom right
cos_da1__ = (cos3__ * cos1__) + (sin3__ * sin1__
) # top right to bottom left
dgy__ = ((g3__ + g2__) - (g0__ * cos_da0__ + g1__ * cos_da1__))
# y-decomposed cosine difference between gs
dgx__ = ((g1__ + g2__) - (g0__ * cos_da0__ + g3__ * cos_da1__))
# x-decomposed cosine difference between gs
gg__ = np.hypot(dgy__, dgx__) # gradient of gradient
mg0__ = np.minimum(g0__, g2__) * cos_da0__ # g match = min(g, _g) *cos(da)
mg1__ = np.minimum(g1__, g3__) * cos_da1__
mg__ = mg0__ + mg1__
g__ = g__[:-1, :
-1] # remove last row and column to align with derived params
dy__ = dy__[:-1, :-1]
dx__ = dx__[:-1, :-1] # -> idy, idx to compute cos for comp rg
# no longer needed: g__.mask = dy__.mask = dx__.mask = gg__.mask?
'''
next comp_rg will use g, dy, dx
next comp_gg will use gg, dgy, dgx
'''
return ma.stack((g__, dy__, dx__, gg__, dgy__, dgx__, mg__))
def generic_interp_hght(h, hght, field, log=False):
'''
Generic interpolation routine
Parameters
----------
h : number, numpy array
Height (m) of the level for which pressure is desired
hght : numpy array
The array of heights
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given height
'''
if ma.isMaskedArray(hght):
not_masked1 = ~hght.mask
else:
not_masked1 = np.ones(hght.shape)
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape)
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(h,
hght[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
if hasattr(h, 'shape') and h.shape == tuple():
h = h[()]
if type(h) != type(ma.masked):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = np.where(np.isclose(h, hght[not_masked][0]),
field[not_masked][0], field_intrp)
field_intrp = np.where(np.isclose(h, hght[not_masked][-1]),
field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
if log:
return 10**field_intrp
else:
return field_intrp
def transform(self, a):
sign = np.sign(a)
masked = ma.masked_inside(a, -self.invlinthresh, self.invlinthresh, copy=False)
exp = sign * self.linthresh * (
ma.power(self.base, (sign * (masked / self.linthresh))
- self._linscale_adj))
if masked.mask.any():
return ma.where(masked.mask, a / self._linscale_adj, exp)
else:
return exp
def transform(self, a):
sign = np.sign(a)
masked = ma.masked_inside(a, -self.linthresh, self.linthresh, copy=False)
log = sign * self.linthresh * (
self._linscale_adj +
ma.log(np.abs(masked) / self.linthresh) / self._log_base)
if masked.mask.any():
return ma.where(masked.mask, a * self._linscale_adj, log)
else:
return log
def format_and_clean_data_main(self):
"""
Main function to format and clean data based on choices by the user.
"""
# Check if over missing_bound percent or missing_bound number of values are missing
too_many_missing = self.has_too_many_missing(self.init_perc_remove)
if ma.any(too_many_missing):
idx, = ma.where(too_many_missing)
self.xs[idx] = ma.mask_rows(self.xs[idx])
# Check array to see if it is filled with values or empty
if ma.all(self.check_for_all()):
return self.xs
# Clean outliers
self.clean_outliers()
# Take average of neighbor values to fill up to a given missing value gap length
self.clean_gaps_w_linspace(fill_gap_length=self.max_gap_length)
if ma.all(ma.count_masked(self.xs[:, :-self.keep_n_values], axis=1)[np.newaxis,:] == 0):
return self.xs # if no masked values remain in values before recent ones
# Remove values if they start the array and are then followed by too many masked values
start_idx = self.find_new_starting_value()
# If there are over x% blank values left in the original data after above changes,
# check to see if x% of the blanks fall after the new start year
too_many_missing = self.has_too_many_missing(self.second_perc_remove) # boolean array
if ma.any(too_many_missing):
n_masked = np.array([ma.count_masked(self.xs[i,s_idx:])
for i, s_idx in enumerate(start_idx)]) / self.N > self.perc_remove_after_start_idx
if ma.any(n_masked):
idx, = ma.where(n_masked)
self.xs[idx] = ma.mask_rows(self.xs[idx])
# To fill in remaining values, run linear regression on non-zero values
self.clean_gaps_w_lin_regress(start_idx)
# If linear regression left negative or zero values, then use linear space to fill in middle gaps
if ma.any(ma.masked_less_equal(self.xs, 0.)):
self.clean_gaps_w_linspace()
def write_tile_to_nc(nc_variable, tiles_array, variable, zoom, _min=0, _max=4,
polarization=None):
# print len(tiles_array)
for row_number in range(len(tiles_array)):
tile_row = tiles_array[row_number]
for col_number in range(len(tile_row)):
m = tile_row[col_number]
m = ma.where(m <= _min, _min + 0.0001, m)
m = ma.where(m >= _max, _max, m)
# 0-254 , 255 for mask
m = (m-_min)/float(_max-_min) * (2**8-2)
m = np.where(m.mask, (2**8-1), m)
m = np.uint8(m)
# m = np.ma.masked_where(m == 255, m)
if polarization is None:
nc_variable[variable, zoom, row_number, col_number, :] = m[:]
else:
nc_variable[variable, polarization, zoom,
row_number, col_number, :] = m[:]
def draw_bar(ax, show_first_operation, x, y, time, init_time, draw_color):
assert ax is not None
if y is None:
if show_first_operation is True:
mask_time = ma.where(time>=init_time)
mask_init_time = ma.where(init_time>=time)
ax.bar(x[mask_time], time[mask_time], width=bar_width, color=draw_color)
ax.bar(x, init_time, width=bar_width, color=first_operation_color)
ax.bar(x[mask_init_time], time[mask_init_time], width=bar_width, color=draw_color)
else:
ax.bar(x, time, width=bar_width, color=draw_color)
else:
if show_first_operation is True:
mask_time = ma.where(time>=init_time)
mask_init_time = ma.where(init_time>=time)
ax.bar(x[mask_time], (time - init_time)[mask_time], y[mask_time], bottom=init_time[mask_time], zdir='y', width=bar_width, color=draw_color)
ax.bar(x[mask_init_time], (init_time - time)[mask_init_time], y[mask_init_time], bottom=time[mask_init_time], zdir='y', width=bar_width, color=first_operation_color)
ax.bar(x[mask_init_time], time[mask_init_time], y[mask_init_time], zdir='y', width=bar_width, color=draw_color)
ax.bar(x[mask_time], init_time[mask_time], y[mask_time], zdir='y', width=bar_width, color=first_operation_color)
else:
ax.bar(x, time, y, zdir='y', width=bar_width, color=draw_color)
def _pfromz_MA(z, lapse_rate, P_bott, T_bott, z_bott):
"""Pressure given altitude in a constant lapse rate layer.
The dry gas constant is used in calculations requiring the gas
constant. See the docstring for press2alt for references.
Input Arguments:
* z: Geopotential altitude [m].
* lapse_rate: -dT/dz [K/m] over the layer.
* P_bott: Pressure [hPa] at the base of the layer.
* T_bott: Temperature [K] at the base of the layer.
* z_bott: Geopotential altitude [m] of the base of the layer.
Output:
* Pressure [hPa] for each element given in the input arguments.
All input arguments can be either a scalar or an MA array. All
arguments that are MA arrays, however, are of the same size and
shape. If every input argument is a scalar, the output is a scalar.
If any of the input arguments is an MA array, the output is an MA
array of the same size and shape.
"""
#jfp was import Numeric as N
import numpy as N
#jfp was import MA
import numpy.ma as MA
from atmconst import AtmConst
const = AtmConst()
if MA.size(lapse_rate) == 1:
#jfp was if MA.array(lapse_rate)[0] == 0.0:
if MA.array(lapse_rate) == 0.0:
return P_bott * \
MA.exp( -const.g / (const.R_d*T_bott) * (z-z_bott) )
else:
exponent = const.g / (const.R_d * lapse_rate)
return P_bott * \
( (1.0 - (lapse_rate * (z-z_bott) / T_bott))**exponent )
else:
exponent = const.g / (const.R_d * lapse_rate)
P = P_bott * \
( (1.0 - (lapse_rate * (z-z_bott) / T_bott))**exponent )
P_at_0 = P_bott * \
MA.exp( -const.g / (const.R_d*T_bott) * (z-z_bott) )
zero_lapse_mask = MA.filled(MA.where(lapse_rate == 0., 1, 0), 0)
zero_lapse_mask_indices_flat = N.nonzero(N.ravel(zero_lapse_mask))
P_flat = MA.ravel(P)
MA.put( P_flat, zero_lapse_mask_indices_flat \
, MA.take(MA.ravel(P_at_0), zero_lapse_mask_indices_flat) )
return MA.reshape(P_flat, P.shape)
def generic_interp_hght(h, hght, field, log=False):
'''
Generic interpolation routine
Parameters
----------
h : number, numpy array
Height (m) of the level for which pressure is desired
hght : numpy array
The array of heights
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given height
'''
if ma.isMaskedArray(hght):
not_masked1 = ~hght.mask
else:
not_masked1 = np.ones(hght.shape)
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape)
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(h, hght[not_masked], field[not_masked],
left=ma.masked, right=ma.masked)
if hasattr(h, 'shape') and h.shape == tuple():
h = h[()]
if type(h) != type(ma.masked):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = np.where(np.isclose(h, hght[not_masked][0]), field[not_masked][0], field_intrp)
field_intrp = np.where(np.isclose(h, hght[not_masked][-1]), field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
if log:
return 10 ** field_intrp
else:
return field_intrp
def estimate_cell_edges(x):
"""Convert one-dimensional vector x of size n into n + 1, where the input
describes the centres of the cells, and the output is an estimate of the
edges of the cell"""
# centres (with extra centres padded at the ends by linear interpolation)
dx = ma.diff(x)
x_c = ma.hstack((x[0] - atleast_1d(dx[0]), x,
x[-1] + atleast_1d(dx[-1])))
# _f is notation from MITgcm (implies faces)
x_f = (x_c[1:] + x_c[:-1])/2
dx_c = np.diff(x_c)
# Catch nan or masked values and estimate edge using dx from previous or
# next cell
nan_before = ma.where(
ma.logical_and(nan_or_masked(x_f[:-1]), ~nan_or_masked(x_f[1:])))[0]
nan_after = ma.where(
ma.logical_and(~nan_or_masked(x_f[:-1]), nan_or_masked(x_f[1:])))[0]
x_f[nan_before] = x_f[nan_before + 1] - dx_c[nan_before + 1]
x_f[nan_after + 1] = x_f[nan_after] + dx_c[nan_after]
return x_f
def _zfromp_MA(P, lapse_rate, P_bott, T_bott, z_bott):
"""Altitude given pressure in a constant lapse rate layer.
The dry gas constant is used in calculations requiring the gas
constant. See the docstring for press2alt for references.
Input Arguments:
* P: Pressure [hPa].
* lapse_rate: -dT/dz [K/m] over the layer.
* P_bott: Pressure [hPa] at the base of the layer.
* T_bott: Temperature [K] at the base of the layer.
* z_bott: Geopotential altitude [m] of the base of the layer.
Output:
* Altitude [m] for each element given in the input arguments.
All input arguments can be either a scalar or an MA array. All
arguments that are MA arrays, however, are of the same size and
shape. If every input argument is a scalar, the output is a scalar.
If any of the input arguments is an MA array, the output is an MA
array of the same size and shape.
"""
import numpy as N
#jfp was import Numeric as N
import numpy.ma as MA
#jfp was import MA
from atmconst import AtmConst
const = AtmConst()
if MA.size(lapse_rate) == 1:
if MA.array(lapse_rate)[0] == 0.0:
return ( (-const.R_d * T_bott / const.g) * MA.log(P/P_bott) ) + \
z_bott
else:
exponent = (const.R_d * lapse_rate) / const.g
return ((T_bott / lapse_rate) * (1. - (P/P_bott)**exponent)) + \
z_bott
else:
exponent = (const.R_d * lapse_rate) / const.g
z = ((T_bott / lapse_rate) * (1. - (P/P_bott)**exponent)) + z_bott
z_at_0 = ( (-const.R_d * T_bott / const.g) * MA.log(P/P_bott) ) + \
z_bott
zero_lapse_mask = MA.filled(MA.where(lapse_rate == 0., 1, 0), 0)
zero_lapse_mask_indices_flat = N.nonzero(N.ravel(zero_lapse_mask))
z_flat = MA.ravel(z)
MA.put( z_flat, zero_lapse_mask_indices_flat \
, MA.take(MA.ravel(z_at_0), zero_lapse_mask_indices_flat) )
return MA.reshape(z_flat, z.shape)
def __fillMissing(self):
scens = range(self.missing.shape[1])
scens = roll(scens, len(scens) - 1) # make 00 last
vidx, sidx, tidx = where(self.missing)
for i in range(len(vidx)):
v, s, t = vidx[i], sidx[i], tidx[i]
filled = False
for tnew, snew in product([t, max(t - 1, 0), max(t - 2, 0)], scens):
if not self.missing[v, snew, tnew]:
self.data[v, s, t] = self.data[v, snew, tnew]
filled = True
break
if not filled:
raise Exception('Failed to fill')
def __init__(self, file, lat, lon, vars = None):
with nc(file) as f:
if vars is None:
vars = setdiff1d(f.variables, ['lat', 'lon', 'time'])
lats, lons = f.variables['lat'][:], f.variables['lon'][:]
if isMaskedArray(f.variables[vars[0]][0]):
mask = f.variables[vars[0]][0].mask # pull mask from first variable, first time
else:
mask = zeros((len(lats), len(lons)))
latd = resize(lats, (len(lons), len(lats))).T - lat
lond = resize(lons, (len(lats), len(lons))) - lon
latd = masked_where(mask, latd)
lond = masked_where(mask, lond)
totd = latd ** 2 + lond ** 2
idx = where(totd == totd.min())
latidx, lonidx = idx[0][0], idx[1][0]
self.time = f.variables['time'][:]
tunits = f.variables['time'].units
ts = tunits.split('months since ')[1].split(' ')
yr0, mth0, day0 = [int(t) for t in ts[0].split('-')[0 : 3]]
if len(ts) > 1:
hr0, min0, sec0 = [int(t) for t in ts[1].split(':')[0 : 3]]
else:
hr0 = min0 = sec0 = 0
self.reftime = datetime(yr0, mth0, day0, hr0, min0, sec0)
nv, nt = len(vars), len(self.time)
self.data = zeros((nv, nt))
self.units = zeros(nv, dtype = '|S32')
for i in range(nv):
if vars[i] in f.variables:
var = f.variables[vars[i]]
else:
vidx = foundVar(f.variables.keys(), vars[i])
var = f.variables[f.variables.keys()[vidx]]
self.data[i] = var[:, latidx, lonidx]
self.units[i] = var.units
self.vars = vars # store variable names
self.pridx = foundVar(vars, 'pr') # variable indices
self.maidx = foundVar(vars, 'tmax')
self.miidx = foundVar(vars, 'tmin')
def clean_gaps_w_lin_regress(self, start_idx):
"""
Function to clean gaps in the data with a linear regression.
Parameters
----------
start_idx : integer
First non-masked value of array.
"""
non_zero_idx = np.transpose(self.xs.nonzero())
for i in xrange(self.rows_N):
idx = non_zero_idx[np.where(non_zero_idx[:, 0] == i)][:, 1]
if idx.any():
slope, intercept, r, p, se = mstats.linregress(self.yrs[idx], self.xs[i,idx])
missing_xs = ma.where(self.xs[i, start_idx[i]:-self.keep_n_values].mask)[0] + start_idx[i]
if np.any(missing_xs):
self.xs[i, missing_xs] = (self.min_year + missing_xs) * slope + intercept
def aggrade_front(self, grid, tstep, source_cells_Qs, elev, SL): #ensure Qs and tstep units match!
self.total_sed_supplied_in_tstep = source_cells_Qs*tstep
self.Qs_sort_order = np.argsort(source_cells_Qs)[::-1] #descending order
self.Qs_sort_order = self.Qs_sort_order[:np.count_nonzero(self.Qs_sort_order>0)]
for i in self.Qs_sort_order:
subaerial_nodes = elev>=SL
subsurface_elev_array = ma.array(elev, subaerial_nodes)
xy_tuple = (grid.node_x[i], grid.node_y[i])
distance_map = grid.get_distances_of_nodes_to_point(xy_tuple)
loop_number = 0
closest_node_list = ma.argsort(ma.masked_array(distance_map, mask=subsurface_elev_array.mask))
smooth_cone_elev_from_apex = subsurface_elev_array[i]-distance_map*self.tan_repose_angle
while 1:
filled_all_cells_flag = 0
accom_space_at_controlling_node = SL - subsurface_elev_array[closest_node_list[loop_number]]
new_max_cone_surface_elev = smooth_cone_elev_from_apex + accom_space_at_controlling_node
subsurface_elev_array.mask = (elev>=SL or new_max_cone_surface_elev<elev)
depth_of_accom_space = new_max_cone_surface_elev - subsurface_elev_array
accom_depth_order = ma.argsort(depth_of_accom_space)[::-1]
#Vectorised method to calc fill volumes:
area_to_fill = ma.cumsum(grid.cellarea[accom_depth_order])
differential_depths = ma.empty_like(depth_of_accom_space)
differential_depths[:-1] = depth_of_accom_space[accom_depth_order[:-1]] - depth_of_accom_space[accom_depth_order[1:]]
differential_depths[-1] = depth_of_accom_space[accom_depth_order[-1]]
incremental_volumes = ma.cumsum(differential_depths*area_to_fill)
match_position_of_Qs_in = ma.searchsorted(incremental_volumes, self.total_sed_supplied_in_tstep[i])
try:
depths_to_add = depth_of_accom_space-depth_of_accom_space[match_position_of_Qs_in]
except:
depths_to_add = depth_of_accom_space-depth_of_accom_space[match_position_of_Qs_in-1]
filled_all_cells_flag = 1
depths_to_add = depths_to_add[ma.where(depths_to_add>=0)]
if not filled_all_cells_flag:
depths_to_add += (self.total_sed_supplied_in_tstep[i] - incremental_volumes[match_position_of_Qs_in-1])/area_to_fill[match_position_of_Qs_in-1]
subsurface_elev_array[accom_depth_order[len(depths_to_add)]] = depths_to_add
self.total_sed_supplied_in_tstep[i] = 0
break
else:
subsurface_elev_array[accom_depth_order] = depths_to_add
self.total_sed_supplied_in_tstep[i] -= incremental_volumes[-1]
loop_number += 1
return elev
copyfile(springfile, outputfile)
with nc(springfile) as f:
planting1 = f.variables['planting'][:]
anthesis1 = f.variables['anthesis'][:]
maturity1 = f.variables['maturity'][:]
with nc(winterfile) as f:
planting2 = f.variables['planting'][:]
anthesis2 = f.variables['anthesis'][:]
maturity2 = f.variables['maturity'][:]
with nc(maskfile) as f:
mask = f.variables['variety'][:]
latidx, lonidx = where(mask == 2)
plantingf = planting1.copy()
anthesisf = anthesis1.copy()
maturityf = maturity1.copy()
plantingf[:, latidx, lonidx, :] = planting2[:, latidx, lonidx, :]
anthesisf[:, latidx, lonidx, :] = anthesis2[:, latidx, lonidx, :]
maturityf[:, latidx, lonidx, :] = maturity2[:, latidx, lonidx, :]
with nc(outputfile, 'a') as f:
p = f.variables['planting']
p[:] = plantingf
a = f.variables['anthesis']
a[:] = anthesisf
m = f.variables['maturity']
def test_testOddFeatures(self):
# Test of other odd features
x = arange(20)
x = x.reshape(4, 5)
x.flat[5] = 12
assert_(x[1, 0] == 12)
z = x + 10j * x
assert_(eq(z.real, x))
assert_(eq(z.imag, 10 * x))
assert_(eq((z * conjugate(z)).real, 101 * x * x))
z.imag[...] = 0.0
x = arange(10)
x[3] = masked
assert_(str(x[3]) == str(masked))
c = x >= 8
assert_(count(where(c, masked, masked)) == 0)
assert_(shape(where(c, masked, masked)) == c.shape)
z = where(c, x, masked)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is masked)
assert_(z[7] is masked)
assert_(z[8] is not masked)
assert_(z[9] is not masked)
assert_(eq(x, z))
z = where(c, masked, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
z = masked_where(c, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
assert_(eq(x, z))
x = array([1., 2., 3., 4., 5.])
c = array([1, 1, 1, 0, 0])
x[2] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
c[0] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
assert_(eq(masked_where(greater_equal(x, 2), x),
masked_greater_equal(x, 2)))
assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
assert_(eq(masked_inside(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 1, 3).mask,
[1, 1, 1, 1, 0]))
assert_(eq(masked_outside(array(list(range(5)),
mask=[0, 1, 0, 0, 0]), 1, 3).mask,
[1, 1, 0, 0, 1]))
assert_(eq(masked_equal(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 0]))
assert_(eq(masked_not_equal(array([2, 2, 1, 2, 1],
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 1]))
assert_(eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
[99, 99, 3, 4, 5]))
atest = ones((10, 10, 10), dtype=np.float32)
btest = zeros(atest.shape, MaskType)
ctest = masked_where(btest, atest)
assert_(eq(atest, ctest))
z = choose(c, (-x, x))
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
x = arange(6)
x[5] = masked
y = arange(6) * 10
y[2] = masked
c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
cm = c.filled(1)
z = where(c, x, y)
zm = where(cm, x, y)
assert_(eq(z, zm))
assert_(getmask(zm) is nomask)
assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
z = where(c, masked, 1)
assert_(eq(z, [99, 99, 99, 1, 1, 1]))
z = where(c, 1, masked)
assert_(eq(z, [99, 1, 1, 99, 99, 99]))
yld1 = f.variables['yield'][:]
area1 = f.variables['area'][:]
marea1 = f.variables['area_mirca'][:]
with nc(winterfile) as f:
yld2 = f.variables['yield'][:]
area2 = f.variables['area'][:]
marea2 = f.variables['area_mirca'][:]
a1 = area1[20 : 31, :, :, 2].mean(axis = 0) # sum, averaged from 2000-2010
a2 = area2[20 : 31, :, :, 2].mean(axis = 0)
latd = resize(lat, (len(lon), len(lat))).T
latd = masked_where(a1.mask, latd)
latidx, lonidx = where(logical_and(a2 > a1, latd <= 49)) # replace with winter wheat if area is greater AND lat <= 49
yldf = yld1.copy()
areaf = area1.copy()
mareaf = marea1.copy()
yldf[:, latidx, lonidx, :] = yld2[:, latidx, lonidx, :]
areaf[:, latidx, lonidx, :] = area2[:, latidx, lonidx, :]
mareaf[:, latidx, lonidx, :] = marea2[:, latidx, lonidx, :]
dvar = a1.copy()
dvar[:] = 1
dvar[logical_and(a1 < a2, latd <= 49)] = 2
dvar = masked_where(a1.mask, dvar)
with nc(outputfile, 'w') as f:
def showSlices(self):
""" Data loading from files, processing and displaying in window.
Data are loaded by nibabel functions, and processed by numpy functions. All files in self.volList must contain same volume size.
For each file (number i), full volume is loaded. Then indifined values (nan) are suppressed, and voxels are normalized by minimum and maximum values for visibility.
Three views are extracted from volume:
- slice in (x,y) plan -> set by self.posz
- slice in (y,z) plan -> set by self.posx
- slice in (x,z) plan -> set by self.posy
Slices coordinates (self.posz, self.posy, self.posz) are initially set to the center of the volume (when self.posx value is -1). Later these coordinates may change (see setViews method).
The slices are set in 3D-matrixes self.ImXY, self.ImYZ and self.XZ, in (:,:,i).
Then, method imSumShow is called to create and show figure.
Note that each time this method is called (window initialization, change of view coordinates by method setViews) complete files are loaded again and previous figure is lost.
Variables:
- vol content of a file, loaded by bibabel fonction load
- self.volList files names list, created by function checkFiles
- self.data voxels values of volume in first file
- self.posx, self.posy, self.posz
slices coordinates for plans (y,z), (x,z) and (x,y) respectively
- self.ImXY, self.ImYZ, self.ImXZ
3D-matrixes, containing slices of each file, one matrix per slice plan
dimensions 1 and 2: size of slices in plans (y,z), (x,z) and (x,y) respectively
dimension 3: file number
- data voxels values of volume vol
- mindata for volume vol, gives lower voxel value
- maxdata for volume vol, gives higher voxel value
- dataNorm volume vol normalized by mindata and maxdata
"""
# images dimensions
vol = nib.load(self.volList[0])
self.data = vol.get_data()
# first reference ccordinates
if self.posx.get() == -1:
self.posx.set(self.data.shape[0] / 2)
self.posy.set(self.data.shape[1] / 2)
self.posz.set(self.data.shape[2] / 2)
# matrixes initialization
self.ImXY = zeros((self.data.shape[0], self.data.shape[1], self.nb))
self.ImYZ = zeros((self.data.shape[1], self.data.shape[2], self.nb))
self.ImXZ = zeros((self.data.shape[0], self.data.shape[2], self.nb))
# normalized slices
for i, v in enumerate(self.volList):
# volume loading
vol = nib.load(v)
data = vol.get_data()
# nan are suppressed
data = ma.where(~isnan(data), data, array(zeros(data.shape)))
# volume normalization
mindata = data.min()
dataNorm = data - mindata
maxdata = dataNorm.max()
dataNorm = dataNorm / maxdata
mindata = dataNorm.min()
maxdata = dataNorm.max()
# slices extraction from volume
self.ImXY[:, :, i] = dataNorm[:, :, self.posz.get()]
self.ImYZ[:, :, i] = dataNorm[self.posx.get(), :, :] ##???
self.ImXZ[:, :, i] = dataNorm[:, self.posy.get(), :]
# show images
self.imSumShow()
