def add_ellipse(ax, x, y, kx, ky, mask, Kxrange, Kyrange, color='k'):
''' '''
if (not np.isnan(kx)) and (
not np.isnan(ky)) and (not mask) and ma.logical_and(
ky > Kyrange[0], ky < Kyrange[-1]) and ma.logical_and(
kx > Kxrange[0], kx < Kxrange[-1]): # and abs(y)<60:
wax = 2.5 #*np.cos(y*np.pi/180.)
hax = wax * ky / kx
#if hax>1:
# hax=hax/np.log(hax)
# wax=wax/np.log(hax)
if hax > 2.0 * wax:
hax = 2.0 * wax #*hax/hax
wax = hax * kx / ky #2.0*wax/hax
#angle=np.arctan(ky/kx)*180/np.pi-90
angle = 0
e = Ellipse(xy=(x, y),
width=wax,
height=hax,
angle=angle,
facecolor='none',
edgecolor=color,
rasterized=True)
ax.add_artist(e)
ax.plot(x, y, '.k', markersize=1)
def sigclip(x, Nsig=3.0, eps=1e-6, ids=None):
xo = x.mean()
xlo = x.mean() - Nsig * x.std(ddof=1)
xhi = x.mean() + Nsig * x.std(ddof=1)
#x = numpy.clip(x,xlo,xhi)
idx = numpy.where(logical_and(x > xlo, x < xhi))
xn = x[idx]
if x.std(ddof=1) == 0.0:
if ids:
idx = numpy.indices(x.shape)[0]
print("idx", idx)
return idx
else:
return x
i = 0
while abs(1 - old_div(xo, xn.mean())) > eps:
xo = xn.mean()
xlo = xo - Nsig * xn.std(ddof=1)
xhi = xo + Nsig * xn.std(ddof=1)
idx = numpy.where(logical_and(x > xlo, x < xhi))
xn = x[idx]
i = i + 1
if ids:
return idx
else:
#return x
return xn
def sigclip_w(x, wt=None, Nsig=2.0, eps=1e-6, ids=None):
i = 1
s = stats(x, wt)
xo = s.mean
std = s.std
xlo = s.mean - Nsig * std
xhi = s.mean + Nsig * std
idx = numpy.where(logical_and(x > xlo, x < xhi))
xn = x[idx]
wn = wt[idx]
i = 0
#print i,xo,stats(xn,wn).mean,stats(xn,wn).std, xn.mean(), xn.stddev(), len(xn),xlo,xhi
while abs(1 - old_div(xo, stats(xn, wn).mean)) > eps:
#print i,xo,stats(xn,wn).mean,stats(xn,wn).std, xn.mean(), xn.stddev(), len(xn),xlo,xhi
s = stats(xn, wn)
xo = s.mean
xlo = xo - Nsig * s.std
xhi = xo + Nsig * s.std
# Clip it
idx = numpy.where(logical_and(x > xlo, x < xhi))
xn = x[idx]
wn = wt[idx]
i = i + 1
if ids:
return idx
else:
return xn, wn
def log_linear_vinterp(T,P,levs):
'''
# Author Charles Doutriaux
# Version 1.1
# Expect 2D field here so there''s no reorder which I suspect to do a memory leak
# email: [email protected]
# Converts a field from sigma levels to pressure levels
# Log linear interpolation
# Input
# T : temperature on sigma levels
# P : pressure field from TOP (level 0) to BOTTOM (last level)
# levs : pressure levels to interplate to (same units as P)
# Output
# t : temperature on pressure levels (levs)
# External: Numeric'''
import numpy.ma as MA
## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16
sh=P.shape
nsigma=sh[0] # Number of sigma levels
try:
nlev=len(levs) # Number of pressure levels
except:
nlev=1 # if only one level len(levs) would breaks
t=[]
for ilv in range(nlev): # loop through pressure levels
try:
lev=levs[ilv] # get value for the level
except:
lev=levs # only 1 level passed
# print ' ......... level:',lev
Pabv=MA.ones(P[0].shape,Numeric.Float)
Tabv=-Pabv # Temperature on sigma level Above
Tbel=-Pabv # Temperature on sigma level Below
Pbel=-Pabv # Pressure on sigma level Below
Pabv=-Pabv # Pressure on sigma level Above
for isg in range(1,nsigma): # loop from second sigma level to last one
## print 'Sigma level #',isg
a = MA.greater(P[isg], lev) # Where is the pressure greater than lev
b = MA.less(P[isg-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Tabv, Tbel
Pabv=MA.where(MA.logical_and(a,b),P[isg],Pabv) # Pressure on sigma level Above
Tabv=MA.where(MA.logical_and(a,b),T[isg],Tabv) # Temperature on sigma level Above
Pbel=MA.where(MA.logical_and(a,b),P[isg-1],Pbel) # Pressure on sigma level Below
Tbel=MA.where(MA.logical_and(a,b),T[isg-1],Tbel) # Temperature on sigma level Below
# end of for isg in range(1,nsigma)
# val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is
tl=MA.masked_where(MA.equal(Pbel,-1.),MA.log(lev/MA.absolute(Pbel))/MA.log(Pabv/Pbel)*(Tabv-Tbel)+Tbel) # Interpolation
t.append(tl) # add a level to the output
# end of for ilv in range(nlev)
return asMA(t).astype(Numeric.Float32) # convert t to an array
def __init__(self,
filename,
varnames=None,
lats=None,
lons=None,
incl_global=False):
f = nc(filename) # open file
if varnames is None: # no variables specified
varnames = f.variables.keys()
varnames = [v for v in varnames
if not v in ['lat', 'lon']] # remove lat, lon
if incl_global: varnames += ['global']
else:
if not isinstance(varnames, list): # make list
varnames = [varnames]
self.lats, self.lons = f.variables['lat'][:], f.variables['lon'][:]
self.dat = {'names': [], 'units': [], 'longnames': [], 'data': []}
for v in varnames:
if v != 'global':
var = f.variables[v]
self.dat['names'].append(v)
self.dat['units'].append(var.units if 'units' in
var.ncattrs() else '')
self.dat['longnames'].append(var.long_name if 'long_name' in
var.ncattrs() else '')
self.dat['data'].append(var[:])
else:
nlats = self.lats.size
nlons = self.lons.size
self.dat['names'].append('global') # global mask
self.dat['units'].append('')
self.dat['longnames'].append('')
self.dat['data'].append(
masked_array(ones((nlats, nlons)),
mask=zeros((nlats, nlons))))
f.close()
tol = 1e-5
if not lats is None: # restrict latitude range
sellat = logical_and(self.lats >= lats.min() - tol,
self.lats <= lats.max() + tol)
self.lats = self.lats[sellat]
for i in range(len(self.dat['names'])):
self.dat['data'][i] = self.dat['data'][i][sellat]
if not lons is None: # restrict longitude range
sellon = logical_and(self.lons >= lons.min() - tol,
self.lons <= lons.max() + tol)
self.lons = self.lons[sellon]
for i in range(len(self.dat['names'])):
self.dat['data'][i] = self.dat['data'][i][:, sellon]
def __init__(self, filename, varnames = None, lats = None, lons = None, incl_global = False):
f = nc(filename) # open file
if varnames is None: # no variables specified
varnames = f.variables.keys()
varnames = [v for v in varnames if not v in ['lat', 'lon']] # remove lat, lon
if incl_global: varnames += ['global']
else:
if not isinstance(varnames, list): # make list
varnames = [varnames]
self.lats, self.lons = f.variables['lat'][:], f.variables['lon'][:]
self.dat = {'names': [], 'units': [], 'longnames': [], 'data': []}
for v in varnames:
if v != 'global' or (v == 'global' and 'global' in f.variables):
var = f.variables[v]
self.dat['names'].append(v)
self.dat['units'].append(var.units if 'units' in var.ncattrs() else '')
self.dat['longnames'].append(var.long_name if 'long_name' in var.ncattrs() else '')
self.dat['data'].append(var[:])
else:
nlats = self.lats.size
nlons = self.lons.size
self.dat['names'].append('global') # global mask
self.dat['units'].append('')
self.dat['longnames'].append('')
self.dat['data'].append(masked_array(ones((nlats, nlons)), mask = zeros((nlats, nlons))))
f.close()
tol = 1e-5
if not lats is None: # restrict latitude range
sellat = logical_and(self.lats >= lats.min() - tol, self.lats <= lats.max() + tol)
self.lats = self.lats[sellat]
for i in range(len(self.dat['names'])):
self.dat['data'][i] = self.dat['data'][i][sellat]
if not lons is None: # restrict longitude range
sellon = logical_and(self.lons >= lons.min() - tol, self.lons <= lons.max() + tol)
self.lons = self.lons[sellon]
for i in range(len(self.dat['names'])):
self.dat['data'][i] = self.dat['data'][i][:, sellon]
def process_aia_map(m, rsun=1.2, threshold=35):
from sunpy.map.maputils import all_coordinates_from_map
hpc_coords = all_coordinates_from_map(m)
r = np.sqrt(hpc_coords.Tx ** 2 + hpc_coords.Ty ** 2) / m.rsun_obs
mask = ma.logical_and(r > rsun, m.data < threshold)
processed_map = Map(m.data, m.meta, mask=mask)
return(processed_map)
def cifar100_complicated_ensemble_v2_submodel5_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
# 50 - 59, 80 - 99, 0 - 9 classes.
labels_array[logical_and(labels_array > 9, labels_array < 50)] = -1
labels_array[logical_and(labels_array > 59, labels_array < 80)] = -1
for i in range(0, 10):
labels_array[labels_array == i] = i + 1
labels_array[labels_array == i + 50] = i + 11
labels_array[labels_array == i + 80] = i + 21
labels_array[labels_array == i + 90] = i + 31
labels_array[labels_array == -1] = 0
return 41
def bayes(xvec, xcrit, label, unknown, fout=sys.stdout):
warm = ma.masked_array(xvec > xcrit)
warm = ma.logical_and(warm, unknown)
nwarm = len(warm.nonzero()[0])
lmask = np.logical_and(landmask, warm)
imask = np.logical_and(icemask, warm)
omask = np.logical_and(watermask, warm)
pwarm = float(nwarm) / float(nobs)
pover_land = len(lmask.nonzero()[0]) / nlandpts
pover_water = len(omask.nonzero()[0]) / nwaterpts
pover_ice = len(imask.nonzero()[0]) / nicepts
if (pwarm > 0):
print(label,
"hot ",
xcrit,
"{:5.3f}".format(pover_ice * pice / pwarm),
"{:5.3f}".format(pover_land * pland / pwarm),
"{:5.3f}".format(pover_water * pwater / pwarm),
nwarm,
file=fout)
cold = ma.masked_array(xvec < xcrit)
cold = ma.logical_and(cold, unknown)
ncold = len(cold.nonzero()[0])
lmask = np.logical_and(landmask, cold)
imask = np.logical_and(icemask, cold)
omask = np.logical_and(watermask, cold)
pcold = float(ncold) / float(nobs)
pover_land = len(lmask.nonzero()[0]) / nlandpts
pover_water = len(omask.nonzero()[0]) / nwaterpts
pover_ice = len(imask.nonzero()[0]) / nicepts
if (pcold > 0):
print(label,
"cold ",
xcrit,
"{:5.3f}".format(pover_ice * pice / pcold),
"{:5.3f}".format(pover_land * pland / pcold),
"{:5.3f}".format(pover_water * pwater / pcold),
ncold,
file=fout)
def estimate_cell_edges(x):
"""Convert one-dimensional vector x of size n into n + 1, where the input
describes the centres of the cells, and the output is an estimate of the
edges of the cell"""
# centres (with extra centres padded at the ends by linear interpolation)
dx = ma.diff(x)
x_c = ma.hstack((x[0] - atleast_1d(dx[0]), x,
x[-1] + atleast_1d(dx[-1])))
# _f is notation from MITgcm (implies faces)
x_f = (x_c[1:] + x_c[:-1])/2
dx_c = np.diff(x_c)
# Catch nan or masked values and estimate edge using dx from previous or
# next cell
nan_before = ma.where(
ma.logical_and(nan_or_masked(x_f[:-1]), ~nan_or_masked(x_f[1:])))[0]
nan_after = ma.where(
ma.logical_and(~nan_or_masked(x_f[:-1]), nan_or_masked(x_f[1:])))[0]
x_f[nan_before] = x_f[nan_before + 1] - dx_c[nan_before + 1]
x_f[nan_after + 1] = x_f[nan_after] + dx_c[nan_after]
return x_f
def combine_Taus(data, weight_coslat, Taus, K_lim=True):
"""Use the CFL criteria to limit Tau, at first choose the min dt (max spped) for each location."""
for key in ['U_global', 'V_global', 'Kx_global', 'Ky_global', 'R_global']:
data[key] = np.reshape(
data[key][:],
(len(Taus), data[key].shape[0] / len(Taus), -1)).squeeze()
lmask = data['R_global'][0, :, :].mask
dt = ma.min(
(1 / (abs(data['U_global'][:]) /
(weight_coslat * 111E3 * 0.25) + abs(data['V_global'][:]) /
(111E3 * 0.25))) / (3600 * 24.), 0)
#make dt's to be integers
dt[np.where(dt < Taus[0])] = Taus[0]
for t, tau in enumerate(Taus[:-2]):
dt[np.where(ma.logical_and(dt > tau, dt <= Taus[t + 1]))] = Taus[t + 1]
dt[np.where(dt > Taus[-2])] = Taus[-1]
if K_lim:
c = 0
while c < max(Taus):
c = c + 1
for t, tau in enumerate(Taus):
dx = (weight_coslat * 111E3 * 0.25)
dy = 111E3 * 0.25
jinds, iinds = np.where(dt.squeeze() == tau)
if len(jinds) > 1:
jindsX = np.where(
data['Kx_global'][t, jinds, iinds].squeeze() * tau *
3600 * 24 / dx[jinds, iinds]**2 > 1)[0]
if len(jindsX) > 1:
dt[jinds[jindsX], iinds[jindsX]] = Taus[max(
t - 1, 0
)] #np.max([dt[jinds[jindsX],iinds[jindsX]]-1,np.ones(len(jindsX))*Taus[0]],axis=0)
jindsY = np.where(
data['Ky_global'][t, jinds, iinds].squeeze() * tau *
3600 * 24 / dy**2 > 1)[0]
if len(jindsY) > 1:
dt[jinds[jindsY], iinds[jindsY]] = Taus[max(
t - 1, 0
)] #np.max([dt[jinds[jindsY],iinds[jindsY]]-1,np.ones(len(jindsY))*Taus[0]],axis=0)
#use dt and pick up each field given the location specific Tau
for key in ['U_global', 'V_global', 'Kx_global', 'Ky_global', 'R_global']:
dum2 = np.zeros(data[key][0, :, :].shape)
for j, ext in enumerate(Taus):
jinds, iinds = np.where(dt.astype('int') == ext)
dum2[jinds, iinds] = data[key][j, jinds, iinds].squeeze()
#dum2[jinds,iinds]=ma.max(data[key][:j+1,jinds,iinds].squeeze(),0)
data[key] = ma.masked_array(dum2, lmask)
#
return data, ma.masked_array(dt.astype('int'), lmask)
def svhn_complicated_ensemble_v2_submodel5_labels_manipulation(
labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
# 5, 8, 9, 0 classes.
labels_array[logical_and(labels_array < 5, labels_array > 0)] = 6
labels_array[labels_array == 0] = 1
labels_array[labels_array == 5] = 2
labels_array[labels_array == 8] = 3
labels_array[labels_array == 9] = 4
labels_array[labels_array == 6] = 0
labels_array[labels_array == 7] = 0
return 5
def stand_est(data_mat, user, sim_means, item):
n = shape(data_mat)[1]
all_sim = 0
item_score = 0
for j in range(n):
score = data_mat[user, j]
if score == 0:
continue
over_lap = nonzero(logical_and(data_mat[:, j] > 0, data_mat[:, item] > 0))[0]
sim = sim_means(data_mat[over_lap, j], data_mat[over_lap, item])
all_sim += sim
item_score += sim * score
if all_sim == 0:
item_score = 0
else:
item_score = item_score / all_sim
return item_score
def stand_est(data_mat, user, sim_means, item):
n = shape(data_mat)[1]
all_sim = 0
item_score = 0
for j in range(n):
score = data_mat[user, j]
if score == 0:
continue
over_lap = nonzero(
logical_and(data_mat[:, j] > 0, data_mat[:, item] > 0))[0]
sim = sim_means(data_mat[over_lap, j], data_mat[over_lap, item])
all_sim += sim
item_score += sim * score
if all_sim == 0:
item_score = 0
else:
item_score = item_score / all_sim
return item_score
def _calc_gsl(self, values, threshold):
""" Calculates GSL for the given values"""
data_shape = values.shape[1:]
gsl_cnt = ma.zeros(data_shape)
warm_cnt = np.zeros(data_shape)
cold_cnt = np.zeros(data_shape)
increment = np.zeros(data_shape)
for i, arr in enumerate(values):
if i < 183: # Take the first half of an year.
mask = arr > threshold
warm_cnt += mask # Count warm days.
warm_cnt *= mask # Reset counter of warm days on a cold one.
increment = ma.logical_or(increment, (warm_cnt == 5)) # Search for cells with 5 consecutive warm days.
else: # Take the second part of an year.
mask = arr < threshold
cold_cnt += mask # Count cold days.
cold_cnt *= mask # Reset counter of cold days on a warm one.
increment = ma.logical_and(increment, ~(cold_cnt == 5)) # Search for cells with 5 consecutive cold days.
gsl_cnt += increment # Count days inside GSL at each cell.
gsl_cnt.mask = values.mask[0] # Take source mask.
return gsl_cnt
yld1 = f.variables['yield'][:]
area1 = f.variables['area'][:]
marea1 = f.variables['area_mirca'][:]
with nc(winterfile) as f:
yld2 = f.variables['yield'][:]
area2 = f.variables['area'][:]
marea2 = f.variables['area_mirca'][:]
a1 = area1[20 : 31, :, :, 2].mean(axis = 0) # sum, averaged from 2000-2010
a2 = area2[20 : 31, :, :, 2].mean(axis = 0)
latd = resize(lat, (len(lon), len(lat))).T
latd = masked_where(a1.mask, latd)
latidx, lonidx = where(logical_and(a2 > a1, latd <= 49)) # replace with winter wheat if area is greater AND lat <= 49
yldf = yld1.copy()
areaf = area1.copy()
mareaf = marea1.copy()
yldf[:, latidx, lonidx, :] = yld2[:, latidx, lonidx, :]
areaf[:, latidx, lonidx, :] = area2[:, latidx, lonidx, :]
mareaf[:, latidx, lonidx, :] = marea2[:, latidx, lonidx, :]
dvar = a1.copy()
dvar[:] = 1
dvar[logical_and(a1 < a2, latd <= 49)] = 2
dvar = masked_where(a1.mask, dvar)
with nc(outputfile, 'w') as f:
ice_land[i] = all[i].ice_land
icec[i] = all[i].ice_sst
sst[i] = all[i].sst
# create logical masks:
unknown = ma.masked_array(
ice_land > -1) # unknown points, which starts as all of them
#include coast points as being land (sidelobe issues)
landmask = ma.masked_array(ice_land >= 157)
watermask = ma.masked_array(ice_land < 100)
icemask = ma.masked_array(icec > 0)
#Distinguish between water and ice-covered water
not_ice = np.logical_not(icemask)
watermask = ma.logical_and(watermask, not_ice)
# Get the indices of the 'true' points
mland = landmask.nonzero()
water = watermask.nonzero()
iceindices = icemask.nonzero()
nicepts = len(iceindices[0])
nlandpts = len(mland[0])
nwaterpts = len(water[0])
#---------------------------------------------------------------------
# All data read in and apportioned,
# ice, land, water, and unknown pts. masks defined
#
#---------------------------------------------------------------------
print("n ice, land, water, nobs ", nicepts, nlandpts, nwaterpts, nobs)
def press2alt(arg, P0=None, T0=None, missing=1e+20, invert=0):
"""Calculate elevation given pressure (or vice versa).
Calculations are made assuming that the temperature distribution
follows the 1976 Standard Atmosphere. Technically the standard
atmosphere defines temperature distribution as a function of
geopotential altitude, and this routine actually calculates geo-
potential altitude rather than geometric altitude.
Method Positional Argument:
* arg: Numeric floating point vector of any shape and size, or a
Numeric floating point scalar. If invert=0 (the default), arg
is air pressure [hPa]. If invert=1, arg is elevation [m].
Method Keyword Arguments:
* P0: Pressure [hPa] at the surface (altitude equals 0). Numeric
floating point vector of same size and shape as arg or a scalar.
Default of keyword is set to None, in which case the routine
uses the value of instance attribute sea_level_press (converted
to hPa) from the AtmConst class. Keyword value is used if the
keyword is set in the function call. This keyword cannot have
any missing values.
* T0: Temperature [K] at the surface (altitude equals 0). Numeric
floating point vector of same size and shape as arg or a scalar.
Default of keyword is set to None, in which case the routine uses
the value of instance attribute sea_level_temp from the AtmConst
class. Keyword value is used if the keyword is set in the func-
tion call. This keyword cannot have any missing values.
* missing: If arg has missing values, this is the missing value
value. Floating point scalar. Default is 1e+20.
* invert: If set to 1, function calculates pressure [hPa] from
altitude [m]. In that case, positional input variable arg is
altitude [m] and the output is pressure [hPa]. Default value of
invert=0, which means the function calculates altitude given
pressure.
Output:
* If invert=0 (the default), output is elevation [m] at each
element of arg, relative to the surface. If invert=1, output
is the air pressure [hPa]. Numeric floating point array of
the same size and shape as arg.
If there are any missing values in output, those values are set
to the value in argument missing from the input. If there are
missing values in the output due to math errors and missing is
set to None, output will fill those missing values with the MA
default value of 1e+20.
References:
* Carmichael, Ralph (2003): "Definition of the 1976 Standard Atmo-
sphere to 86 km," Public Domain Aeronautical Software (PDAS).
URL: http://www.pdas.com/coesa.htm.
* Wallace, J. M., and P. V. Hobbs (1977): Atmospheric Science:
An Introductory Survey. San Diego, CA: Academic Press, ISBN
0-12-732950-1, pp. 60-61.
Examples:
(1) Calculating altitude given pressure:
>>> from press2alt import press2alt
>>> import Numeric as N
>>> press = N.array([200., 350., 850., 1e+20, 50.])
>>> alt = press2alt(press, missing=1e+20)
>>> ['%.7g' % alt[i] for i in range(5)]
['11783.94', '8117.19', '1457.285', '1e+20', '20575.96']
(2) Calculating pressure given altitude:
>>> alt = N.array([0., 10000., 15000., 20000., 50000.])
>>> press = press2alt(alt, missing=1e+20, invert=1)
>>> ['%.7g' % press[i] for i in range(5)]
['1013.25', '264.3589', '120.443', '54.74718', '0.7593892']
(3) Input is a Numeric floating point scalar, and using a keyword
set surface pressure to a different scalar:
>>> alt = press2alt(N.array(850.), P0=1000.)
>>> ['%.7g' % alt[0]]
['1349.778']
"""
import numpy as N
import numpy.ma as MA
#jfp was import MA
#jfp was import Numeric as N
from atmconst import AtmConst
from is_numeric_float import is_numeric_float
#- Check input is of the correct type:
if is_numeric_float(arg) != 1:
raise TypeError, "press2alt: Arg not Numeric floating"
#- Import general constants and set additional constants. h1_std
# is the lower limit of the Standard Atmosphere layer geopoten-
# tial altitude [m], h2_std is the upper limit [m] of the layer,
# and dT/dh is the temperature gradient (i.e. negative of the
# lapse rate) [K/m]:
const = AtmConst()
h1_std = N.array([0., 11., 20., 32., 47., 51., 71.]) * 1000.
h2_std = N.array(MA.concatenate([h1_std[1:], [84.852 * 1000.]]))
dTdh_std = N.array([-6.5, 0.0, 1.0, 2.8, 0.0, -2.8, -2.0]) / 1000.
#- Prep arrays for masked array calculation and set conditions
# at sea-level. Pressures are in hPa and temperatures in K.
# Sea-level conditions arrays are same shape/size as P_or_z.
# If input argument is a scalar, make the local variable used
# for calculations a 1-element vector:
if missing == None: P_or_z = MA.masked_array(arg)
else: P_or_z = MA.masked_values(arg, missing, copy=0)
if P_or_z.shape == ():
P_or_z = MA.reshape(P_or_z, (1, ))
if P0 == None:
#jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
P0_use = MA.zeros(P_or_z.shape) \
+ (const.sea_level_press / 100.)
else:
#jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
P0_use = MA.zeros(P_or_z.shape) \
+ MA.masked_array(P0)
if T0 == None:
#jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
T0_use = MA.zeros(P_or_z.shape) \
+ const.sea_level_temp
else:
#jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
T0_use = MA.zeros(P_or_z.shape) \
+ MA.masked_array(T0)
#- Calculate P and T for the boundaries of the 7 layers of the
# Standard Atmosphere for the given P0 and T0 (layer 0 goes from
# P0 to P1, layer 1 from P1 to P2, etc.). These are given as
# 8 element dictionaries P_std and T_std where the key is the
# location (P_std[0] is at the bottom of layer 0, P_std[1] is the
# top of layer 0 and bottom of layer 1, ... and P_std[7] is the
# top of layer 6). Remember P_std and T_std are dictionaries but
# dTdh_std, h1_std, and h2_std are vectors:
P_std = {0: P0_use}
T_std = {0: T0_use}
for i in range(len(h1_std)):
P_std[i+1] = _pfromz_MA( h2_std[i], -dTdh_std[i] \
, P_std[i], T_std[i], h1_std[i] )
T_std[i + 1] = T_std[i] + (dTdh_std[i] * (h2_std[i] - h1_std[i]))
#- Test input is within Standard Atmosphere limits:
if invert == 0:
tmp = MA.where(P_or_z < P_std[len(h1_std)], 1, 0)
if MA.sum(MA.ravel(tmp)) > 0:
raise ValueError, "press2alt: Pressure out-of-range"
else:
tmp = MA.where(P_or_z > MA.maximum(h2_std), 1, 0)
if MA.sum(MA.ravel(tmp)) > 0:
raise ValueError, "press2alt: Altitude out-of-range"
#- What layer number is each element of P_or_z in?
P_or_z_layer = MA.zeros(P_or_z.shape)
if invert == 0:
for i in range(len(h1_std)):
tmp = MA.where( MA.logical_and( (P_or_z <= P_std[i]) \
, (P_or_z > P_std[i+1]) ) \
, i, 0 )
P_or_z_layer += tmp
else:
for i in range(len(h1_std)):
tmp = MA.where( MA.logical_and( (P_or_z >= h1_std[i]) \
, (P_or_z < h2_std[i]) ) \
, i, 0 )
P_or_z_layer += tmp
#- Fill in the bottom-of-the-layer variables and the lapse rate
# for the layers that the levels are in. The *_actual variables
# are the values of dTdh, P_bott, etc. for each element in the
# P_or_z_flat array:
P_or_z_flat = MA.ravel(P_or_z)
P_or_z_flat_mask = P_or_z_flat.mask
if P_or_z_flat.mask == False:
P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape)
#jfp was:
#if P_or_z_flat.mask() == None:
# P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape)
#else:
# P_or_z_flat_mask = P_or_z_flat.mask()
P_or_z_layer_flat = MA.ravel(P_or_z_layer)
#jfp was dTdh_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
#jfp was P_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
#jfp was T_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
#jfp was z_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
dTdh_actual = MA.zeros(P_or_z_flat.shape)
P_bott_actual = MA.zeros(P_or_z_flat.shape)
T_bott_actual = MA.zeros(P_or_z_flat.shape)
z_bott_actual = MA.zeros(P_or_z_flat.shape)
for i in xrange(MA.size(P_or_z_flat)):
if P_or_z_flat_mask[i] != 1:
layer_number = P_or_z_layer_flat[i]
dTdh_actual[i] = dTdh_std[layer_number]
P_bott_actual[i] = MA.ravel(P_std[layer_number])[i]
T_bott_actual[i] = MA.ravel(T_std[layer_number])[i]
z_bott_actual[i] = h1_std[layer_number]
else:
dTdh_actual[i] = MA.masked
P_bott_actual[i] = MA.masked
T_bott_actual[i] = MA.masked
z_bott_actual[i] = MA.masked
#- Calculate pressure/altitude from altitude/pressure (output is
# a flat array):
if invert == 0:
output = _zfromp_MA( P_or_z_flat, -dTdh_actual \
, P_bott_actual, T_bott_actual, z_bott_actual )
else:
output = _pfromz_MA( P_or_z_flat, -dTdh_actual \
, P_bott_actual, T_bott_actual, z_bott_actual )
#- Return output as same shape as input positional argument:
return MA.filled(MA.reshape(output, arg.shape), missing)
ice_land[i] = all[i].ice_land
icec[i] = all[i].ice_sst
sst[i] = all[i].sst
# create logical masks:
unknown = ma.masked_array(
ice_land > -1) # unknown points, which starts as all of them
#include coast points as being land (sidelobe issues)
icemask = ma.masked_array(icec > 0)
landmask = ma.masked_array(ice_land >= 157)
watermask = ma.masked_array(ice_land < 100)
#Distinguish between water and ice-covered water
not_ice = np.logical_not(icemask)
watermask = ma.logical_and(watermask, not_ice)
# Get the indices of the 'true' points
iceindices = icemask.nonzero()
mland = landmask.nonzero()
water = watermask.nonzero()
nicepts = len(iceindices[0])
nlandpts = len(mland[0])
nwaterpts = len(water[0])
#---------------------------------------------------------------------
# All data read in and apportioned,
# ice, land, water, and unknown pts. masks defined
#
#---------------------------------------------------------------------
print("n ice, land, water, nobs ", nicepts, nlandpts, nwaterpts, nobs)
# load latitude, longitude information from first file if var in['sss'] and not model_data: first_file = xr.open_dataset(Data_directory+File_names[0],decode_times=False) else: first_file = xr.open_dataset(Data_directory+File_names[0]) #Dataset(Data_directory+File_names[0]) if coarse: Lat_vector_glob = first_file[Lat_cdf_name].values[::ndp] #first_file.variables[Lat_cdf_name][::ndp] Lon_vector_glob = first_file[Lon_cdf_name].values[::ndp] #first_file.variables[Lon_cdf_name][::ndp] else: Lat_vector_glob = first_file[Lat_cdf_name].values #first_file.variables[Lat_cdf_name][:] Lon_vector_glob = first_file[Lon_cdf_name].values #first_file.variables[Lon_cdf_name][:] if (coarse_ave and ndp not in ['0.25']) or (profiles and ndp not in ['0.25']): Lat_vector_glob=Lat_vector_glob[:,0] Lon_vector_glob=Lon_vector_glob[0,:] if profiles: Lat_vector = Lat_vector_glob[ma.where(ma.logical_and(Lat_vector_glob>=Lat_range[0],Lat_vector_glob<=Lat_range[1]))] Lon_vector = Lon_vector_glob[ma.where(ma.logical_and(Lon_vector_glob>=Lon_range[0],Lon_vector_glob<=Lon_range[1]))] else: Lat_vector = Lat_vector_glob[ma.where(ma.logical_and(Lat_vector_glob>Lat_range[0],Lat_vector_glob<Lat_range[1]))] Lon_vector = Lon_vector_glob[ma.where(ma.logical_and(Lon_vector_glob>Lon_range[0],Lon_vector_glob<Lon_range[1]))] # Num_lats_global = len(Lat_vector); Num_lons_global = len(Lon_vector); Num_pts_global = Num_lats_global * Num_lons_global; Lat_grid,Lon_grid = np.meshgrid(Lat_vector,Lon_vector); Lon_grid = Lon_grid.T # transpose to suit rows/cols matrix format Lat_grid = Lat_grid.T # note: field is upside down for image plots # # load time info from first file if not profiles and not coarse_ave and not smooth and var not in ['sss']: Time_vector = first_file[Time_cdf_name].values #first_file.variables[Time_cdf_name][:]
area = masked_where(mk, area) # mask
areasum1 = areasum1 * area / 100.
with nc(weightfile) as f:
areair2 = f.variables['irrigated'][:]
arearf2 = f.variables['rainfed'][:]
areair2 = resize(areair2, sh)
arearf2 = resize(arearf2, sh)
areasum2 = areair2 + arearf2
areair1 = masked_array(zeros(sh), mask = mk)
arearf1 = masked_array(zeros(sh), mask = mk)
# no data -> all rainfed
nodata = logical_and(~mk, logical_or(areasum2.mask, areasum2 == 0))
idx1, idx2, idx3 = where(nodata)
areair1[idx1, idx2, idx3] = 0.
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3]
# has data
hasdata = logical_and(~mk, logical_not(nodata))
idx1, idx2, idx3 = where(hasdata)
areair1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * areair2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * arearf2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
with nc(outputfile, 'w') as f:
f.createDimension('time', sh[0])
tvar = f.createVariable('time', 'i4', 'time')
tvar[:] = range(0, sh[0])
tvar.units = 'years since %d' % year
def press2alt(arg, P0=None, T0=None, missing=1e+20, invert=0):
"""Calculate elevation given pressure (or vice versa).
Calculations are made assuming that the temperature distribution
follows the 1976 Standard Atmosphere. Technically the standard
atmosphere defines temperature distribution as a function of
geopotential altitude, and this routine actually calculates geo-
potential altitude rather than geometric altitude.
Method Positional Argument:
* arg: Numeric floating point vector of any shape and size, or a
Numeric floating point scalar. If invert=0 (the default), arg
is air pressure [hPa]. If invert=1, arg is elevation [m].
Method Keyword Arguments:
* P0: Pressure [hPa] at the surface (altitude equals 0). Numeric
floating point vector of same size and shape as arg or a scalar.
Default of keyword is set to None, in which case the routine
uses the value of instance attribute sea_level_press (converted
to hPa) from the AtmConst class. Keyword value is used if the
keyword is set in the function call. This keyword cannot have
any missing values.
* T0: Temperature [K] at the surface (altitude equals 0). Numeric
floating point vector of same size and shape as arg or a scalar.
Default of keyword is set to None, in which case the routine uses
the value of instance attribute sea_level_temp from the AtmConst
class. Keyword value is used if the keyword is set in the func-
tion call. This keyword cannot have any missing values.
* missing: If arg has missing values, this is the missing value
value. Floating point scalar. Default is 1e+20.
* invert: If set to 1, function calculates pressure [hPa] from
altitude [m]. In that case, positional input variable arg is
altitude [m] and the output is pressure [hPa]. Default value of
invert=0, which means the function calculates altitude given
pressure.
Output:
* If invert=0 (the default), output is elevation [m] at each
element of arg, relative to the surface. If invert=1, output
is the air pressure [hPa]. Numeric floating point array of
the same size and shape as arg.
If there are any missing values in output, those values are set
to the value in argument missing from the input. If there are
missing values in the output due to math errors and missing is
set to None, output will fill those missing values with the MA
default value of 1e+20.
References:
* Carmichael, Ralph (2003): "Definition of the 1976 Standard Atmo-
sphere to 86 km," Public Domain Aeronautical Software (PDAS).
URL: http://www.pdas.com/coesa.htm.
* Wallace, J. M., and P. V. Hobbs (1977): Atmospheric Science:
An Introductory Survey. San Diego, CA: Academic Press, ISBN
0-12-732950-1, pp. 60-61.
Examples:
(1) Calculating altitude given pressure:
>>> from press2alt import press2alt
>>> import Numeric as N
>>> press = N.array([200., 350., 850., 1e+20, 50.])
>>> alt = press2alt(press, missing=1e+20)
>>> ['%.7g' % alt[i] for i in range(5)]
['11783.94', '8117.19', '1457.285', '1e+20', '20575.96']
(2) Calculating pressure given altitude:
>>> alt = N.array([0., 10000., 15000., 20000., 50000.])
>>> press = press2alt(alt, missing=1e+20, invert=1)
>>> ['%.7g' % press[i] for i in range(5)]
['1013.25', '264.3589', '120.443', '54.74718', '0.7593892']
(3) Input is a Numeric floating point scalar, and using a keyword
set surface pressure to a different scalar:
>>> alt = press2alt(N.array(850.), P0=1000.)
>>> ['%.7g' % alt[0]]
['1349.778']
"""
import numpy as N
import numpy.ma as MA
#jfp was import MA
#jfp was import Numeric as N
from atmconst import AtmConst
from is_numeric_float import is_numeric_float
#- Check input is of the correct type:
if is_numeric_float(arg) != 1:
raise TypeError, "press2alt: Arg not Numeric floating"
#- Import general constants and set additional constants. h1_std
# is the lower limit of the Standard Atmosphere layer geopoten-
# tial altitude [m], h2_std is the upper limit [m] of the layer,
# and dT/dh is the temperature gradient (i.e. negative of the
# lapse rate) [K/m]:
const = AtmConst()
h1_std = N.array([0., 11., 20., 32., 47., 51., 71.]) * 1000.
h2_std = N.array( MA.concatenate([h1_std[1:], [84.852*1000.]]) )
dTdh_std = N.array([-6.5, 0.0, 1.0, 2.8, 0.0, -2.8, -2.0]) / 1000.
#- Prep arrays for masked array calculation and set conditions
# at sea-level. Pressures are in hPa and temperatures in K.
# Sea-level conditions arrays are same shape/size as P_or_z.
# If input argument is a scalar, make the local variable used
# for calculations a 1-element vector:
if missing == None: P_or_z = MA.masked_array(arg)
else: P_or_z = MA.masked_values(arg, missing, copy=0)
if P_or_z.shape == ():
P_or_z = MA.reshape(P_or_z, (1,))
if P0 == None:
#jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
P0_use = MA.zeros(P_or_z.shape) \
+ (const.sea_level_press / 100.)
else:
#jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
P0_use = MA.zeros(P_or_z.shape) \
+ MA.masked_array(P0)
if T0 == None:
#jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
T0_use = MA.zeros(P_or_z.shape) \
+ const.sea_level_temp
else:
#jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \
T0_use = MA.zeros(P_or_z.shape) \
+ MA.masked_array(T0)
#- Calculate P and T for the boundaries of the 7 layers of the
# Standard Atmosphere for the given P0 and T0 (layer 0 goes from
# P0 to P1, layer 1 from P1 to P2, etc.). These are given as
# 8 element dictionaries P_std and T_std where the key is the
# location (P_std[0] is at the bottom of layer 0, P_std[1] is the
# top of layer 0 and bottom of layer 1, ... and P_std[7] is the
# top of layer 6). Remember P_std and T_std are dictionaries but
# dTdh_std, h1_std, and h2_std are vectors:
P_std = {0:P0_use}
T_std = {0:T0_use}
for i in range(len(h1_std)):
P_std[i+1] = _pfromz_MA( h2_std[i], -dTdh_std[i] \
, P_std[i], T_std[i], h1_std[i] )
T_std[i+1] = T_std[i] + ( dTdh_std[i] * (h2_std[i]-h1_std[i]) )
#- Test input is within Standard Atmosphere limits:
if invert == 0:
tmp = MA.where(P_or_z < P_std[len(h1_std)], 1, 0)
if MA.sum(MA.ravel(tmp)) > 0:
raise ValueError, "press2alt: Pressure out-of-range"
else:
tmp = MA.where(P_or_z > MA.maximum(h2_std), 1, 0)
if MA.sum(MA.ravel(tmp)) > 0:
raise ValueError, "press2alt: Altitude out-of-range"
#- What layer number is each element of P_or_z in?
P_or_z_layer = MA.zeros(P_or_z.shape)
if invert == 0:
for i in range(len(h1_std)):
tmp = MA.where( MA.logical_and( (P_or_z <= P_std[i]) \
, (P_or_z > P_std[i+1]) ) \
, i, 0 )
P_or_z_layer += tmp
else:
for i in range(len(h1_std)):
tmp = MA.where( MA.logical_and( (P_or_z >= h1_std[i]) \
, (P_or_z < h2_std[i]) ) \
, i, 0 )
P_or_z_layer += tmp
#- Fill in the bottom-of-the-layer variables and the lapse rate
# for the layers that the levels are in. The *_actual variables
# are the values of dTdh, P_bott, etc. for each element in the
# P_or_z_flat array:
P_or_z_flat = MA.ravel(P_or_z)
P_or_z_flat_mask = P_or_z_flat.mask
if P_or_z_flat.mask==False:
P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape)
#jfp was:
#if P_or_z_flat.mask() == None:
# P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape)
#else:
# P_or_z_flat_mask = P_or_z_flat.mask()
P_or_z_layer_flat = MA.ravel(P_or_z_layer)
#jfp was dTdh_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
#jfp was P_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
#jfp was T_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
#jfp was z_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float)
dTdh_actual = MA.zeros(P_or_z_flat.shape)
P_bott_actual = MA.zeros(P_or_z_flat.shape)
T_bott_actual = MA.zeros(P_or_z_flat.shape)
z_bott_actual = MA.zeros(P_or_z_flat.shape)
for i in xrange(MA.size(P_or_z_flat)):
if P_or_z_flat_mask[i] != 1:
layer_number = P_or_z_layer_flat[i]
dTdh_actual[i] = dTdh_std[layer_number]
P_bott_actual[i] = MA.ravel(P_std[layer_number])[i]
T_bott_actual[i] = MA.ravel(T_std[layer_number])[i]
z_bott_actual[i] = h1_std[layer_number]
else:
dTdh_actual[i] = MA.masked
P_bott_actual[i] = MA.masked
T_bott_actual[i] = MA.masked
z_bott_actual[i] = MA.masked
#- Calculate pressure/altitude from altitude/pressure (output is
# a flat array):
if invert == 0:
output = _zfromp_MA( P_or_z_flat, -dTdh_actual \
, P_bott_actual, T_bott_actual, z_bott_actual )
else:
output = _pfromz_MA( P_or_z_flat, -dTdh_actual \
, P_bott_actual, T_bott_actual, z_bott_actual )
#- Return output as same shape as input positional argument:
return MA.filled( MA.reshape(output, arg.shape), missing )
def _calc_half_htc(self, prcp_values, temp_values, threshold, start, end):
""" Calculates Selyaninov's hydrothermal coeffcient for a given time period.
Arguments:
prcp_values -- daily total precipitation values
temp_values -- daily mean temperature values
threshold -- threshold temperature (10C, by default)
start -- first index in the time grid, we start analysis from it (inclusive)
end -- last index in the time grid, we stop analysis before it (NOT inclusive)
If start <= stop algorithm runs forth in time. Runs back, otherwise.
Returns:
result -- array of Selyaninov's hydrothermal coefficient values
"""
# The idea is to subtract threshold from temperature values
# and to catch cases when values change sign from negative to positive (a transition)
# i.e., pass x-axis upwards.
# Each crossing corresponds to a possible beginning/ending
# (depends on the direction of the analysis) of the vegetation period.
# We sum values between crossings (in segments) and analyse sums according to Ped' (1951).
# Define a function to detect False->True transition.
trans = lambda a, b: ma.logical_and(ma.logical_not(a), b)
# Create shape for new arrays without time dimmension.
dims = list(temp_values.shape)
_ = dims.pop(0)
# Define some useful arrays.
temp_sums_1 = ma.zeros(dims) # Temperature sums for the first segment.
temp_sums_2 = ma.zeros(dims) # Temperature sums for the second segment.
temp_sums_3 = ma.zeros(dims) # Temperature sums for the third segment.
temp_cnt_1 = ma.zeros(dims) # Temperature counter for the first segment.
temp_cnt_2 = ma.zeros(dims) # Temperature counter for the second segment.
temp_cnt_3 = ma.zeros(dims) # Temperature counter for the third segment.
temp_total = ma.zeros(dims) # Temperature total sum for a vegetation period.
prcp_sums_1 = ma.zeros(dims) # Precipitation sums for the first segment.
prcp_sums_2 = ma.zeros(dims) # Precipitation sums for the second segment.
prcp_sums_3 = ma.zeros(dims) # Precipitation sums for the third segment.
prcp_total = ma.zeros(dims) # Precipitation total sum for a vegetation period.
trans_mask_1 = ma.zeros(dims) # Mask of cells in the first segment state
# (i.e., where the first transition was detected).
trans_mask_2 = ma.zeros(dims) # Mask of cells in the second segment state.
trans_mask_3 = ma.zeros(dims) # Mask of cells in the third segment state.
prev_pos_mask = ma.zeros(dims) # Matrix of previous positive mask state.
step = 1 if start <= end else -1 # Set step depending on start and stop values.
# Search for upward transitions and calculate sums.
for i in range(start, end, step):
cur_temp = temp_values[i]
cur_prcp = prcp_values[i]
temp_deviation = cur_temp - threshold
temp_pos_mask = temp_deviation >= 0 # Mask of positive values.
cur_trans = trans(prev_pos_mask, temp_pos_mask) # Detect negative->positive transition.
prev_pos_mask = temp_pos_mask # Store current positive mask for the next iteration.
# Turn on the third state, if a transition is detected and the second state is on.
# When the state is on, it is never off (provided by a boolean 'or').
trans_mask_3 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_3), trans_mask_2)
# Turn on the second state, if a transition is detected and the first state is on.
trans_mask_2 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_2), trans_mask_1)
# Turn on the first state, if a transition is detected or leave it as is.
trans_mask_1 = ma.logical_or(cur_trans, trans_mask_1)
# Sum values in the first segment.
seg_1_mask = ma.logical_and(trans_mask_1, ma.logical_not(trans_mask_2)) # Only for cells in the first segemnt state.
temp_sums_1[seg_1_mask] += temp_deviation[seg_1_mask]
temp_cnt_1[seg_1_mask] += 1
prcp_sums_1[seg_1_mask] += cur_prcp[seg_1_mask]
# Sum values in the second segment.
seg_2_mask = ma.logical_and(trans_mask_2, ma.logical_not(trans_mask_3)) # Only for cells in the second segemnt state.
temp_sums_2[seg_2_mask] += temp_deviation[seg_2_mask]
temp_cnt_2[seg_2_mask] += 1
prcp_sums_2[seg_2_mask] += cur_prcp[seg_2_mask]
# Sum values in the third segment (in fact, everything after the second segment).
temp_sums_3[trans_mask_3] += temp_deviation[trans_mask_3]
temp_cnt_3[trans_mask_3] += 1
prcp_sums_3[trans_mask_3] += cur_prcp[trans_mask_3]
# Create some intermediate sums.
temp_sums_12 = temp_sums_1 + temp_sums_2 # S1+S2+S3+S4
temp_cnt_12 = temp_cnt_1 + temp_cnt_2
temp_sums_23 = temp_sums_2 + temp_sums_3 # S3+S4+... all the rest
temp_cnt_23 = temp_cnt_2 + temp_cnt_3
temp_sums_123 = temp_sums_12 + temp_sums_3 # S1+S2+S3+S4+... all the rest
temp_cnt_123 = temp_cnt_12 + temp_cnt_3
# Check Ped's conditions and calculate temperature sums for the vegetation period.
# By adding temp_cnt_...*threshold we restore original values of the temperature.
# If vegetation period starts at the first transition point.
start_at_seg_1 = ma.logical_and(temp_sums_1 >= 0, temp_sums_12 >= 0) # S1 >= S2 and S1-S2+S3 >= S4
temp_total[start_at_seg_1] = temp_sums_123[start_at_seg_1] + temp_cnt_123[start_at_seg_1] * threshold
prcp_total[start_at_seg_1] = prcp_sums_1[start_at_seg_1] + prcp_sums_2[start_at_seg_1] + prcp_sums_3[start_at_seg_1]
# If vegetation period starts at the second transition point.
start_at_seg_2 = ma.logical_and(temp_sums_1 < 0, temp_sums_2 >= 0) # S1 < S2 and S3-S4 >= 0
temp_total[start_at_seg_2] = temp_sums_23[start_at_seg_2] + temp_cnt_23[start_at_seg_2] * threshold
prcp_total[start_at_seg_2] = prcp_sums_2[start_at_seg_2] + prcp_sums_3[start_at_seg_2]
# If vegetation period starts at the third transition point.
start_at_seg_3 = ma.logical_or(ma.logical_and(temp_sums_1 < 0, temp_sums_2 < 0), # S1 < S2 and S3 < S4
ma.logical_and(ma.logical_and(temp_sums_1 > 0, temp_sums_2 < 0), # or
temp_sums_12 < 0)) # S3 < S4 and S1 > S2 and S1-S2+S3 < S4
temp_total[start_at_seg_3] = temp_sums_3[start_at_seg_3] + temp_cnt_3[start_at_seg_3] * threshold
prcp_total[start_at_seg_3] = prcp_sums_3[start_at_seg_3]
return (prcp_total, temp_total)
def contingency_table_4x4(obs, fcst, th, av, hy):
"""
:func:`contingency_table_4x4`: Contingency_table_4x4, for further
comprehensive study for the skill, Hit Rate(HR), Bias(BS), Threat
Score(TS),..etc and hit rates for different categories are calcul-
ated.We categorise the 2x2 condigency table to 4x4 contigency
table. Hit rate for different categories moderate, heavy and very
heavy is calculated. The 2x2 contigency table useful for forecast
verification. From 2x2 condigency table we can find the statistical
scores such as Hit Rate(HR), Bias(BS), Threat Score(TS), Odds Ratio
(ODR)...etc
Inputs:
obs -the observed values has to be a list(or whatever you decide)
fcst-the forecast values
th -the threshold value for which the contingency table needs to
be created (floating point value please!!)
av-the data points between 'th' and 'av' we say that data
point is moderate value
hy-the data points above 'hy' we say that the data point
is higher value
Outputs:
A list of values [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p
are entries of 4x4 contingency table
where a = No of values such that both observed and
predicted < threshold
b = No of values such that 'av'>observed is > threshold and
predicted < threshold
c = No of values such that 'av'>observed is < 'hy'
predicted < threshold
d = No of values such that observed is > 'hy'
predicted < threshold
e =No of values such that observed< threshold and
threshold<predicted < 'av'
f =No of values such that 'av'>observed is > threshold and
threshold<predicted < 'av'
g =No of values such that 'av'>observed is < 'hy' and
threshold<predicted < 'av'
h =No of values such that observed is > 'hy'and
threshold<predicted < 'av'
i=No of values such that observed< threshold and 'av'<predicted< 'hy'
j=No of values such that 'av'>observed is > thresholdand and
'av'<predicted< 'hy'
k=No of values such that 'av'>observed is < 'hy' and
'av'<predicted< 'hy'
l=No of values such that observed is > 'hy'and 'av'<predicted< 'hy'
m=No of values such that observed< threshold and predicted> 'hy'
n=No of values such that 'av'>observed is > thresholdand
and predicted> 'hy'
o=No of values such that 'av'>observed is < 'hy' and
p=No of values such that observed is > 'hy'and predicted> 'hy
A list of values A, B, C, D are list are entries of
4x4 contigency table
A = No of values such that both observed and predicted > threshold
B = No of values such that observed is < threshold and
predicted > threshold
C = No of values such that observed is > threshold and
predicted < threshold
D = No of values such that both observed and predicted < threshold
Usage: Comparing the statistical score, we can analyse the models
example:
From the contingency table the following statistics can be calculated.
HR=(a+d)/(a+b+c+d) (for 2x2 contigency table)
HR =(A+F+K+P)/N( for 4x4 contigency table, N is total all the
frequencies)
ETS=a/(a+b+c)
BS=(a+b)/(a+c)
ODR= (a*d)/(b*c)
N=(a+ b+ c+ d+ e+ f+ g+ h+ i+ j+ k+ l+ m+ n+ o+ p)
N(Oi)= um of entries of i th column of 4x4 contingency table.
N(Fi)= Sum of entries of i th row of 4x4 contingency table.
Reference:
"Statistical Methods in the Atmospheric Sciences",
Daniel S Wilks, ACADEMIC PRESS
"NCMRWF Report, Monsoon-2009 Performance of T254L64 Global
Assimilation-Forecast System"
Written by: Dileepkumar R,
JRF, IIT Delhi
Date: 29/02/2011
"""
import numpy
import numpy.ma as MA
# If 'obs' or 'fcst' are not list then an assertion error will show
#assert type(obs) is list," 'obs' is not a list"
#assert type(fcst) is list," 'fcst' is not a list"
obs=MA.array(obs)
fcst= MA.array(fcst)
#If we give 'obs' & 'fcst' of different dimention an assertion
#error will show
assert (obs.shape == fcst.shape), \
"Dimention of 'obs' and 'fcst' are different"
# Defining threshold
th = float(th)
av = float(av)
hy = float(hy)
#We are masking the observation < threshold
obs_1= MA.masked_where(obs<th, obs)
obs_1_mask= MA.getmask(obs_1)
# Masking the observation between "threshold" & "av"
condition_1o= MA.logical_and(th<=obs, obs< av)
obs_2 = MA.masked_where(condition_1o, obs)
obs_2_mask = MA.getmask(obs_2)
# Masking the observation between "av" & "hv"
condition_2o= MA.logical_and(obs<hy, obs>= av)
obs_3 = MA.masked_where(condition_2o, obs)
obs_3_mask = MA.getmask(obs_3)
# Masking the observation > hy
obs_4=MA.masked_where(obs>=hy, obs)
obs_4_mask = MA.getmask(obs_4)
#Masking the forecast < threshold
fcst_1= MA.masked_where(fcst<th, fcst)
fcst_1_mask= MA.getmask(fcst_1)
# Masking the forecast between "threshold" & "av"
condition_1f= MA.logical_and(th<=fcst, fcst< av)
fcst_2 = MA.masked_where(condition_1f, fcst)
fcst_2_mask = MA.getmask(fcst_2)
# Masking the forecast between "av" & "hv"
condition_2f= MA.logical_and(fcst<hy, fcst>= av)
fcst_3 = MA.masked_where(condition_2f, fcst)
fcst_3_mask = MA.getmask(fcst_3)
# Masking the forecast > hy
fcst_4=MA.masked_where(fcst>=hy, fcst)
fcst_4_mask = MA.getmask(fcst_4)
#o1 , f1 ; less than threshold(Th) is no rain
#o2, f2 ; greater than Th and less than 'av' is light to moderate rain
#o3, f3 ; greater than 'av' and less than 'hy' is heavy rain
#o4, f4 ; greater than 'hy' is very heavy rain.
# Masking such that forcast < threshold vs other four condition
# for observation
f1o1=MA.logical_and(fcst_1_mask, obs_1_mask)# for a
f1o2=MA.logical_and(fcst_1_mask, obs_2_mask)# for b
f1o3=MA.logical_and(fcst_1_mask, obs_3_mask)# for c
f1o4=MA.logical_and(fcst_1_mask, obs_4_mask)# for d
# Masking such that forecast between "threshold" & "av" vs other four
# condition for observation
f2o1=MA.logical_and(fcst_2_mask, obs_1_mask)# for e
f2o2=MA.logical_and(fcst_2_mask, obs_2_mask)# for f
f2o3=MA.logical_and(fcst_2_mask, obs_3_mask)# for g
f2o4=MA.logical_and(fcst_2_mask, obs_4_mask)# for h
# Masking such that forecast between "av" & "hv" vs other four condition
# for observation
f3o1=MA.logical_and(fcst_3_mask, obs_1_mask)# for i
f3o2=MA.logical_and(fcst_3_mask, obs_2_mask)# for j
f3o3=MA.logical_and(fcst_3_mask, obs_3_mask)# for k
f3o4=MA.logical_and(fcst_3_mask, obs_4_mask)# for l
# Making an array of "1"s of order same as observed or predicted
f4o1=MA.logical_and(fcst_4_mask, obs_1_mask)# for m
f4o2=MA.logical_and(fcst_4_mask, obs_2_mask)# for n
f4o3=MA.logical_and(fcst_4_mask, obs_3_mask)# for o
f4o4=MA.logical_and(fcst_4_mask, obs_4_mask)# for p
#Making an array of "1"s of order same as 'obs' or 'fcst'
dummy = numpy.ones(obs.shape, numpy.int)
# Calculating the total no. of entries
total = float(numpy.sum(dummy))
# Masking the entries of dummy array with respect to the above conditions
a_mat = MA.array(dummy, mask=f1o1)
b_mat = MA.array(dummy, mask=f1o2)
c_mat = MA.array(dummy, mask=f1o3)
d_mat = MA.array(dummy, mask=f1o4)
e_mat = MA.array(dummy, mask=f2o1)
f_mat = MA.array(dummy, mask=f2o2)
g_mat = MA.array(dummy, mask=f2o3)
h_mat = MA.array(dummy, mask=f2o4)
i_mat = MA.array(dummy, mask=f3o1)
j_mat = MA.array(dummy, mask=f3o2)
k_mat = MA.array(dummy, mask=f3o3)
l_mat = MA.array(dummy, mask=f3o4)
m_mat = MA.array(dummy, mask=f4o1)
n_mat = MA.array(dummy, mask=f4o2)
o_mat = MA.array(dummy, mask=f4o3)
p_mat = MA.array(dummy, mask=f4o4)
# Subtact this from total (WHY???)--->Actually we are masking the needed
# values, therefore to compute the actual value we have to substract it
# from total
a = total - numpy.sum(a_mat)
b = total - numpy.sum(b_mat)
c = total - numpy.sum(c_mat)
d = total - numpy.sum(d_mat)
e = total - numpy.sum(e_mat)
f = total - numpy.sum(f_mat)
g = total - numpy.sum(g_mat)
h = total - numpy.sum(h_mat)
i = total - numpy.sum(i_mat)
j = total - numpy.sum(j_mat)
k = total - numpy.sum(k_mat)
l = total - numpy.sum(l_mat)
m = total - numpy.sum(m_mat)
n = total - numpy.sum(n_mat)
o = total - numpy.sum(o_mat)
p = total - numpy.sum(p_mat)
#return [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ,p]
"""
From 4x4 contingency table we can create 2x2 contingency table
"""
A = f+g+h+j+k+l+n+o+p #A is total no of values both observed and
# forcast are > threshold
B = e+i+m # B is total number of values such that observed< threshold &
# predicted > threshold
C = b+c+d #C is total number of values such that observed> threshold &
# predicted < threshold
D = a #D is total no of values such that both observed and
# forcast are < threshold
# We are changing the entries of 4x4 & 2x2 as float and represnt as
# a matrix form
tb4= ([[a, b, c, d], [e, f, g, h], [i, j, k, l], [m, n, o, p]])
tb4 =numpy.array(tb4, dtype = float)
tb2= ([A, B], [C, D])
tb2 = numpy.array(tb2, dtype = float)
return tb4
from numpy.ma import logical_and from matplotlib.pyplot import subplots, show from numpy import array from nnet.prepare import LogTransform from utils.loading import get_training_data from utils.shuffling import shuffle from nndist.distance import DistanceFeatureGenerator N = 500 train, labels = get_training_data()[:2] train = train[logical_and(5 <= labels, labels <= 7), :] labels = labels[logical_and(5 <= labels, labels <= 7)] - 4 train, labels = shuffle(train, labels)[:2] train = train[:N, :] labels = labels[:N] print train.shape, train.dtype train = LogTransform().fit_transform(train) print train.shape, train.dtype gen = DistanceFeatureGenerator(n_neighbors=3, distance_p=2, nr_classes=3) train = gen.fit_transform(train, labels) print train.shape, train.dtype #train = MinMaxScaler(feature_range = (0, 3)).fit_transform(train) #print train.shape, train.dtype fig, ax = subplots(figsize=(7, 6)) colors = ['r', 'g', 'b']
def log_linear_vinterp(T, P, levs):
'''
# Author Charles Doutriaux
# Version 1.1
# Expect 2D field here so there''s no reorder which I suspect to do a memory leak
# email: [email protected]
# Converts a field from sigma levels to pressure levels
# Log linear interpolation
# Input
# T : temperature on sigma levels
# P : pressure field from TOP (level 0) to BOTTOM (last level)
# levs : pressure levels to interplate to (same units as P)
# Output
# t : temperature on pressure levels (levs)
# External: Numeric'''
import numpy.ma as MA
## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16
sh = P.shape
nsigma = sh[0] # Number of sigma levels
try:
nlev = len(levs) # Number of pressure levels
except:
nlev = 1 # if only one level len(levs) would breaks
t = []
for ilv in range(nlev): # loop through pressure levels
try:
lev = levs[ilv] # get value for the level
except:
lev = levs # only 1 level passed
# print ' ......... level:',lev
Pabv = MA.ones(P[0].shape, Numeric.Float)
Tabv = -Pabv # Temperature on sigma level Above
Tbel = -Pabv # Temperature on sigma level Below
Pbel = -Pabv # Pressure on sigma level Below
Pabv = -Pabv # Pressure on sigma level Above
for isg in range(1,
nsigma): # loop from second sigma level to last one
## print 'Sigma level #',isg
a = MA.greater(P[isg],
lev) # Where is the pressure greater than lev
b = MA.less(P[isg - 1], lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Tabv, Tbel
Pabv = MA.where(MA.logical_and(a, b), P[isg],
Pabv) # Pressure on sigma level Above
Tabv = MA.where(MA.logical_and(a, b), T[isg],
Tabv) # Temperature on sigma level Above
Pbel = MA.where(MA.logical_and(a, b), P[isg - 1],
Pbel) # Pressure on sigma level Below
Tbel = MA.where(MA.logical_and(a, b), T[isg - 1],
Tbel) # Temperature on sigma level Below
# end of for isg in range(1,nsigma)
# val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is
tl = MA.masked_where(
MA.equal(Pbel, -1.),
MA.log(lev / MA.absolute(Pbel)) / MA.log(Pabv / Pbel) *
(Tabv - Tbel) + Tbel) # Interpolation
t.append(tl) # add a level to the output
# end of for ilv in range(nlev)
return asMA(t).astype(Numeric.Float32) # convert t to an array
linewidth=0.25,
fontsize=20)
p2 = m2.drawmeridians(np.arange(60., 360., 60.),
labels=[False, False, False, True],
linewidth=0.25,
fontsize=20)
m2.fillcontinents(color='DarkGray', lake_color='LightGray', zorder=0)
#c2=m2.pcolormesh(Lon_vector.squeeze(),Lat_vector.squeeze(),Kb,cmap=cmap,norm=norm,latlon=True,rasterized=True)
#m2.plot(x2,y2, '.',color='gray', markersize=0.2, rasterized=True)
#
lo, la = np.meshgrid(Lon_vector.squeeze(), Lat_vector.squeeze())
#x3,y3=m2(lo,la)
Kxrange = np.percentile(Kx[np.where(Kx > 10)], [1, 99])
Kyrange = np.percentile(Ky[np.where(Ky > 10)], [1, 99])
jinds, iinds = np.where(
ma.logical_and(Kx > Kxrange[0], Kx < Kxrange[1]))
Kx2 = griddata((lo[jinds, iinds], la[jinds, iinds]), Kx[jinds, iinds],
(lo, la))
jinds, iinds = np.where(
ma.logical_and(Ky > Kyrange[0], Ky < Kyrange[1]))
Ky2 = griddata((lo[jinds, iinds], la[jinds, iinds]), Ky[jinds, iinds],
(lo, la))
newmask = Kb.mask.copy()
newmask2 = Kb.mask.copy()
newmask3 = Kb.mask.copy()
newmask2[np.where(Kx < Kxrange[0])] = 1
# newmask2[np.where(Kx>Kxrange[1])]=1
newmask2[np.where(Ky < Kyrange[0])] = 1
# newmask2[np.where(Ky>Kyrange[1])]=1
newmask2[np.where(icemask)] = 1
newmask[np.where(icemask)] = 1
