def add_ellipse(ax, x, y, kx, ky, mask, Kxrange, Kyrange, color='k'): ''' ''' if (not np.isnan(kx)) and ( not np.isnan(ky)) and (not mask) and ma.logical_and( ky > Kyrange[0], ky < Kyrange[-1]) and ma.logical_and( kx > Kxrange[0], kx < Kxrange[-1]): # and abs(y)<60: wax = 2.5 #*np.cos(y*np.pi/180.) hax = wax * ky / kx #if hax>1: # hax=hax/np.log(hax) # wax=wax/np.log(hax) if hax > 2.0 * wax: hax = 2.0 * wax #*hax/hax wax = hax * kx / ky #2.0*wax/hax #angle=np.arctan(ky/kx)*180/np.pi-90 angle = 0 e = Ellipse(xy=(x, y), width=wax, height=hax, angle=angle, facecolor='none', edgecolor=color, rasterized=True) ax.add_artist(e) ax.plot(x, y, '.k', markersize=1)
def sigclip(x, Nsig=3.0, eps=1e-6, ids=None): xo = x.mean() xlo = x.mean() - Nsig * x.std(ddof=1) xhi = x.mean() + Nsig * x.std(ddof=1) #x = numpy.clip(x,xlo,xhi) idx = numpy.where(logical_and(x > xlo, x < xhi)) xn = x[idx] if x.std(ddof=1) == 0.0: if ids: idx = numpy.indices(x.shape)[0] print("idx", idx) return idx else: return x i = 0 while abs(1 - old_div(xo, xn.mean())) > eps: xo = xn.mean() xlo = xo - Nsig * xn.std(ddof=1) xhi = xo + Nsig * xn.std(ddof=1) idx = numpy.where(logical_and(x > xlo, x < xhi)) xn = x[idx] i = i + 1 if ids: return idx else: #return x return xn
def sigclip_w(x, wt=None, Nsig=2.0, eps=1e-6, ids=None): i = 1 s = stats(x, wt) xo = s.mean std = s.std xlo = s.mean - Nsig * std xhi = s.mean + Nsig * std idx = numpy.where(logical_and(x > xlo, x < xhi)) xn = x[idx] wn = wt[idx] i = 0 #print i,xo,stats(xn,wn).mean,stats(xn,wn).std, xn.mean(), xn.stddev(), len(xn),xlo,xhi while abs(1 - old_div(xo, stats(xn, wn).mean)) > eps: #print i,xo,stats(xn,wn).mean,stats(xn,wn).std, xn.mean(), xn.stddev(), len(xn),xlo,xhi s = stats(xn, wn) xo = s.mean xlo = xo - Nsig * s.std xhi = xo + Nsig * s.std # Clip it idx = numpy.where(logical_and(x > xlo, x < xhi)) xn = x[idx] wn = wt[idx] i = i + 1 if ids: return idx else: return xn, wn
def log_linear_vinterp(T,P,levs): ''' # Author Charles Doutriaux # Version 1.1 # Expect 2D field here so there''s no reorder which I suspect to do a memory leak # email: [email protected] # Converts a field from sigma levels to pressure levels # Log linear interpolation # Input # T : temperature on sigma levels # P : pressure field from TOP (level 0) to BOTTOM (last level) # levs : pressure levels to interplate to (same units as P) # Output # t : temperature on pressure levels (levs) # External: Numeric''' import numpy.ma as MA ## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16 sh=P.shape nsigma=sh[0] # Number of sigma levels try: nlev=len(levs) # Number of pressure levels except: nlev=1 # if only one level len(levs) would breaks t=[] for ilv in range(nlev): # loop through pressure levels try: lev=levs[ilv] # get value for the level except: lev=levs # only 1 level passed # print ' ......... level:',lev Pabv=MA.ones(P[0].shape,Numeric.Float) Tabv=-Pabv # Temperature on sigma level Above Tbel=-Pabv # Temperature on sigma level Below Pbel=-Pabv # Pressure on sigma level Below Pabv=-Pabv # Pressure on sigma level Above for isg in range(1,nsigma): # loop from second sigma level to last one ## print 'Sigma level #',isg a = MA.greater(P[isg], lev) # Where is the pressure greater than lev b = MA.less(P[isg-1],lev) # Where is the pressure less than lev # Now looks if the pressure level is in between the 2 sigma levels # If yes, sets Pabv, Pbel and Tabv, Tbel Pabv=MA.where(MA.logical_and(a,b),P[isg],Pabv) # Pressure on sigma level Above Tabv=MA.where(MA.logical_and(a,b),T[isg],Tabv) # Temperature on sigma level Above Pbel=MA.where(MA.logical_and(a,b),P[isg-1],Pbel) # Pressure on sigma level Below Tbel=MA.where(MA.logical_and(a,b),T[isg-1],Tbel) # Temperature on sigma level Below # end of for isg in range(1,nsigma) # val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is tl=MA.masked_where(MA.equal(Pbel,-1.),MA.log(lev/MA.absolute(Pbel))/MA.log(Pabv/Pbel)*(Tabv-Tbel)+Tbel) # Interpolation t.append(tl) # add a level to the output # end of for ilv in range(nlev) return asMA(t).astype(Numeric.Float32) # convert t to an array
def __init__(self, filename, varnames=None, lats=None, lons=None, incl_global=False): f = nc(filename) # open file if varnames is None: # no variables specified varnames = f.variables.keys() varnames = [v for v in varnames if not v in ['lat', 'lon']] # remove lat, lon if incl_global: varnames += ['global'] else: if not isinstance(varnames, list): # make list varnames = [varnames] self.lats, self.lons = f.variables['lat'][:], f.variables['lon'][:] self.dat = {'names': [], 'units': [], 'longnames': [], 'data': []} for v in varnames: if v != 'global': var = f.variables[v] self.dat['names'].append(v) self.dat['units'].append(var.units if 'units' in var.ncattrs() else '') self.dat['longnames'].append(var.long_name if 'long_name' in var.ncattrs() else '') self.dat['data'].append(var[:]) else: nlats = self.lats.size nlons = self.lons.size self.dat['names'].append('global') # global mask self.dat['units'].append('') self.dat['longnames'].append('') self.dat['data'].append( masked_array(ones((nlats, nlons)), mask=zeros((nlats, nlons)))) f.close() tol = 1e-5 if not lats is None: # restrict latitude range sellat = logical_and(self.lats >= lats.min() - tol, self.lats <= lats.max() + tol) self.lats = self.lats[sellat] for i in range(len(self.dat['names'])): self.dat['data'][i] = self.dat['data'][i][sellat] if not lons is None: # restrict longitude range sellon = logical_and(self.lons >= lons.min() - tol, self.lons <= lons.max() + tol) self.lons = self.lons[sellon] for i in range(len(self.dat['names'])): self.dat['data'][i] = self.dat['data'][i][:, sellon]
def __init__(self, filename, varnames = None, lats = None, lons = None, incl_global = False): f = nc(filename) # open file if varnames is None: # no variables specified varnames = f.variables.keys() varnames = [v for v in varnames if not v in ['lat', 'lon']] # remove lat, lon if incl_global: varnames += ['global'] else: if not isinstance(varnames, list): # make list varnames = [varnames] self.lats, self.lons = f.variables['lat'][:], f.variables['lon'][:] self.dat = {'names': [], 'units': [], 'longnames': [], 'data': []} for v in varnames: if v != 'global' or (v == 'global' and 'global' in f.variables): var = f.variables[v] self.dat['names'].append(v) self.dat['units'].append(var.units if 'units' in var.ncattrs() else '') self.dat['longnames'].append(var.long_name if 'long_name' in var.ncattrs() else '') self.dat['data'].append(var[:]) else: nlats = self.lats.size nlons = self.lons.size self.dat['names'].append('global') # global mask self.dat['units'].append('') self.dat['longnames'].append('') self.dat['data'].append(masked_array(ones((nlats, nlons)), mask = zeros((nlats, nlons)))) f.close() tol = 1e-5 if not lats is None: # restrict latitude range sellat = logical_and(self.lats >= lats.min() - tol, self.lats <= lats.max() + tol) self.lats = self.lats[sellat] for i in range(len(self.dat['names'])): self.dat['data'][i] = self.dat['data'][i][sellat] if not lons is None: # restrict longitude range sellon = logical_and(self.lons >= lons.min() - tol, self.lons <= lons.max() + tol) self.lons = self.lons[sellon] for i in range(len(self.dat['names'])): self.dat['data'][i] = self.dat['data'][i][:, sellon]
def process_aia_map(m, rsun=1.2, threshold=35): from sunpy.map.maputils import all_coordinates_from_map hpc_coords = all_coordinates_from_map(m) r = np.sqrt(hpc_coords.Tx ** 2 + hpc_coords.Ty ** 2) / m.rsun_obs mask = ma.logical_and(r > rsun, m.data < threshold) processed_map = Map(m.data, m.meta, mask=mask) return(processed_map)
def cifar100_complicated_ensemble_v2_submodel5_labels_manipulation(labels_array: ndarray) -> int: """ The model's labels manipulator. :param labels_array: the labels to manipulate. :return: the number of classes predicted by the model. """ # 50 - 59, 80 - 99, 0 - 9 classes. labels_array[logical_and(labels_array > 9, labels_array < 50)] = -1 labels_array[logical_and(labels_array > 59, labels_array < 80)] = -1 for i in range(0, 10): labels_array[labels_array == i] = i + 1 labels_array[labels_array == i + 50] = i + 11 labels_array[labels_array == i + 80] = i + 21 labels_array[labels_array == i + 90] = i + 31 labels_array[labels_array == -1] = 0 return 41
def bayes(xvec, xcrit, label, unknown, fout=sys.stdout): warm = ma.masked_array(xvec > xcrit) warm = ma.logical_and(warm, unknown) nwarm = len(warm.nonzero()[0]) lmask = np.logical_and(landmask, warm) imask = np.logical_and(icemask, warm) omask = np.logical_and(watermask, warm) pwarm = float(nwarm) / float(nobs) pover_land = len(lmask.nonzero()[0]) / nlandpts pover_water = len(omask.nonzero()[0]) / nwaterpts pover_ice = len(imask.nonzero()[0]) / nicepts if (pwarm > 0): print(label, "hot ", xcrit, "{:5.3f}".format(pover_ice * pice / pwarm), "{:5.3f}".format(pover_land * pland / pwarm), "{:5.3f}".format(pover_water * pwater / pwarm), nwarm, file=fout) cold = ma.masked_array(xvec < xcrit) cold = ma.logical_and(cold, unknown) ncold = len(cold.nonzero()[0]) lmask = np.logical_and(landmask, cold) imask = np.logical_and(icemask, cold) omask = np.logical_and(watermask, cold) pcold = float(ncold) / float(nobs) pover_land = len(lmask.nonzero()[0]) / nlandpts pover_water = len(omask.nonzero()[0]) / nwaterpts pover_ice = len(imask.nonzero()[0]) / nicepts if (pcold > 0): print(label, "cold ", xcrit, "{:5.3f}".format(pover_ice * pice / pcold), "{:5.3f}".format(pover_land * pland / pcold), "{:5.3f}".format(pover_water * pwater / pcold), ncold, file=fout)
def estimate_cell_edges(x): """Convert one-dimensional vector x of size n into n + 1, where the input describes the centres of the cells, and the output is an estimate of the edges of the cell""" # centres (with extra centres padded at the ends by linear interpolation) dx = ma.diff(x) x_c = ma.hstack((x[0] - atleast_1d(dx[0]), x, x[-1] + atleast_1d(dx[-1]))) # _f is notation from MITgcm (implies faces) x_f = (x_c[1:] + x_c[:-1])/2 dx_c = np.diff(x_c) # Catch nan or masked values and estimate edge using dx from previous or # next cell nan_before = ma.where( ma.logical_and(nan_or_masked(x_f[:-1]), ~nan_or_masked(x_f[1:])))[0] nan_after = ma.where( ma.logical_and(~nan_or_masked(x_f[:-1]), nan_or_masked(x_f[1:])))[0] x_f[nan_before] = x_f[nan_before + 1] - dx_c[nan_before + 1] x_f[nan_after + 1] = x_f[nan_after] + dx_c[nan_after] return x_f
def combine_Taus(data, weight_coslat, Taus, K_lim=True): """Use the CFL criteria to limit Tau, at first choose the min dt (max spped) for each location.""" for key in ['U_global', 'V_global', 'Kx_global', 'Ky_global', 'R_global']: data[key] = np.reshape( data[key][:], (len(Taus), data[key].shape[0] / len(Taus), -1)).squeeze() lmask = data['R_global'][0, :, :].mask dt = ma.min( (1 / (abs(data['U_global'][:]) / (weight_coslat * 111E3 * 0.25) + abs(data['V_global'][:]) / (111E3 * 0.25))) / (3600 * 24.), 0) #make dt's to be integers dt[np.where(dt < Taus[0])] = Taus[0] for t, tau in enumerate(Taus[:-2]): dt[np.where(ma.logical_and(dt > tau, dt <= Taus[t + 1]))] = Taus[t + 1] dt[np.where(dt > Taus[-2])] = Taus[-1] if K_lim: c = 0 while c < max(Taus): c = c + 1 for t, tau in enumerate(Taus): dx = (weight_coslat * 111E3 * 0.25) dy = 111E3 * 0.25 jinds, iinds = np.where(dt.squeeze() == tau) if len(jinds) > 1: jindsX = np.where( data['Kx_global'][t, jinds, iinds].squeeze() * tau * 3600 * 24 / dx[jinds, iinds]**2 > 1)[0] if len(jindsX) > 1: dt[jinds[jindsX], iinds[jindsX]] = Taus[max( t - 1, 0 )] #np.max([dt[jinds[jindsX],iinds[jindsX]]-1,np.ones(len(jindsX))*Taus[0]],axis=0) jindsY = np.where( data['Ky_global'][t, jinds, iinds].squeeze() * tau * 3600 * 24 / dy**2 > 1)[0] if len(jindsY) > 1: dt[jinds[jindsY], iinds[jindsY]] = Taus[max( t - 1, 0 )] #np.max([dt[jinds[jindsY],iinds[jindsY]]-1,np.ones(len(jindsY))*Taus[0]],axis=0) #use dt and pick up each field given the location specific Tau for key in ['U_global', 'V_global', 'Kx_global', 'Ky_global', 'R_global']: dum2 = np.zeros(data[key][0, :, :].shape) for j, ext in enumerate(Taus): jinds, iinds = np.where(dt.astype('int') == ext) dum2[jinds, iinds] = data[key][j, jinds, iinds].squeeze() #dum2[jinds,iinds]=ma.max(data[key][:j+1,jinds,iinds].squeeze(),0) data[key] = ma.masked_array(dum2, lmask) # return data, ma.masked_array(dt.astype('int'), lmask)
def svhn_complicated_ensemble_v2_submodel5_labels_manipulation( labels_array: ndarray) -> int: """ The model's labels manipulator. :param labels_array: the labels to manipulate. :return: the number of classes predicted by the model. """ # 5, 8, 9, 0 classes. labels_array[logical_and(labels_array < 5, labels_array > 0)] = 6 labels_array[labels_array == 0] = 1 labels_array[labels_array == 5] = 2 labels_array[labels_array == 8] = 3 labels_array[labels_array == 9] = 4 labels_array[labels_array == 6] = 0 labels_array[labels_array == 7] = 0 return 5
def stand_est(data_mat, user, sim_means, item): n = shape(data_mat)[1] all_sim = 0 item_score = 0 for j in range(n): score = data_mat[user, j] if score == 0: continue over_lap = nonzero(logical_and(data_mat[:, j] > 0, data_mat[:, item] > 0))[0] sim = sim_means(data_mat[over_lap, j], data_mat[over_lap, item]) all_sim += sim item_score += sim * score if all_sim == 0: item_score = 0 else: item_score = item_score / all_sim return item_score
def stand_est(data_mat, user, sim_means, item): n = shape(data_mat)[1] all_sim = 0 item_score = 0 for j in range(n): score = data_mat[user, j] if score == 0: continue over_lap = nonzero( logical_and(data_mat[:, j] > 0, data_mat[:, item] > 0))[0] sim = sim_means(data_mat[over_lap, j], data_mat[over_lap, item]) all_sim += sim item_score += sim * score if all_sim == 0: item_score = 0 else: item_score = item_score / all_sim return item_score
def _calc_gsl(self, values, threshold): """ Calculates GSL for the given values""" data_shape = values.shape[1:] gsl_cnt = ma.zeros(data_shape) warm_cnt = np.zeros(data_shape) cold_cnt = np.zeros(data_shape) increment = np.zeros(data_shape) for i, arr in enumerate(values): if i < 183: # Take the first half of an year. mask = arr > threshold warm_cnt += mask # Count warm days. warm_cnt *= mask # Reset counter of warm days on a cold one. increment = ma.logical_or(increment, (warm_cnt == 5)) # Search for cells with 5 consecutive warm days. else: # Take the second part of an year. mask = arr < threshold cold_cnt += mask # Count cold days. cold_cnt *= mask # Reset counter of cold days on a warm one. increment = ma.logical_and(increment, ~(cold_cnt == 5)) # Search for cells with 5 consecutive cold days. gsl_cnt += increment # Count days inside GSL at each cell. gsl_cnt.mask = values.mask[0] # Take source mask. return gsl_cnt
yld1 = f.variables['yield'][:] area1 = f.variables['area'][:] marea1 = f.variables['area_mirca'][:] with nc(winterfile) as f: yld2 = f.variables['yield'][:] area2 = f.variables['area'][:] marea2 = f.variables['area_mirca'][:] a1 = area1[20 : 31, :, :, 2].mean(axis = 0) # sum, averaged from 2000-2010 a2 = area2[20 : 31, :, :, 2].mean(axis = 0) latd = resize(lat, (len(lon), len(lat))).T latd = masked_where(a1.mask, latd) latidx, lonidx = where(logical_and(a2 > a1, latd <= 49)) # replace with winter wheat if area is greater AND lat <= 49 yldf = yld1.copy() areaf = area1.copy() mareaf = marea1.copy() yldf[:, latidx, lonidx, :] = yld2[:, latidx, lonidx, :] areaf[:, latidx, lonidx, :] = area2[:, latidx, lonidx, :] mareaf[:, latidx, lonidx, :] = marea2[:, latidx, lonidx, :] dvar = a1.copy() dvar[:] = 1 dvar[logical_and(a1 < a2, latd <= 49)] = 2 dvar = masked_where(a1.mask, dvar) with nc(outputfile, 'w') as f:
ice_land[i] = all[i].ice_land icec[i] = all[i].ice_sst sst[i] = all[i].sst # create logical masks: unknown = ma.masked_array( ice_land > -1) # unknown points, which starts as all of them #include coast points as being land (sidelobe issues) landmask = ma.masked_array(ice_land >= 157) watermask = ma.masked_array(ice_land < 100) icemask = ma.masked_array(icec > 0) #Distinguish between water and ice-covered water not_ice = np.logical_not(icemask) watermask = ma.logical_and(watermask, not_ice) # Get the indices of the 'true' points mland = landmask.nonzero() water = watermask.nonzero() iceindices = icemask.nonzero() nicepts = len(iceindices[0]) nlandpts = len(mland[0]) nwaterpts = len(water[0]) #--------------------------------------------------------------------- # All data read in and apportioned, # ice, land, water, and unknown pts. masks defined # #--------------------------------------------------------------------- print("n ice, land, water, nobs ", nicepts, nlandpts, nwaterpts, nobs)
def press2alt(arg, P0=None, T0=None, missing=1e+20, invert=0): """Calculate elevation given pressure (or vice versa). Calculations are made assuming that the temperature distribution follows the 1976 Standard Atmosphere. Technically the standard atmosphere defines temperature distribution as a function of geopotential altitude, and this routine actually calculates geo- potential altitude rather than geometric altitude. Method Positional Argument: * arg: Numeric floating point vector of any shape and size, or a Numeric floating point scalar. If invert=0 (the default), arg is air pressure [hPa]. If invert=1, arg is elevation [m]. Method Keyword Arguments: * P0: Pressure [hPa] at the surface (altitude equals 0). Numeric floating point vector of same size and shape as arg or a scalar. Default of keyword is set to None, in which case the routine uses the value of instance attribute sea_level_press (converted to hPa) from the AtmConst class. Keyword value is used if the keyword is set in the function call. This keyword cannot have any missing values. * T0: Temperature [K] at the surface (altitude equals 0). Numeric floating point vector of same size and shape as arg or a scalar. Default of keyword is set to None, in which case the routine uses the value of instance attribute sea_level_temp from the AtmConst class. Keyword value is used if the keyword is set in the func- tion call. This keyword cannot have any missing values. * missing: If arg has missing values, this is the missing value value. Floating point scalar. Default is 1e+20. * invert: If set to 1, function calculates pressure [hPa] from altitude [m]. In that case, positional input variable arg is altitude [m] and the output is pressure [hPa]. Default value of invert=0, which means the function calculates altitude given pressure. Output: * If invert=0 (the default), output is elevation [m] at each element of arg, relative to the surface. If invert=1, output is the air pressure [hPa]. Numeric floating point array of the same size and shape as arg. If there are any missing values in output, those values are set to the value in argument missing from the input. If there are missing values in the output due to math errors and missing is set to None, output will fill those missing values with the MA default value of 1e+20. References: * Carmichael, Ralph (2003): "Definition of the 1976 Standard Atmo- sphere to 86 km," Public Domain Aeronautical Software (PDAS). URL: http://www.pdas.com/coesa.htm. * Wallace, J. M., and P. V. Hobbs (1977): Atmospheric Science: An Introductory Survey. San Diego, CA: Academic Press, ISBN 0-12-732950-1, pp. 60-61. Examples: (1) Calculating altitude given pressure: >>> from press2alt import press2alt >>> import Numeric as N >>> press = N.array([200., 350., 850., 1e+20, 50.]) >>> alt = press2alt(press, missing=1e+20) >>> ['%.7g' % alt[i] for i in range(5)] ['11783.94', '8117.19', '1457.285', '1e+20', '20575.96'] (2) Calculating pressure given altitude: >>> alt = N.array([0., 10000., 15000., 20000., 50000.]) >>> press = press2alt(alt, missing=1e+20, invert=1) >>> ['%.7g' % press[i] for i in range(5)] ['1013.25', '264.3589', '120.443', '54.74718', '0.7593892'] (3) Input is a Numeric floating point scalar, and using a keyword set surface pressure to a different scalar: >>> alt = press2alt(N.array(850.), P0=1000.) >>> ['%.7g' % alt[0]] ['1349.778'] """ import numpy as N import numpy.ma as MA #jfp was import MA #jfp was import Numeric as N from atmconst import AtmConst from is_numeric_float import is_numeric_float #- Check input is of the correct type: if is_numeric_float(arg) != 1: raise TypeError, "press2alt: Arg not Numeric floating" #- Import general constants and set additional constants. h1_std # is the lower limit of the Standard Atmosphere layer geopoten- # tial altitude [m], h2_std is the upper limit [m] of the layer, # and dT/dh is the temperature gradient (i.e. negative of the # lapse rate) [K/m]: const = AtmConst() h1_std = N.array([0., 11., 20., 32., 47., 51., 71.]) * 1000. h2_std = N.array(MA.concatenate([h1_std[1:], [84.852 * 1000.]])) dTdh_std = N.array([-6.5, 0.0, 1.0, 2.8, 0.0, -2.8, -2.0]) / 1000. #- Prep arrays for masked array calculation and set conditions # at sea-level. Pressures are in hPa and temperatures in K. # Sea-level conditions arrays are same shape/size as P_or_z. # If input argument is a scalar, make the local variable used # for calculations a 1-element vector: if missing == None: P_or_z = MA.masked_array(arg) else: P_or_z = MA.masked_values(arg, missing, copy=0) if P_or_z.shape == (): P_or_z = MA.reshape(P_or_z, (1, )) if P0 == None: #jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ P0_use = MA.zeros(P_or_z.shape) \ + (const.sea_level_press / 100.) else: #jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ P0_use = MA.zeros(P_or_z.shape) \ + MA.masked_array(P0) if T0 == None: #jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ T0_use = MA.zeros(P_or_z.shape) \ + const.sea_level_temp else: #jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ T0_use = MA.zeros(P_or_z.shape) \ + MA.masked_array(T0) #- Calculate P and T for the boundaries of the 7 layers of the # Standard Atmosphere for the given P0 and T0 (layer 0 goes from # P0 to P1, layer 1 from P1 to P2, etc.). These are given as # 8 element dictionaries P_std and T_std where the key is the # location (P_std[0] is at the bottom of layer 0, P_std[1] is the # top of layer 0 and bottom of layer 1, ... and P_std[7] is the # top of layer 6). Remember P_std and T_std are dictionaries but # dTdh_std, h1_std, and h2_std are vectors: P_std = {0: P0_use} T_std = {0: T0_use} for i in range(len(h1_std)): P_std[i+1] = _pfromz_MA( h2_std[i], -dTdh_std[i] \ , P_std[i], T_std[i], h1_std[i] ) T_std[i + 1] = T_std[i] + (dTdh_std[i] * (h2_std[i] - h1_std[i])) #- Test input is within Standard Atmosphere limits: if invert == 0: tmp = MA.where(P_or_z < P_std[len(h1_std)], 1, 0) if MA.sum(MA.ravel(tmp)) > 0: raise ValueError, "press2alt: Pressure out-of-range" else: tmp = MA.where(P_or_z > MA.maximum(h2_std), 1, 0) if MA.sum(MA.ravel(tmp)) > 0: raise ValueError, "press2alt: Altitude out-of-range" #- What layer number is each element of P_or_z in? P_or_z_layer = MA.zeros(P_or_z.shape) if invert == 0: for i in range(len(h1_std)): tmp = MA.where( MA.logical_and( (P_or_z <= P_std[i]) \ , (P_or_z > P_std[i+1]) ) \ , i, 0 ) P_or_z_layer += tmp else: for i in range(len(h1_std)): tmp = MA.where( MA.logical_and( (P_or_z >= h1_std[i]) \ , (P_or_z < h2_std[i]) ) \ , i, 0 ) P_or_z_layer += tmp #- Fill in the bottom-of-the-layer variables and the lapse rate # for the layers that the levels are in. The *_actual variables # are the values of dTdh, P_bott, etc. for each element in the # P_or_z_flat array: P_or_z_flat = MA.ravel(P_or_z) P_or_z_flat_mask = P_or_z_flat.mask if P_or_z_flat.mask == False: P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape) #jfp was: #if P_or_z_flat.mask() == None: # P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape) #else: # P_or_z_flat_mask = P_or_z_flat.mask() P_or_z_layer_flat = MA.ravel(P_or_z_layer) #jfp was dTdh_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) #jfp was P_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) #jfp was T_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) #jfp was z_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) dTdh_actual = MA.zeros(P_or_z_flat.shape) P_bott_actual = MA.zeros(P_or_z_flat.shape) T_bott_actual = MA.zeros(P_or_z_flat.shape) z_bott_actual = MA.zeros(P_or_z_flat.shape) for i in xrange(MA.size(P_or_z_flat)): if P_or_z_flat_mask[i] != 1: layer_number = P_or_z_layer_flat[i] dTdh_actual[i] = dTdh_std[layer_number] P_bott_actual[i] = MA.ravel(P_std[layer_number])[i] T_bott_actual[i] = MA.ravel(T_std[layer_number])[i] z_bott_actual[i] = h1_std[layer_number] else: dTdh_actual[i] = MA.masked P_bott_actual[i] = MA.masked T_bott_actual[i] = MA.masked z_bott_actual[i] = MA.masked #- Calculate pressure/altitude from altitude/pressure (output is # a flat array): if invert == 0: output = _zfromp_MA( P_or_z_flat, -dTdh_actual \ , P_bott_actual, T_bott_actual, z_bott_actual ) else: output = _pfromz_MA( P_or_z_flat, -dTdh_actual \ , P_bott_actual, T_bott_actual, z_bott_actual ) #- Return output as same shape as input positional argument: return MA.filled(MA.reshape(output, arg.shape), missing)
ice_land[i] = all[i].ice_land icec[i] = all[i].ice_sst sst[i] = all[i].sst # create logical masks: unknown = ma.masked_array( ice_land > -1) # unknown points, which starts as all of them #include coast points as being land (sidelobe issues) icemask = ma.masked_array(icec > 0) landmask = ma.masked_array(ice_land >= 157) watermask = ma.masked_array(ice_land < 100) #Distinguish between water and ice-covered water not_ice = np.logical_not(icemask) watermask = ma.logical_and(watermask, not_ice) # Get the indices of the 'true' points iceindices = icemask.nonzero() mland = landmask.nonzero() water = watermask.nonzero() nicepts = len(iceindices[0]) nlandpts = len(mland[0]) nwaterpts = len(water[0]) #--------------------------------------------------------------------- # All data read in and apportioned, # ice, land, water, and unknown pts. masks defined # #--------------------------------------------------------------------- print("n ice, land, water, nobs ", nicepts, nlandpts, nwaterpts, nobs)
# load latitude, longitude information from first file if var in['sss'] and not model_data: first_file = xr.open_dataset(Data_directory+File_names[0],decode_times=False) else: first_file = xr.open_dataset(Data_directory+File_names[0]) #Dataset(Data_directory+File_names[0]) if coarse: Lat_vector_glob = first_file[Lat_cdf_name].values[::ndp] #first_file.variables[Lat_cdf_name][::ndp] Lon_vector_glob = first_file[Lon_cdf_name].values[::ndp] #first_file.variables[Lon_cdf_name][::ndp] else: Lat_vector_glob = first_file[Lat_cdf_name].values #first_file.variables[Lat_cdf_name][:] Lon_vector_glob = first_file[Lon_cdf_name].values #first_file.variables[Lon_cdf_name][:] if (coarse_ave and ndp not in ['0.25']) or (profiles and ndp not in ['0.25']): Lat_vector_glob=Lat_vector_glob[:,0] Lon_vector_glob=Lon_vector_glob[0,:] if profiles: Lat_vector = Lat_vector_glob[ma.where(ma.logical_and(Lat_vector_glob>=Lat_range[0],Lat_vector_glob<=Lat_range[1]))] Lon_vector = Lon_vector_glob[ma.where(ma.logical_and(Lon_vector_glob>=Lon_range[0],Lon_vector_glob<=Lon_range[1]))] else: Lat_vector = Lat_vector_glob[ma.where(ma.logical_and(Lat_vector_glob>Lat_range[0],Lat_vector_glob<Lat_range[1]))] Lon_vector = Lon_vector_glob[ma.where(ma.logical_and(Lon_vector_glob>Lon_range[0],Lon_vector_glob<Lon_range[1]))] # Num_lats_global = len(Lat_vector); Num_lons_global = len(Lon_vector); Num_pts_global = Num_lats_global * Num_lons_global; Lat_grid,Lon_grid = np.meshgrid(Lat_vector,Lon_vector); Lon_grid = Lon_grid.T # transpose to suit rows/cols matrix format Lat_grid = Lat_grid.T # note: field is upside down for image plots # # load time info from first file if not profiles and not coarse_ave and not smooth and var not in ['sss']: Time_vector = first_file[Time_cdf_name].values #first_file.variables[Time_cdf_name][:]
area = masked_where(mk, area) # mask areasum1 = areasum1 * area / 100. with nc(weightfile) as f: areair2 = f.variables['irrigated'][:] arearf2 = f.variables['rainfed'][:] areair2 = resize(areair2, sh) arearf2 = resize(arearf2, sh) areasum2 = areair2 + arearf2 areair1 = masked_array(zeros(sh), mask = mk) arearf1 = masked_array(zeros(sh), mask = mk) # no data -> all rainfed nodata = logical_and(~mk, logical_or(areasum2.mask, areasum2 == 0)) idx1, idx2, idx3 = where(nodata) areair1[idx1, idx2, idx3] = 0. arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] # has data hasdata = logical_and(~mk, logical_not(nodata)) idx1, idx2, idx3 = where(hasdata) areair1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * areair2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3] arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * arearf2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3] with nc(outputfile, 'w') as f: f.createDimension('time', sh[0]) tvar = f.createVariable('time', 'i4', 'time') tvar[:] = range(0, sh[0]) tvar.units = 'years since %d' % year
def press2alt(arg, P0=None, T0=None, missing=1e+20, invert=0): """Calculate elevation given pressure (or vice versa). Calculations are made assuming that the temperature distribution follows the 1976 Standard Atmosphere. Technically the standard atmosphere defines temperature distribution as a function of geopotential altitude, and this routine actually calculates geo- potential altitude rather than geometric altitude. Method Positional Argument: * arg: Numeric floating point vector of any shape and size, or a Numeric floating point scalar. If invert=0 (the default), arg is air pressure [hPa]. If invert=1, arg is elevation [m]. Method Keyword Arguments: * P0: Pressure [hPa] at the surface (altitude equals 0). Numeric floating point vector of same size and shape as arg or a scalar. Default of keyword is set to None, in which case the routine uses the value of instance attribute sea_level_press (converted to hPa) from the AtmConst class. Keyword value is used if the keyword is set in the function call. This keyword cannot have any missing values. * T0: Temperature [K] at the surface (altitude equals 0). Numeric floating point vector of same size and shape as arg or a scalar. Default of keyword is set to None, in which case the routine uses the value of instance attribute sea_level_temp from the AtmConst class. Keyword value is used if the keyword is set in the func- tion call. This keyword cannot have any missing values. * missing: If arg has missing values, this is the missing value value. Floating point scalar. Default is 1e+20. * invert: If set to 1, function calculates pressure [hPa] from altitude [m]. In that case, positional input variable arg is altitude [m] and the output is pressure [hPa]. Default value of invert=0, which means the function calculates altitude given pressure. Output: * If invert=0 (the default), output is elevation [m] at each element of arg, relative to the surface. If invert=1, output is the air pressure [hPa]. Numeric floating point array of the same size and shape as arg. If there are any missing values in output, those values are set to the value in argument missing from the input. If there are missing values in the output due to math errors and missing is set to None, output will fill those missing values with the MA default value of 1e+20. References: * Carmichael, Ralph (2003): "Definition of the 1976 Standard Atmo- sphere to 86 km," Public Domain Aeronautical Software (PDAS). URL: http://www.pdas.com/coesa.htm. * Wallace, J. M., and P. V. Hobbs (1977): Atmospheric Science: An Introductory Survey. San Diego, CA: Academic Press, ISBN 0-12-732950-1, pp. 60-61. Examples: (1) Calculating altitude given pressure: >>> from press2alt import press2alt >>> import Numeric as N >>> press = N.array([200., 350., 850., 1e+20, 50.]) >>> alt = press2alt(press, missing=1e+20) >>> ['%.7g' % alt[i] for i in range(5)] ['11783.94', '8117.19', '1457.285', '1e+20', '20575.96'] (2) Calculating pressure given altitude: >>> alt = N.array([0., 10000., 15000., 20000., 50000.]) >>> press = press2alt(alt, missing=1e+20, invert=1) >>> ['%.7g' % press[i] for i in range(5)] ['1013.25', '264.3589', '120.443', '54.74718', '0.7593892'] (3) Input is a Numeric floating point scalar, and using a keyword set surface pressure to a different scalar: >>> alt = press2alt(N.array(850.), P0=1000.) >>> ['%.7g' % alt[0]] ['1349.778'] """ import numpy as N import numpy.ma as MA #jfp was import MA #jfp was import Numeric as N from atmconst import AtmConst from is_numeric_float import is_numeric_float #- Check input is of the correct type: if is_numeric_float(arg) != 1: raise TypeError, "press2alt: Arg not Numeric floating" #- Import general constants and set additional constants. h1_std # is the lower limit of the Standard Atmosphere layer geopoten- # tial altitude [m], h2_std is the upper limit [m] of the layer, # and dT/dh is the temperature gradient (i.e. negative of the # lapse rate) [K/m]: const = AtmConst() h1_std = N.array([0., 11., 20., 32., 47., 51., 71.]) * 1000. h2_std = N.array( MA.concatenate([h1_std[1:], [84.852*1000.]]) ) dTdh_std = N.array([-6.5, 0.0, 1.0, 2.8, 0.0, -2.8, -2.0]) / 1000. #- Prep arrays for masked array calculation and set conditions # at sea-level. Pressures are in hPa and temperatures in K. # Sea-level conditions arrays are same shape/size as P_or_z. # If input argument is a scalar, make the local variable used # for calculations a 1-element vector: if missing == None: P_or_z = MA.masked_array(arg) else: P_or_z = MA.masked_values(arg, missing, copy=0) if P_or_z.shape == (): P_or_z = MA.reshape(P_or_z, (1,)) if P0 == None: #jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ P0_use = MA.zeros(P_or_z.shape) \ + (const.sea_level_press / 100.) else: #jfp was P0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ P0_use = MA.zeros(P_or_z.shape) \ + MA.masked_array(P0) if T0 == None: #jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ T0_use = MA.zeros(P_or_z.shape) \ + const.sea_level_temp else: #jfp was T0_use = MA.zeros(P_or_z.shape, typecode=MA.Float) \ T0_use = MA.zeros(P_or_z.shape) \ + MA.masked_array(T0) #- Calculate P and T for the boundaries of the 7 layers of the # Standard Atmosphere for the given P0 and T0 (layer 0 goes from # P0 to P1, layer 1 from P1 to P2, etc.). These are given as # 8 element dictionaries P_std and T_std where the key is the # location (P_std[0] is at the bottom of layer 0, P_std[1] is the # top of layer 0 and bottom of layer 1, ... and P_std[7] is the # top of layer 6). Remember P_std and T_std are dictionaries but # dTdh_std, h1_std, and h2_std are vectors: P_std = {0:P0_use} T_std = {0:T0_use} for i in range(len(h1_std)): P_std[i+1] = _pfromz_MA( h2_std[i], -dTdh_std[i] \ , P_std[i], T_std[i], h1_std[i] ) T_std[i+1] = T_std[i] + ( dTdh_std[i] * (h2_std[i]-h1_std[i]) ) #- Test input is within Standard Atmosphere limits: if invert == 0: tmp = MA.where(P_or_z < P_std[len(h1_std)], 1, 0) if MA.sum(MA.ravel(tmp)) > 0: raise ValueError, "press2alt: Pressure out-of-range" else: tmp = MA.where(P_or_z > MA.maximum(h2_std), 1, 0) if MA.sum(MA.ravel(tmp)) > 0: raise ValueError, "press2alt: Altitude out-of-range" #- What layer number is each element of P_or_z in? P_or_z_layer = MA.zeros(P_or_z.shape) if invert == 0: for i in range(len(h1_std)): tmp = MA.where( MA.logical_and( (P_or_z <= P_std[i]) \ , (P_or_z > P_std[i+1]) ) \ , i, 0 ) P_or_z_layer += tmp else: for i in range(len(h1_std)): tmp = MA.where( MA.logical_and( (P_or_z >= h1_std[i]) \ , (P_or_z < h2_std[i]) ) \ , i, 0 ) P_or_z_layer += tmp #- Fill in the bottom-of-the-layer variables and the lapse rate # for the layers that the levels are in. The *_actual variables # are the values of dTdh, P_bott, etc. for each element in the # P_or_z_flat array: P_or_z_flat = MA.ravel(P_or_z) P_or_z_flat_mask = P_or_z_flat.mask if P_or_z_flat.mask==False: P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape) #jfp was: #if P_or_z_flat.mask() == None: # P_or_z_flat_mask = MA.make_mask_none(P_or_z_flat.shape) #else: # P_or_z_flat_mask = P_or_z_flat.mask() P_or_z_layer_flat = MA.ravel(P_or_z_layer) #jfp was dTdh_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) #jfp was P_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) #jfp was T_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) #jfp was z_bott_actual = MA.zeros(P_or_z_flat.shape, typecode=MA.Float) dTdh_actual = MA.zeros(P_or_z_flat.shape) P_bott_actual = MA.zeros(P_or_z_flat.shape) T_bott_actual = MA.zeros(P_or_z_flat.shape) z_bott_actual = MA.zeros(P_or_z_flat.shape) for i in xrange(MA.size(P_or_z_flat)): if P_or_z_flat_mask[i] != 1: layer_number = P_or_z_layer_flat[i] dTdh_actual[i] = dTdh_std[layer_number] P_bott_actual[i] = MA.ravel(P_std[layer_number])[i] T_bott_actual[i] = MA.ravel(T_std[layer_number])[i] z_bott_actual[i] = h1_std[layer_number] else: dTdh_actual[i] = MA.masked P_bott_actual[i] = MA.masked T_bott_actual[i] = MA.masked z_bott_actual[i] = MA.masked #- Calculate pressure/altitude from altitude/pressure (output is # a flat array): if invert == 0: output = _zfromp_MA( P_or_z_flat, -dTdh_actual \ , P_bott_actual, T_bott_actual, z_bott_actual ) else: output = _pfromz_MA( P_or_z_flat, -dTdh_actual \ , P_bott_actual, T_bott_actual, z_bott_actual ) #- Return output as same shape as input positional argument: return MA.filled( MA.reshape(output, arg.shape), missing )
def _calc_half_htc(self, prcp_values, temp_values, threshold, start, end): """ Calculates Selyaninov's hydrothermal coeffcient for a given time period. Arguments: prcp_values -- daily total precipitation values temp_values -- daily mean temperature values threshold -- threshold temperature (10C, by default) start -- first index in the time grid, we start analysis from it (inclusive) end -- last index in the time grid, we stop analysis before it (NOT inclusive) If start <= stop algorithm runs forth in time. Runs back, otherwise. Returns: result -- array of Selyaninov's hydrothermal coefficient values """ # The idea is to subtract threshold from temperature values # and to catch cases when values change sign from negative to positive (a transition) # i.e., pass x-axis upwards. # Each crossing corresponds to a possible beginning/ending # (depends on the direction of the analysis) of the vegetation period. # We sum values between crossings (in segments) and analyse sums according to Ped' (1951). # Define a function to detect False->True transition. trans = lambda a, b: ma.logical_and(ma.logical_not(a), b) # Create shape for new arrays without time dimmension. dims = list(temp_values.shape) _ = dims.pop(0) # Define some useful arrays. temp_sums_1 = ma.zeros(dims) # Temperature sums for the first segment. temp_sums_2 = ma.zeros(dims) # Temperature sums for the second segment. temp_sums_3 = ma.zeros(dims) # Temperature sums for the third segment. temp_cnt_1 = ma.zeros(dims) # Temperature counter for the first segment. temp_cnt_2 = ma.zeros(dims) # Temperature counter for the second segment. temp_cnt_3 = ma.zeros(dims) # Temperature counter for the third segment. temp_total = ma.zeros(dims) # Temperature total sum for a vegetation period. prcp_sums_1 = ma.zeros(dims) # Precipitation sums for the first segment. prcp_sums_2 = ma.zeros(dims) # Precipitation sums for the second segment. prcp_sums_3 = ma.zeros(dims) # Precipitation sums for the third segment. prcp_total = ma.zeros(dims) # Precipitation total sum for a vegetation period. trans_mask_1 = ma.zeros(dims) # Mask of cells in the first segment state # (i.e., where the first transition was detected). trans_mask_2 = ma.zeros(dims) # Mask of cells in the second segment state. trans_mask_3 = ma.zeros(dims) # Mask of cells in the third segment state. prev_pos_mask = ma.zeros(dims) # Matrix of previous positive mask state. step = 1 if start <= end else -1 # Set step depending on start and stop values. # Search for upward transitions and calculate sums. for i in range(start, end, step): cur_temp = temp_values[i] cur_prcp = prcp_values[i] temp_deviation = cur_temp - threshold temp_pos_mask = temp_deviation >= 0 # Mask of positive values. cur_trans = trans(prev_pos_mask, temp_pos_mask) # Detect negative->positive transition. prev_pos_mask = temp_pos_mask # Store current positive mask for the next iteration. # Turn on the third state, if a transition is detected and the second state is on. # When the state is on, it is never off (provided by a boolean 'or'). trans_mask_3 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_3), trans_mask_2) # Turn on the second state, if a transition is detected and the first state is on. trans_mask_2 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_2), trans_mask_1) # Turn on the first state, if a transition is detected or leave it as is. trans_mask_1 = ma.logical_or(cur_trans, trans_mask_1) # Sum values in the first segment. seg_1_mask = ma.logical_and(trans_mask_1, ma.logical_not(trans_mask_2)) # Only for cells in the first segemnt state. temp_sums_1[seg_1_mask] += temp_deviation[seg_1_mask] temp_cnt_1[seg_1_mask] += 1 prcp_sums_1[seg_1_mask] += cur_prcp[seg_1_mask] # Sum values in the second segment. seg_2_mask = ma.logical_and(trans_mask_2, ma.logical_not(trans_mask_3)) # Only for cells in the second segemnt state. temp_sums_2[seg_2_mask] += temp_deviation[seg_2_mask] temp_cnt_2[seg_2_mask] += 1 prcp_sums_2[seg_2_mask] += cur_prcp[seg_2_mask] # Sum values in the third segment (in fact, everything after the second segment). temp_sums_3[trans_mask_3] += temp_deviation[trans_mask_3] temp_cnt_3[trans_mask_3] += 1 prcp_sums_3[trans_mask_3] += cur_prcp[trans_mask_3] # Create some intermediate sums. temp_sums_12 = temp_sums_1 + temp_sums_2 # S1+S2+S3+S4 temp_cnt_12 = temp_cnt_1 + temp_cnt_2 temp_sums_23 = temp_sums_2 + temp_sums_3 # S3+S4+... all the rest temp_cnt_23 = temp_cnt_2 + temp_cnt_3 temp_sums_123 = temp_sums_12 + temp_sums_3 # S1+S2+S3+S4+... all the rest temp_cnt_123 = temp_cnt_12 + temp_cnt_3 # Check Ped's conditions and calculate temperature sums for the vegetation period. # By adding temp_cnt_...*threshold we restore original values of the temperature. # If vegetation period starts at the first transition point. start_at_seg_1 = ma.logical_and(temp_sums_1 >= 0, temp_sums_12 >= 0) # S1 >= S2 and S1-S2+S3 >= S4 temp_total[start_at_seg_1] = temp_sums_123[start_at_seg_1] + temp_cnt_123[start_at_seg_1] * threshold prcp_total[start_at_seg_1] = prcp_sums_1[start_at_seg_1] + prcp_sums_2[start_at_seg_1] + prcp_sums_3[start_at_seg_1] # If vegetation period starts at the second transition point. start_at_seg_2 = ma.logical_and(temp_sums_1 < 0, temp_sums_2 >= 0) # S1 < S2 and S3-S4 >= 0 temp_total[start_at_seg_2] = temp_sums_23[start_at_seg_2] + temp_cnt_23[start_at_seg_2] * threshold prcp_total[start_at_seg_2] = prcp_sums_2[start_at_seg_2] + prcp_sums_3[start_at_seg_2] # If vegetation period starts at the third transition point. start_at_seg_3 = ma.logical_or(ma.logical_and(temp_sums_1 < 0, temp_sums_2 < 0), # S1 < S2 and S3 < S4 ma.logical_and(ma.logical_and(temp_sums_1 > 0, temp_sums_2 < 0), # or temp_sums_12 < 0)) # S3 < S4 and S1 > S2 and S1-S2+S3 < S4 temp_total[start_at_seg_3] = temp_sums_3[start_at_seg_3] + temp_cnt_3[start_at_seg_3] * threshold prcp_total[start_at_seg_3] = prcp_sums_3[start_at_seg_3] return (prcp_total, temp_total)
def contingency_table_4x4(obs, fcst, th, av, hy): """ :func:`contingency_table_4x4`: Contingency_table_4x4, for further comprehensive study for the skill, Hit Rate(HR), Bias(BS), Threat Score(TS),..etc and hit rates for different categories are calcul- ated.We categorise the 2x2 condigency table to 4x4 contigency table. Hit rate for different categories moderate, heavy and very heavy is calculated. The 2x2 contigency table useful for forecast verification. From 2x2 condigency table we can find the statistical scores such as Hit Rate(HR), Bias(BS), Threat Score(TS), Odds Ratio (ODR)...etc Inputs: obs -the observed values has to be a list(or whatever you decide) fcst-the forecast values th -the threshold value for which the contingency table needs to be created (floating point value please!!) av-the data points between 'th' and 'av' we say that data point is moderate value hy-the data points above 'hy' we say that the data point is higher value Outputs: A list of values [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p are entries of 4x4 contingency table where a = No of values such that both observed and predicted < threshold b = No of values such that 'av'>observed is > threshold and predicted < threshold c = No of values such that 'av'>observed is < 'hy' predicted < threshold d = No of values such that observed is > 'hy' predicted < threshold e =No of values such that observed< threshold and threshold<predicted < 'av' f =No of values such that 'av'>observed is > threshold and threshold<predicted < 'av' g =No of values such that 'av'>observed is < 'hy' and threshold<predicted < 'av' h =No of values such that observed is > 'hy'and threshold<predicted < 'av' i=No of values such that observed< threshold and 'av'<predicted< 'hy' j=No of values such that 'av'>observed is > thresholdand and 'av'<predicted< 'hy' k=No of values such that 'av'>observed is < 'hy' and 'av'<predicted< 'hy' l=No of values such that observed is > 'hy'and 'av'<predicted< 'hy' m=No of values such that observed< threshold and predicted> 'hy' n=No of values such that 'av'>observed is > thresholdand and predicted> 'hy' o=No of values such that 'av'>observed is < 'hy' and p=No of values such that observed is > 'hy'and predicted> 'hy A list of values A, B, C, D are list are entries of 4x4 contigency table A = No of values such that both observed and predicted > threshold B = No of values such that observed is < threshold and predicted > threshold C = No of values such that observed is > threshold and predicted < threshold D = No of values such that both observed and predicted < threshold Usage: Comparing the statistical score, we can analyse the models example: From the contingency table the following statistics can be calculated. HR=(a+d)/(a+b+c+d) (for 2x2 contigency table) HR =(A+F+K+P)/N( for 4x4 contigency table, N is total all the frequencies) ETS=a/(a+b+c) BS=(a+b)/(a+c) ODR= (a*d)/(b*c) N=(a+ b+ c+ d+ e+ f+ g+ h+ i+ j+ k+ l+ m+ n+ o+ p) N(Oi)= um of entries of i th column of 4x4 contingency table. N(Fi)= Sum of entries of i th row of 4x4 contingency table. Reference: "Statistical Methods in the Atmospheric Sciences", Daniel S Wilks, ACADEMIC PRESS "NCMRWF Report, Monsoon-2009 Performance of T254L64 Global Assimilation-Forecast System" Written by: Dileepkumar R, JRF, IIT Delhi Date: 29/02/2011 """ import numpy import numpy.ma as MA # If 'obs' or 'fcst' are not list then an assertion error will show #assert type(obs) is list," 'obs' is not a list" #assert type(fcst) is list," 'fcst' is not a list" obs=MA.array(obs) fcst= MA.array(fcst) #If we give 'obs' & 'fcst' of different dimention an assertion #error will show assert (obs.shape == fcst.shape), \ "Dimention of 'obs' and 'fcst' are different" # Defining threshold th = float(th) av = float(av) hy = float(hy) #We are masking the observation < threshold obs_1= MA.masked_where(obs<th, obs) obs_1_mask= MA.getmask(obs_1) # Masking the observation between "threshold" & "av" condition_1o= MA.logical_and(th<=obs, obs< av) obs_2 = MA.masked_where(condition_1o, obs) obs_2_mask = MA.getmask(obs_2) # Masking the observation between "av" & "hv" condition_2o= MA.logical_and(obs<hy, obs>= av) obs_3 = MA.masked_where(condition_2o, obs) obs_3_mask = MA.getmask(obs_3) # Masking the observation > hy obs_4=MA.masked_where(obs>=hy, obs) obs_4_mask = MA.getmask(obs_4) #Masking the forecast < threshold fcst_1= MA.masked_where(fcst<th, fcst) fcst_1_mask= MA.getmask(fcst_1) # Masking the forecast between "threshold" & "av" condition_1f= MA.logical_and(th<=fcst, fcst< av) fcst_2 = MA.masked_where(condition_1f, fcst) fcst_2_mask = MA.getmask(fcst_2) # Masking the forecast between "av" & "hv" condition_2f= MA.logical_and(fcst<hy, fcst>= av) fcst_3 = MA.masked_where(condition_2f, fcst) fcst_3_mask = MA.getmask(fcst_3) # Masking the forecast > hy fcst_4=MA.masked_where(fcst>=hy, fcst) fcst_4_mask = MA.getmask(fcst_4) #o1 , f1 ; less than threshold(Th) is no rain #o2, f2 ; greater than Th and less than 'av' is light to moderate rain #o3, f3 ; greater than 'av' and less than 'hy' is heavy rain #o4, f4 ; greater than 'hy' is very heavy rain. # Masking such that forcast < threshold vs other four condition # for observation f1o1=MA.logical_and(fcst_1_mask, obs_1_mask)# for a f1o2=MA.logical_and(fcst_1_mask, obs_2_mask)# for b f1o3=MA.logical_and(fcst_1_mask, obs_3_mask)# for c f1o4=MA.logical_and(fcst_1_mask, obs_4_mask)# for d # Masking such that forecast between "threshold" & "av" vs other four # condition for observation f2o1=MA.logical_and(fcst_2_mask, obs_1_mask)# for e f2o2=MA.logical_and(fcst_2_mask, obs_2_mask)# for f f2o3=MA.logical_and(fcst_2_mask, obs_3_mask)# for g f2o4=MA.logical_and(fcst_2_mask, obs_4_mask)# for h # Masking such that forecast between "av" & "hv" vs other four condition # for observation f3o1=MA.logical_and(fcst_3_mask, obs_1_mask)# for i f3o2=MA.logical_and(fcst_3_mask, obs_2_mask)# for j f3o3=MA.logical_and(fcst_3_mask, obs_3_mask)# for k f3o4=MA.logical_and(fcst_3_mask, obs_4_mask)# for l # Making an array of "1"s of order same as observed or predicted f4o1=MA.logical_and(fcst_4_mask, obs_1_mask)# for m f4o2=MA.logical_and(fcst_4_mask, obs_2_mask)# for n f4o3=MA.logical_and(fcst_4_mask, obs_3_mask)# for o f4o4=MA.logical_and(fcst_4_mask, obs_4_mask)# for p #Making an array of "1"s of order same as 'obs' or 'fcst' dummy = numpy.ones(obs.shape, numpy.int) # Calculating the total no. of entries total = float(numpy.sum(dummy)) # Masking the entries of dummy array with respect to the above conditions a_mat = MA.array(dummy, mask=f1o1) b_mat = MA.array(dummy, mask=f1o2) c_mat = MA.array(dummy, mask=f1o3) d_mat = MA.array(dummy, mask=f1o4) e_mat = MA.array(dummy, mask=f2o1) f_mat = MA.array(dummy, mask=f2o2) g_mat = MA.array(dummy, mask=f2o3) h_mat = MA.array(dummy, mask=f2o4) i_mat = MA.array(dummy, mask=f3o1) j_mat = MA.array(dummy, mask=f3o2) k_mat = MA.array(dummy, mask=f3o3) l_mat = MA.array(dummy, mask=f3o4) m_mat = MA.array(dummy, mask=f4o1) n_mat = MA.array(dummy, mask=f4o2) o_mat = MA.array(dummy, mask=f4o3) p_mat = MA.array(dummy, mask=f4o4) # Subtact this from total (WHY???)--->Actually we are masking the needed # values, therefore to compute the actual value we have to substract it # from total a = total - numpy.sum(a_mat) b = total - numpy.sum(b_mat) c = total - numpy.sum(c_mat) d = total - numpy.sum(d_mat) e = total - numpy.sum(e_mat) f = total - numpy.sum(f_mat) g = total - numpy.sum(g_mat) h = total - numpy.sum(h_mat) i = total - numpy.sum(i_mat) j = total - numpy.sum(j_mat) k = total - numpy.sum(k_mat) l = total - numpy.sum(l_mat) m = total - numpy.sum(m_mat) n = total - numpy.sum(n_mat) o = total - numpy.sum(o_mat) p = total - numpy.sum(p_mat) #return [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ,p] """ From 4x4 contingency table we can create 2x2 contingency table """ A = f+g+h+j+k+l+n+o+p #A is total no of values both observed and # forcast are > threshold B = e+i+m # B is total number of values such that observed< threshold & # predicted > threshold C = b+c+d #C is total number of values such that observed> threshold & # predicted < threshold D = a #D is total no of values such that both observed and # forcast are < threshold # We are changing the entries of 4x4 & 2x2 as float and represnt as # a matrix form tb4= ([[a, b, c, d], [e, f, g, h], [i, j, k, l], [m, n, o, p]]) tb4 =numpy.array(tb4, dtype = float) tb2= ([A, B], [C, D]) tb2 = numpy.array(tb2, dtype = float) return tb4
from numpy.ma import logical_and from matplotlib.pyplot import subplots, show from numpy import array from nnet.prepare import LogTransform from utils.loading import get_training_data from utils.shuffling import shuffle from nndist.distance import DistanceFeatureGenerator N = 500 train, labels = get_training_data()[:2] train = train[logical_and(5 <= labels, labels <= 7), :] labels = labels[logical_and(5 <= labels, labels <= 7)] - 4 train, labels = shuffle(train, labels)[:2] train = train[:N, :] labels = labels[:N] print train.shape, train.dtype train = LogTransform().fit_transform(train) print train.shape, train.dtype gen = DistanceFeatureGenerator(n_neighbors=3, distance_p=2, nr_classes=3) train = gen.fit_transform(train, labels) print train.shape, train.dtype #train = MinMaxScaler(feature_range = (0, 3)).fit_transform(train) #print train.shape, train.dtype fig, ax = subplots(figsize=(7, 6)) colors = ['r', 'g', 'b']
def log_linear_vinterp(T, P, levs): ''' # Author Charles Doutriaux # Version 1.1 # Expect 2D field here so there''s no reorder which I suspect to do a memory leak # email: [email protected] # Converts a field from sigma levels to pressure levels # Log linear interpolation # Input # T : temperature on sigma levels # P : pressure field from TOP (level 0) to BOTTOM (last level) # levs : pressure levels to interplate to (same units as P) # Output # t : temperature on pressure levels (levs) # External: Numeric''' import numpy.ma as MA ## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16 sh = P.shape nsigma = sh[0] # Number of sigma levels try: nlev = len(levs) # Number of pressure levels except: nlev = 1 # if only one level len(levs) would breaks t = [] for ilv in range(nlev): # loop through pressure levels try: lev = levs[ilv] # get value for the level except: lev = levs # only 1 level passed # print ' ......... level:',lev Pabv = MA.ones(P[0].shape, Numeric.Float) Tabv = -Pabv # Temperature on sigma level Above Tbel = -Pabv # Temperature on sigma level Below Pbel = -Pabv # Pressure on sigma level Below Pabv = -Pabv # Pressure on sigma level Above for isg in range(1, nsigma): # loop from second sigma level to last one ## print 'Sigma level #',isg a = MA.greater(P[isg], lev) # Where is the pressure greater than lev b = MA.less(P[isg - 1], lev) # Where is the pressure less than lev # Now looks if the pressure level is in between the 2 sigma levels # If yes, sets Pabv, Pbel and Tabv, Tbel Pabv = MA.where(MA.logical_and(a, b), P[isg], Pabv) # Pressure on sigma level Above Tabv = MA.where(MA.logical_and(a, b), T[isg], Tabv) # Temperature on sigma level Above Pbel = MA.where(MA.logical_and(a, b), P[isg - 1], Pbel) # Pressure on sigma level Below Tbel = MA.where(MA.logical_and(a, b), T[isg - 1], Tbel) # Temperature on sigma level Below # end of for isg in range(1,nsigma) # val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is tl = MA.masked_where( MA.equal(Pbel, -1.), MA.log(lev / MA.absolute(Pbel)) / MA.log(Pabv / Pbel) * (Tabv - Tbel) + Tbel) # Interpolation t.append(tl) # add a level to the output # end of for ilv in range(nlev) return asMA(t).astype(Numeric.Float32) # convert t to an array
linewidth=0.25, fontsize=20) p2 = m2.drawmeridians(np.arange(60., 360., 60.), labels=[False, False, False, True], linewidth=0.25, fontsize=20) m2.fillcontinents(color='DarkGray', lake_color='LightGray', zorder=0) #c2=m2.pcolormesh(Lon_vector.squeeze(),Lat_vector.squeeze(),Kb,cmap=cmap,norm=norm,latlon=True,rasterized=True) #m2.plot(x2,y2, '.',color='gray', markersize=0.2, rasterized=True) # lo, la = np.meshgrid(Lon_vector.squeeze(), Lat_vector.squeeze()) #x3,y3=m2(lo,la) Kxrange = np.percentile(Kx[np.where(Kx > 10)], [1, 99]) Kyrange = np.percentile(Ky[np.where(Ky > 10)], [1, 99]) jinds, iinds = np.where( ma.logical_and(Kx > Kxrange[0], Kx < Kxrange[1])) Kx2 = griddata((lo[jinds, iinds], la[jinds, iinds]), Kx[jinds, iinds], (lo, la)) jinds, iinds = np.where( ma.logical_and(Ky > Kyrange[0], Ky < Kyrange[1])) Ky2 = griddata((lo[jinds, iinds], la[jinds, iinds]), Ky[jinds, iinds], (lo, la)) newmask = Kb.mask.copy() newmask2 = Kb.mask.copy() newmask3 = Kb.mask.copy() newmask2[np.where(Kx < Kxrange[0])] = 1 # newmask2[np.where(Kx>Kxrange[1])]=1 newmask2[np.where(Ky < Kyrange[0])] = 1 # newmask2[np.where(Ky>Kyrange[1])]=1 newmask2[np.where(icemask)] = 1 newmask[np.where(icemask)] = 1