def met_rho(temp, rh, p, undef=-9999.): """ Calculation of air density x = met_rho(T, rh, p) where T is the air temperature in oC, rh is the relative humidity in %, p is the air pressure in mbar (hPa). The air density in kg m-3 is calculated according to the following formulas: Es = 6.1078*exp(17.08085*T/ (234.175 + T)) E = Es * rh/100 sh = 622 * E/(p-0.378*E) Tv = ((T + 273.16) * (1 + 0.000608 * sh)) - 273.16 x = p * 100 / (287.05 * (Tv + 273.16)) All parameters may be variables or numbers. Definition ---------- def met_rho(temp, rh, p, undef=-9999.): Input ----- temp ND-array rh ND-array p ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ air density in kg m-3 History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=((temp==undef)|(rh==undef)|(p==undef))))*0.01 ea = es*rh*0.01 sh = division(622.*ea, (p-0.378*ea), undef) Tv = ((temp+273.15)*(1+0.000608*sh)) - 273.15 rho = division(p*100., (287.05*(Tv+273.15)), undef) return rho.filled(undef)
def met_sh(temp, rh, p, undef=-9999.): """ Calculation of specific humidity x = met_sh(T, rh, p) where T is the air temperature in oC, rh is the relative humidity in %, p is the air pressure in mbar (hPa). The specific humidity in g kg-1 is calculated according to the following formulas: Es = 6.1078*exp(17.08085*T/ (234.175 + T)) E = Es * rh/100 x = 622 * E/(p-0.378*E) All parameters may be variables or numbers. Definition ---------- def met_sh(temp, rh, p, undef=-9999.): Input ----- temp ND-array rh ND-array p ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ specific humidity in g kg-1 History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=((temp==undef)|(rh==undef)|(p==undef))))*0.01 ea = es*rh*0.01 sh = division(622.*ea, (p-0.378*ea), undef) return sh.filled(undef)
def met_h2oc_rh(temp, h, p, undef=-9999.): """ Calculation of relative humidity from water vapour concentration x = met_h2oc_rh(T, [H2O], p) where T is the air temperature in oC, [H2O] is the water vapour concentration in mmolmol-1, p is the air pressure in mbar (hPa). The relative humidity in % is calculated according to the following formulas: Es = 6.1078*exp(17.08085*T/(234.175 + T)) E = 10 * [H2O] * 0.001 * p * 100 * 0.001 x = 100 * E / Es All parameters may be variables or numbers. Definition ---------- def met_h2oc_rh(temp, h, p, undef=-9999.): Input ----- temp ND-array rh ND-array p ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ relative humidity in % History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=((temp==undef)|(h==undef)|(p==undef))))*0.01 ea = 0.001 * h * p c = 100.*ea/es return c.filled(undef)
def met_h2oc(temp, rh, p, undef=-9999.): """ Calculation of water vapour concentration x = met_h2oc(T, rh, p) where T is the air temperature in oC, rh is the relative humidity in %, p is the air pressure in mbar (hPa). The water vapour concentration in mmol mol-1 is calculated according to the following formu- las: Es = 6.1078*exp(17.08085*T/ (234.175 + T)) E = Es * rh/100 x = 0.1 * E /(0.001*p*100*0.001) All parameters may be variables or numbers. Definition ---------- def met_h2oc(temp, rh, p, undef=-9999.): Input ----- temp ND-array rh ND-array p ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ water vapour concentration in mmol mol-1 History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=((temp==undef)|(rh==undef)|(p==undef))))*0.01 ea = es*rh*0.01 c = division(1000.*ea, p, undef) return c.filled(undef)
def met_dpt(temp, rh, undef=-9999.): """ Calculation of dew point temperature x = met_dpt(T, rh) where T is the air temperature in oC, rh is the relative humidity in %. The dew point temperature in oC is calculated according to the following formulas: Es = 6.1078*exp(17.08085*T/(234.175 + T)) E = Es * rh/100 x = 234.175 * ln(E/6.1078)/(17.08085 - ln(E/6.1078)) Both parameters may be variables or numbers. Definition ---------- def met_dpt(temp, rh, undef=-9999.): Input ----- temp ND-array rh ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ dew point temperature in oC History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=((temp==undef)|(rh==undef))))*0.01 ea = es*rh*0.01 dpt = 234.175 * np.ma.log(ea/6.1078) / (17.08085 - np.ma.log(ea/6.1078)) return dpt.filled(undef)
def met_vpdef(temp, rh, undef=-9999.): """ Calculation of water vapour pressure deficit x = met_vpdef(T, rh) where T is the air temperature in oC, rh is the relative humidity in %. The water vapour pressure deficit in mbar (hPa) is calculated according to the following for- mulas: Es = 6.1078*exp(17.08085*T/ (234.175 + T)) E = Es * rh/100 x = Es - E Both parameters may be variables or numbers. Definition ---------- def met_vpdef(temp, rh, undef=-9999.): Input ----- temp ND-array rh ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ water vapour pressure deficit in mbar (hPa) History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=((temp==undef)|(rh==undef))))*0.01 ea = es*rh*0.01 vpd = es - ea return vpd.filled(undef)
def met_vpmax(temp, undef=-9999.): """ Calculation of saturation water vapour pressure x = met_vpmax(T) where T is the air temperature in oC. The saturation water vapour pressure in mbar (hPa) is calculated according to the following formula: x = 6.1078 * exp(17.08085 * T / (234.175 + T)) The parameter may be a variable or a number. Definition ---------- def met_vpmax(temp, undef=-9999.): Input ----- var1 ND-array Optional Input -------------- undef elements are excluded from the calculations if any of the inputs equals undef (default: -9999.) Output ------ saturation water vapour pressure in mbar (hPa) History ------- Written, MC, Jun 2014 """ es = esat(np.ma.array(temp+273.15, mask=(temp==undef)))*0.01 return es.filled(undef)
def leafmodel(ci_ini, Tl, PAR, ea, ca, ga, gb, P, par1, par2, gs_method='Leuning', temp_method='June', n=5, eta=0.9): ''' Model to calculate photosynthesis and stomatal conductance of canopies. The Farquhar model (Farquhar et al, 1980) is used for photosynthesis. Stomatal conductance can either be modeled with Ball & Berry (1987) or with Leuning (1995). Aerodynamic and leaf boundary layer conductances are used to calculate leaf surface concentrations and the model iterates until a conversion of leaf internal CO2 concentration. Canopy transpiration and gross primary production are among the results. The temperature dependency of the photosynthesis model can be modeled according to June et al. (2004), Medlyn et al. (2002) and von Caemmerer (2000). Definition ---------- def leafmodel(ci_ini, Tl, PAR, ea, ca, ga, gb, P, par1, par2, gs_method='Leuning', temp_method='June', n=5, eta=0.9): Input ----- ci_ini np.array(N), initial leaf internal co2 concentration [mol/mol] Can be approximated with 0.8*ca Tl np.array(N), leaf temperature [degC] PAR np.array(N), total (direct+diffuse) photon flux density [mol/m2s] ea np.array(N), vapour pressure of the atmosphere [Pa] ca np.array(N), atmospheric CO2 concentration [mol/mol] ga np.array(N), aerodynamic conductivity [mol/m2s] (of heat and water, carbon is calulated in the model) gb np.array(N), leaf boundary layer conductivity [mol/m2s] (of heat and water, carbon is calulated in the model) P np.array(N), atmospheric pressure [Pa] par1 if gs_method='Leuning' (default): list(3) [0] m slope of the Leuning model [mol(H2O)/mol(air)] [1] b offset of the Leuning model [mol(H2O)/m2leaf s] [2] d0 sensitivity parameter to vpds [Pa] if gs_method='BB': list(2) [0] m slope of the BB model [mol(H2O)/mol(air)] [1] b offset of the BB model [mol(H2O)/m2leaf s] par2 if temp_method is June (default): list(3) [0] Vcmax25 maximum carboxylation rate at 25 degC [mol/m2s] [1] Topt optimum temperature of Jmax [degC] [2] omega the difference in temperature from Topt at which Jmax falls to e-1 (0.37) of its value at Topt. (In the paper its on average 18+-0.6 degC) if temp_method is Medlyn: list(3) [0] Vcmax25 maximum carboxylation rate at 25 degC [mol/m2s] [1] delSV entropy factor for Vcmax25 [J/mol/K] [2] delSJ entropy factor for Jmax25 [J/mol/K] if temp_method is Caemmerer: list(2) [0] Vcmax25 maximum carboxylation rate at 25 degC [mol/m2s] [1] delSJ entropy factor for Jmax25 [J/mol/K] Optional Input -------------- gs_method str, stomatal conductance method: 'Leuning' (default), 'BB' (Ball&Berry) temp_method str, temperature dependecy method: 'June', 'Medlyn', 'Caemmerer' n int, number of iteration for leaf internal CO2 convergence eta float, smoothing factor for transition between Jmax and Vcmax dominated assimilation rate [-] Output ------ ci_conv np.array(N), leaf internal CO2 concentration [mol/mol] gc_c np.array(N), canopy conductivity for carbon [mol/m2s] gc_h np.array(N), canopy conductivity for water and heat [mol/m2s] gs_c np.array(N), stomatal conductivity for carbon [mol/m2s] gs_h np.array(N), stomatal conductivity for water and heat [mol/m2s] no_conv np.array(x), indices of values where no ci conversion was achieved with n iterations Jc np.array(N), carboxylation limited assimilation rate [mol/m2s] Je np.array(N), electron transport limited assimilation rate [mol/m2s] Rd np.array(N), leaf dark respiration rate [mol/m2s] GPPmod np.array(N), gross primary productivity [mol/m2s] Emod np.array(N), transpiration [mol/m2s] Examples -------- >>> # Define input >>> ci_ini = np.array([3.361e-4, 3.354e-4, 3.350e-4, 3.345e-4, 3.337e-4]) >>> Tl = np.array([21.041, 21.939, 21.997, 21.026, 20.237]) >>> PAR = np.array([1.144e-3, 1.294e-3, 1.445e-3, 1.733e-3, 1.752e-3]) >>> ea = np.array([1636.967, 1609.575, 1569.418, 1561.334, 1460.839]) >>> ca = np.array([4.202e-4, 4.193e-4, 4.188e-4, 4.182e-4, 4.171e-4]) >>> ga = np.array([3.925, 5.998, 3.186, 7.924, 5.501]) >>> gb = np.array([1.403, 1.678, 1.732, 1.691, 1.782]) >>> P = np.array([99893.7, 99907.7, 99926.5, 99928.0, 99924.7]) >>> # use default methods Leuning + June >>> par1 = [15., 0.0041, 5.e3] >>> par2 = [60.e-6, 25., 18.] >>> ci, gc_c, gc_h, gs_c, gs_h, no_conv, Jc, Je, Rd, GPPmod, Emod = \ leafmodel(ci_ini, Tl, PAR, ea, ca, ga, gb, P, par1, par2) >>> # print GPP [mumol(m2s] and E [mmol/m2s] >>> print(np.ma.round(GPPmod*1.e6, 3)) [12.333 12.685 12.696 12.462 12.157] >>> print(np.ma.round(Emod*1.e3, 3)) [3.339 4.338 4.287 4.02 3.845] >>> # use methods Ball&Berry + Medlyn >>> par1 = [15., 0.0041] >>> par2 = [60.e-6, 700., 700.] >>> ci, gc_c, gc_h, gs_c, gs_h, no_conv, Jc, Je, Rd, GPPmod, Emod = \ leafmodel(ci_ini, Tl, PAR, ea, ca, ga, gb, P, par1, par2,\ gs_method='BB', temp_method='Medlyn') >>> # print GPP [mumol(m2s] and E [mmol/m2s] >>> print(np.ma.round(GPPmod*1.e6, 3)) [14.286 14.496 14.499 14.526 14.287] >>> print(np.ma.round(Emod*1.e3, 3)) [2.718 3.326 3.22 3.167 2.994] License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP+MC, Jul 2014 ''' # array for convergence of leaf internal carbon concentration ci ci_conv = np.ma.empty((n,ci_ini.size)) # initial conditions for ci convergence ci_conv[0,:] = ci_ini # [mol(CO2)/mol(air)] Emod = 0. # [mol(H2O)/m2leaf s] gs_h = 1. # [mol(H2O)/m2leaf s] # atmospheric (wa) and leaf internal (wi) water concentration wa = ea/P # [mol(H2O)/mol(air)] ei = esat(Tl+T0) wi = ei/P # [mol(H2O)/mol(air)] # loop for ci convergence for i in range(n-1): # assimilation A, carboxylation limited assimilation Jc, # electron transport limited assimilation Je and leaf dark respiration # rate Rd, all in [mol/m2leaf s] A, Jc, Je, Rd = farquhar(Tl, ci_conv[i,:], PAR, par2, temp_method=temp_method, eta=eta) # aerodynamic + leaf boundary layer conductances gab_c = 1./(1.37/gb + 1./ga) # [mol(CO2)/m2leaf s] gab_h = 1./(1./gb + 1./ga) # [mol(H2O)/m2leaf s] # carbon concentration at the leaf surface cs = ca - A/gab_c #[mol(CO2)/mol(air)] # water concentration at the leaf surface ws = wa - Emod/gab_h # [mol(H2O)/mol(air)] # vapour pressure at the leaf surface es = ws*P # [Pa] # relative humidity at the leaf surface rHs = es/ei # [-] # stomatal conductance for carbon [mol(CO2)/m2leaf s] if gs_method=='Leuning': gs_c = leuning(A, ei-es, cs, Tl, par1) #gs_c = leuning(A, wi-ws, cs, Tl, par1) if gs_method=='BB': gs_c = ball_berry(A, rHs, cs, par1) # stomatal conductance for water [mol(H2O)/m2leaf s] gs_h = gs_c * 1.6 # canopy conductances gc_c = 1./(1./gs_c + 1./gab_c) # [mol(CO2)/m2leaf s] gc_h = 1./(1./gs_h + 1./gab_h) # [mol(H2O)/m2leaf s] # new ci value ci_conv[i+1,:] = ca - A/(gc_c) # [mol(CO2)/mol(air)] # new modelled transpiration Emod = gc_h * (wi-wa) # [mol(H2O)/m2leaf s] # number of non-coverging ci values no_conv = np.ma.where(np.ma.abs(ci_conv[-1,:]-ci_conv[-2,:])>10.e-6)[0] # modeled transpiration Emod = gc_h * ((ei-ea)/P) # [mol(H2O)/m2leaf s] # modeled gross primary production GPPmod = gc_c * (ca - ci_conv[-1,:]) + Rd # [mol(CO2)/m2leaf s] return ci_conv[-1,:], gc_c, gc_h, gs_c, gs_h, no_conv, Jc, Je, Rd, GPPmod, Emod
def fluxpart(fluxfile, metfile, outdir, swdr, tair, rh, method='local', nogppnight=False, delimiter=[',', ','], skiprows=[1, 1], format=['ascii', 'ascii'], undef=-9999, plot=False): ''' Wrapper function for nee2gpp with file management and plotting. Definition ---------- fluxpart(fluxfile, metfile, outdir, rg, tair, rh, method='local', delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag or fluxfill output file containing fluxes and flags metfile str, path and file name of the meteorology file (must be in sync with fluxfile) outdir str, path of the output folder swdr int, column number of short wave downward radiation [W m-2] in metfile, column number starts with 0 which is first data column. swdr is used for lasslopp and for swdr>0=isday tair int, column number of air temperature [deg C] in metfile, column number starts with 0 which is first data column. rh int, column number of relative humidity [%] in metfile, column number starts with 0 which is first data column. Optional Input -------------- method str, if 'global', fit of Reco vs. temperature to all nighttime data if 'local' | 'reichstein', method of Reichstein et al. (2005) if 'day' | 'lasslop', method of Lasslop et al. (2010) (default: 'local') nogppnight if True: Resp=NEE, GPP=0 at night, GPP always positive if False: Resp=lloyd_taylor, GPP=Resp-NEE at night (default) delimiter list of str, delimiters of fluxfile and metfile (default: [',',',']) skiprows list of int, lines to skip at the beginning of fluxfile and metfile, e.g. header lines (default: [1,1]) format list of str, time formats of fluxfile and metfile, 'ascii' and 'eng' possible (default: ['ascii','ascii']) undef int/float, missing value of fluxfile and metfile (default: -9999, np.nan is not possible) plot bool, if True performs plotting (default: False) Output ------ fluxpart.csv file containing fluxes with original flags and quality flags of the gap filling plus gpp and reco fluxes. gpp and reco get flags of c flux. fluxpart.pdf plot of c flux, gpp and reco License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' ########################################################################### # reading input files d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0], skiprows=skiprows[0]) m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1], skiprows=skiprows[1]) assert (d.shape[1] == 16) | ( d.shape[1] == 24 ), 'fluxfill: fluxfile must be from fluxfill and have 16 or 24 cols' if format[0] == 'ascii': datev = date2dec(ascii=d[:, 0]) elif format[0] == 'eng': datev = date2dec(eng=d[:, 0]) else: raise ValueError('fluxpart: unknown format') if format[1] == 'ascii': datem = date2dec(ascii=m[:, 0]) elif format[1] == 'eng': datem = date2dec(eng=m[:, 0]) else: raise ValueError('fluxpart: unknown format') val = np.where(d[:, 1:] == '', str(undef), d[:, 1:]).astype(np.float) met = np.where(m[:, 1:] == '', str(undef), m[:, 1:]).astype(np.float) ########################################################################### # assign variables if (d.shape[1] == 16): nee = val[:, 9] #corr. C else: nee = val[:, 12] # oder 13 #corr. C nee_stor = val[:, 13] # oder 13 #corr. C swdr = met[:, swdr] isday = swdr > 0. tair = met[:, tair] + 273.15 rh = met[:, rh] #vpd = (1.-rh/100.)*esat(tair) vpd = np.empty_like(tair) vpd[(tair == undef) | (rh == undef)] = undef vpd[~((tair == undef) | (rh == undef))] = (1. - rh[~((tair == undef) | (rh == undef))] / 100.) * esat(tair[~( (tair == undef) | (rh == undef))]) ########################################################################### # do partitioning if (d.shape[1] == 16): gpp, reco = nee2gpp.nee2gpp(datev, nee, tair, isday, rg=swdr, vpd=vpd, undef=undef, method=method, shape=False, masked=False, nogppnight=nogppnight) else: gpp, reco = nee2gpp.nee2gpp(datev, nee, tair, isday, rg=swdr, vpd=vpd, undef=undef, method=method, shape=False, masked=False, nogppnight=nogppnight) gpp_stor, reco_stor = nee2gpp.nee2gpp(datev, nee_stor, tair, isday, rg=swdr, vpd=vpd, undef=undef, method=method, shape=False, masked=False, nogppnight=nogppnight) ####################################################################### # plot if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf pp1 = pdf.PdfPages(outdir + '/fluxpart.pdf') majticks = mpl.dates.MonthLocator(bymonthday=1) format_str = '%d %m %Y %H:%M' date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) fig1 = plt.figure(1) if (d.shape[1] == 16): sub1 = fig1.add_subplot(111) else: sub1 = fig1.add_subplot(211) sub2 = fig1.add_subplot(212) l1 = sub1.plot(datev - date01, nee, '-k', label='nee') l2 = sub1.plot(datev - date01, gpp, '-g', label='gpp') l3 = sub1.plot(datev - date01, reco, '-r', label='reco') if (d.shape[1] != 16): l4 = sub2.plot(datev - date01, nee_stor, '-k', label='nee') l5 = sub2.plot(datev - date01, gpp_stor, '-g', label='gpp') l6 = sub2.plot(datev - date01, reco_stor, '-r', label='reco') sub1.set_xlim(datev[0] - date01, datev[-1] - date01) sub1.xaxis.set_major_locator(majticks) sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) if (d.shape[1] != 16): sub2.set_xlim(datev[0] - date01, datev[-1] - date01) sub2.xaxis.set_major_locator(majticks) sub2.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) fig1.autofmt_xdate() plt.legend(loc='best') plt.show() fig1.savefig(pp1, format='pdf') pp1.close() ########################################################################### # prepare output and save file if (d.shape[1] == 16): header = np.array([ ' H', ' Hflag', ' Hgf', ' LE', ' LEflag', ' LEgf', ' E', ' Eflag', ' Egf', ' C', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf', ' GPP', ' GPPflag', ' GPPgf', ' Reco', 'Recoflag', ' Recogf' ]) flux = np.vstack((gpp, reco)).transpose() flux_str = np.array([['%11.5f' % x for x in y] for y in flux]).repeat(3, axis=1) flux_str[:, [1, 2, 4, 5]] = np.tile(d[:, 11:13], 2) output = np.hstack((d[:, :], flux_str)) else: header = np.array([ ' H', ' H+sT', ' Hflag', ' Hgf', ' LE', ' LE+sLE', ' LEflag', ' LEgf', ' E', ' E+sE', ' Eflag', ' Egf', ' C', ' C+sC', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf', ' sT', ' sLE', ' sE', ' sC', ' GPP', ' GPP+sC', ' GPPflag', ' GPPfg', ' Reco', ' Reco+sC', 'Recoflag', ' Recofg' ]) flux = np.vstack((gpp, gpp_stor, reco, reco_stor)).transpose() flux_str = np.array([['%11.5f' % x for x in y] for y in flux]) flux_str = np.insert(flux_str, [2, 2, 4, 4], flux_str, axis=1) flux_str[:, [2, 3, 6, 7]] = np.tile(d[:, 11:13], 2) output = np.hstack((d[:, :], flux_str)) np.savetxt('%s/fluxpart.csv' % outdir, np.vstack((np.concatenate( ([' date'], header))[np.newaxis, :], output)), '%s', delimiter=',')
def fluxfill(fluxfile, metfile, outdir, rg, tair, rh, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): ''' Wrapper function for gapfill with file management and plotting. Definition ---------- fluxfill(fluxfile, metfile, outdir, rg, tair, rh, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag output file containing fluxes and flags metfile str, path and file name of the meteorology file (must be in sync with fluxfile) outdir str, path of the output folder rg int, column number of global radiation [W m-2] in metfile, column number starts with 0 which is first data column. tair int, column number of air temperature [deg C] in metfile, column number starts with 0 which is first data column. rh int, column number of relative humidity [%] in metfile, column number starts with 0 which is first data column. Optional Input -------------- delimiter list of str, delimiters of fluxfile and metfile (default: [',',',']) skiprows list of int, lines to skip at the beginning of fluxfile and metfile, e.g. header lines (default: [1,1]) format list of str, time formats of fluxfile and metfile, 'ascii' and 'eng' possible (default: ['ascii','ascii']) undef int/float, missing value of fluxfile and metfile (default: -9999, np.nan not possible) plot bool, if True performs plotting (default: False) Output ------ fluxfilled.csv file containing fluxes with original flags and quality flags of the gap filling. Fluxes are filled where flag>1. fluxfilled.pdf plot of each gap filled flux. License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' ########################################################################### # reading input files d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0], skiprows=skiprows[0]) m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1], skiprows=skiprows[1]) assert (d.shape[1]==11) | (d.shape[1]==19), 'fluxfill: fluxfile must be from fluxflag or profile2storage and have 11 or 19 cols' if format[0]=='ascii': datev = date2dec(ascii=d[:,0]) elif format[0]=='eng': datev = date2dec(eng=d[:,0]) else: raise ValueError('fluxfill: unknown format') if format[1]=='ascii': datem = date2dec(ascii=m[:,0]) elif format[1]=='eng': datem = date2dec(eng=m[:,0]) else: raise ValueError('fluxfill: unknown format') val = np.where(d[:,1:]=='', str(undef), d[:,1:]).astype(np.float) met = np.where(m[:,1:]=='', str(undef), m[:,1:]).astype(np.float) ########################################################################### # assign variables if (d.shape[1]==11): data = val[:,0::2] # corr. H, corr. LE, corr. E, corr. C, tau data_flag = (val[:,1::2]>1) | (val[:,0::2]==undef) else: data = val[:,[0,1, 3,4, 6,7, 9,10, 12]] # corr. H, corr. H+s, corr. LE, corr. LE+s, corr. E, corr. E+s, corr. C, corr. C+s, tau data_flag = np.repeat((val[:,[2,5,8,11,13]]>1) | (val[:,[0,1, 3,4, 6,7, 9,10, 12]]==undef), 2, axis=1)[:,:-1] rg = met[:,rg] rg_flag = rg==undef tair = met[:,tair] tair_flag = tair==undef rh = met[:,rh] rh_flag = rh==undef vpd = np.empty_like(tair) vpd[tair_flag | rh_flag] = undef vpd[~(tair_flag | rh_flag)] = (1.-rh[~(tair_flag | rh_flag)]/100.)*esat(tair[~(tair_flag | rh_flag)]+273.14) vpd_flag = vpd==undef flux = np.zeros_like(data) flag = np.zeros_like(data_flag, dtype=np.int) ########################################################################### if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf pp1 = pdf.PdfPages(outdir+'/fluxfilled.pdf') ########################################################################### # do gapfill for i in range(data.shape[1]): flux[:,i], flag[:,i] = gapfill.gapfill(datev, data[:,i], rg, tair, vpd, data_flag[:,i], rg_flag, tair_flag, vpd_flag, rg_dev=50., tair_dev=2.5, vpd_dev=5., longgap=60, fullday=False, undef=undef, ddof=1, err=False, shape=False) ####################################################################### # plot if plot: majticks = mpl.dates.MonthLocator(bymonthday=1) format_str='%d %m %Y %H:%M' date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) fig1 = plt.figure(1) sub1 = fig1.add_subplot(111) l1 =sub1.plot(datev-date01, flux[:,i], '-g') l2 =sub1.plot(datev-date01, np.ma.array(data[:,i],mask=data_flag[:,i]), '-b') sub1.set_xlim(datev[0]-date01,datev[-1]-date01) sub1.xaxis.set_major_locator(majticks) sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) fig1.autofmt_xdate() plt.show() fig1.savefig(pp1, format='pdf') if plot: pp1.close() ########################################################################### # prepare output and save file flux_str = np.array([['%11.5f'%x for x in y] for y in flux]) if (d.shape[1]==11): header = np.array([' H', ' Hflag', ' Hgf', ' LE', ' LEflag', ' LEgf', ' E', ' Eflag', ' Egf', ' C', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf']) output = np.hstack((d[:,0:1], flux_str.repeat(3, axis=1))) output[:,2::3] = astr(val[:,1::2].astype(int), prec=8) output[:,3::3] = astr(flag, prec=8) else: header = np.array([' H', ' H+sT', ' Hflag', ' Hgf', ' LE', ' LE+sLE', ' LEflag', ' LEgf', ' E', ' E+sE', ' Eflag', ' Egf', ' C', ' C+sC', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf', ' sT', ' sLE', ' sE', ' sC']) output = np.hstack((d[:,0:1], np.insert(flux_str, [2,2,4,4,6,6,8,8], flux_str[:,:-1], axis=1), flux_str[:,-1:].repeat(2, axis=1), d[:,-4:])) output[:,[3,7,11,15,18]] = astr(val[:,[2,5,8,11,13]].astype(int), prec=8) output[:,[4,8,12,16,19]] = astr(flag[:,[0,2,4,6,8]], prec=8) np.savetxt('%s/fluxfilled.csv'%outdir, np.vstack((np.concatenate(([' date'], header))[np.newaxis,:], output)), '%s', delimiter=',')
def profile2storage(fluxfile, fluxfile2, profilefile, outdir, heights, CO2=None, H2O=None, T=None, rH=None, delimiter=[',', ',', ','], skiprows=[1, 1, 1], format=['ascii', 'ascii', 'ascii'], undef=-9999, plot=False): ''' Calculates storage fluxes for changes in CO2, H2O, air temperature and air moisture from profile data or meteorological data to correct Eddy Covariance fluxes. FLux files from EddySoft and from fluxflag are needed as well as a file with the profile or meteo data. Fluxes will be updated with the respective storage fluxes and saved in a new file. Multiple application of this routine with different profile or meteo files are possible to correct e.g. the CO2, H2O and latent heat fluxes with profile data of CO2 and H2O concentrations and afterwards the H flux with temperature data from another file. Definition ---------- profile2storage(fluxfile, fluxfile2, profilefile, outdir, heights, CO2=None, H2O=None, T=None, rH=None, delimiter=[',',',',','], skiprows=[1,1,1], format=['ascii','ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag output file containing fluxes and flags. These fluxes will be updated by the storage fluxes and saved as a new file fluxfile2 str, path and file name of EddyFlux output file (timestep checked) containing original fluxes profilefile str, path and file name of the profile file or meteorology file containing CO2, H2O, T or rH values to compute the profile storage from outdir str, path of the output folder heights list of floats, observation heights of the profile [m], increasing e.g. [0.5,1.0,10.0,20.0]. CO2 list of int, column numbers of CO2 concentrations for the different heights (in the same order) [mumol/mol] in profilefile, column number starts with 0 which is first data column. H2O list of int, column numbers of H2O concentrations for the different heights (in the same order) [mmol/mol] in profilefile, column number starts with 0 which is first data column. T list of int, column numbers of air temperatures for the different heights (in the same order) [degC] in profilefile, column number starts with 0 which is first data column. rH list of int, column numbers of relative humidity for the different heights (in the same order) [%] in profilefile, column number starts with 0 which is first data column. The calculation of air vapour energy storage change within the profile works only when T is given as well. Optional Input -------------- delimiter list of str, delimiters of fluxfile, fluxfile and profilefile (default: [',',',',',']) skiprows list of int, lines to skip at the beginning of fluxfile, fluxfile and profilefile, e.g. header lines (default: [1,1,1]) format list of str, time formats of fluxfile, fluxfile and profilefile, 'ascii' and 'eng' possible (default: ['ascii','ascii','ascii']) undef int/float, missing value of fluxfile, fluxfile and profilefile (default: -9999, np.nan is not possible) plot bool, if True performs plotting (default: False) Output ------ flux+stor.csv file containing fluxes and flags where storage fluxes are added in an additional column and storage fluxes are appended to the end of the file Restrictions ------------ Works only with half hourly time steps, all files in sync License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Sep 2014 ''' ########################################################################### # time interval int = 30. dt = int * 60. if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf ########################################################################### # reading input files # fluxes to correct for storage changes d1 = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0]) # original flux file from EddyFlux containing air density rho_a d2 = np.loadtxt(fluxfile2, dtype='|S100', delimiter=delimiter[1]) # file containing profile data (can be meteo file if no profile available) d3 = np.loadtxt(profilefile, dtype='|S100', delimiter=delimiter[2]) assert (d1.shape[1] == 11) | ( d1.shape[1] == 19 ), 'profile2storage: fluxfile must be from fluxflag or profiletostorage and have 11 or 19 cols' assert d2.shape[ 1] == 68, 'profile2storage: fluxfile2 must be from EddyFlux and have 68 cols' assert d1.shape[0] == d2.shape[ 0], 'profile2storage: fluxfile and fluxfile2 must be in sync' assert d1.shape[0] == d3.shape[ 0], 'profile2storage: fluxfile and profilefile must be in sync' assert ( ((H2O == None) & (rH == None)) ^ ((H2O != None) ^ (rH != None)) ), 'profile2storage: give either H2O or rH, both would be double correction' if format[0] == 'ascii': datev = date2dec(ascii=d1[skiprows[0]:, 0]) elif format[0] == 'eng': datev = date2dec(eng=d1[skiprows[0]:, 0]) else: raise ValueError('profile2storage: unknown format') if format[2] == 'ascii': datem = date2dec(ascii=d2[skiprows[2]:, 0]) elif format[2] == 'eng': datem = date2dec(eng=d2[skiprows[2]:, 0]) else: raise ValueError('profile2storage: unknown format') flux1 = np.where(d1[skiprows[0]:, 1:] == '', str(undef), d1[skiprows[0]:, 1:]).astype(np.float) flux2 = np.where(d2[skiprows[1]:, 1:] == '', str(undef), d2[skiprows[1]:, 1:]).astype(np.float) prof = np.where(d3[skiprows[2]:, 1:] == '', str(undef), d3[skiprows[2]:, 1:]).astype(np.float) flux1 = np.ma.array(flux1, mask=flux1 == undef, hard_mask=True) flux2 = np.ma.array(flux2, mask=flux2 == undef) prof = np.ma.array(prof, mask=prof == undef) ########################################################################### # assign variables if d1.shape[1] == 11: H, Hflag = flux1[:, 0], flux1[:, 1] Le, Leflag = flux1[:, 2], flux1[:, 3] E, Eflag = flux1[:, 4], flux1[:, 5] C, Cflag = flux1[:, 6], flux1[:, 7] else: H, Hflag = flux1[:, 0], flux1[:, 2] Le, Leflag = flux1[:, 3], flux1[:, 5] E, Eflag = flux1[:, 6], flux1[:, 8] C, Cflag = flux1[:, 9], flux1[:, 11] p = flux2[:, 58] # [hPa] rho = flux2[:, 62] # [kg/m3] ########################################################################### # prepare output array d4 = np.copy(d1) if d1.shape[1] == 11: temp = np.empty((d1.shape[0], 4), dtype='|S100') temp[:] = ' ' * (11 - len(str(undef))) + str(undef) temp[0, :] = [ ' H+sT', ' LE+sLE', ' E+sE', ' C+sC' ] d4 = np.insert(d4, [2, 4, 6, 8], temp, axis=1) temp[0, :] = [ ' sT', ' sLE', ' sE', ' sC' ] d4 = np.append(d4, temp, axis=1) ########################################################################### # calls if CO2: CO2 = prof[:, CO2] assert CO2.shape[1] == len( heights), 'profile2storage: number of CO2 cols must equal heights' # calculate storage flux and storage flux flag sfCO2 = stor2flux(CO2, rho, heights, dt, 'CO2') sfCO2flag = sfCO2.mask.astype(np.int) # add to eddy flux newC = C + np.ma.filled(sfCO2, 0) # format and write into output array newC_str = np.array(['%11.5f' % x for x in np.ma.filled(newC, undef)]) newC_str = np.where(newC_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newC_str) sfCO2_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfCO2, undef)]) sfCO2_str = np.where(sfCO2_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfCO2_str) d4[skiprows[0]:, 11] = newC_str d4[skiprows[0]:, 18] = sfCO2_str if plot: storplot(CO2, datev, heights, C, sfCO2, newC, 'storageCO2.pdf', pdf, plt, mpl, outdir) if H2O: H2O = prof[:, H2O] assert H2O.shape[1] == len( heights), 'profile2storage: number of H2O cols must equal heights' # calculate storage flux and storage flux flag sfH2O = stor2flux(H2O, rho, heights, dt, 'H2O') sfH2O_Wm2 = sfH2O * mmol_h2o * latentheat_vaporization / 1.e6 sfH2Oflag = sfH2O.mask.astype(np.int) # add to eddy flux newE = E + np.ma.filled(sfH2O, 0) newLe = Le + np.ma.filled(sfH2O_Wm2, 0) # format and write into output array newE_str = np.array(['%11.5f' % x for x in np.ma.filled(newE, undef)]) newLe_str = np.array( ['%11.5f' % x for x in np.ma.filled(newLe, undef)]) sfH2O_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfH2O, undef)]) sfH2O_Wm2_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfH2O_Wm2, undef)]) newE_str = np.where(newE_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newE_str) newLe_str = np.where(newLe_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newLe_str) sfH2O_str = np.where(sfH2O_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfH2O_str) sfH2O_Wm2_str = np.where(sfH2O_Wm2_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfH2O_Wm2_str) d4[skiprows[0]:, 8] = newE_str d4[skiprows[0]:, 17] = sfH2O_str d4[skiprows[0]:, 5] = newLe_str d4[skiprows[0]:, 16] = sfH2O_Wm2_str if plot: storplot(H2O, datev, heights, E, sfH2O, newE, 'storageH2O.pdf', pdf, plt, mpl, outdir) if T: T = prof[:, T] assert T.shape[1] == len( heights), 'profile2storage: number of T cols must equal heights' # calculate storage flux and storage flux flag sfT = stor2flux(T, rho, heights, dt, 'T') sfTflag = sfT.mask.astype(np.int) # add to eddy flux newH = H + np.ma.filled(sfT, 0) # format and write into output array newH_str = np.array(['%11.5f' % x for x in np.ma.filled(newH, undef)]) newH_str = np.where(newH_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newH_str) sfT_str = np.array(['%11.5f' % x for x in np.ma.filled(sfT, undef)]) sfT_str = np.where(sfT_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfT_str) d4[skiprows[0]:, 2] = newH_str d4[skiprows[0]:, 15] = sfT_str if plot: storplot(T, datev, heights, H, sfT, newH, 'storageT.pdf', pdf, plt, mpl, outdir) if rH: rH = prof[:, rH] assert rH.shape[1] == len( heights), 'profile2storage: number of rH cols must equal heights' # calculate specific humidity vapourpressure = esat(T + T0) * (rH / 100.) / 100. #[hPa] specifichumidity = (mmol_h2o / mmol_air * vapourpressure) / ( p - (1. - mmol_h2o / mmol_air) * vapourpressure) # calculate storage flux and storage flux flag sfrH_Wm2 = stor2flux(specifichumidity, rho, heights, dt, 'rH') sfrH = sfrH_Wm2 * 1.e6 / (mmol_h2o * latentheat_vaporization) sfrHflag = sfrH.mask.astype(np.int) # add to eddy flux newE = E + np.ma.filled(sfrH, 0) newLe = Le + np.ma.filled(sfrH_Wm2, 0) # format and write into output array newE_str = np.array(['%11.5f' % x for x in np.ma.filled(newE, undef)]) newLe_str = np.array( ['%11.5f' % x for x in np.ma.filled(newLe, undef)]) sfrH_str = np.array(['%11.5f' % x for x in np.ma.filled(sfrH, undef)]) sfrH_Wm2_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfrH_Wm2, undef)]) newE_str = np.where(newE_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newE_str) newLe_str = np.where(newLe_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newLe_str) sfrH_str = np.where(sfrH_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfrH_str) sfrH_Wm2_str = np.where(sfrH_Wm2_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfrH_Wm2_str) d4[skiprows[0]:, 8] = newE_str d4[skiprows[0]:, 17] = sfrH_str d4[skiprows[0]:, 5] = newLe_str d4[skiprows[0]:, 16] = sfrH_Wm2_str if plot: storplot(rH, datev, heights, E, sfH2O, newE, 'storagerH.pdf', pdf, plt, mpl, outdir) ########################################################################### # write output np.savetxt('%s/flux+stor.csv' % outdir, d4, '%s', delimiter=',')