def storplot(conc, date, heights, oriflux, storflux, newflux, name, pdf, plt, mpl, outdir): ''' ''' majticks = mpl.dates.MonthLocator(bymonthday=1) format_str = '%d %m %Y %H:%M' date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) conc = np.ma.copy(conc.transpose()) date = np.ma.copy(date - date01) pp1 = pdf.PdfPages(outdir + '/' + name) fig1 = plt.figure(name) sub1 = fig1.add_subplot(211) for i, item in enumerate(conc): sub1.plot(date, item, label='%2.1f m' % (heights[i])) plt.legend(loc='best') sub2 = fig1.add_subplot(212) sub2.axhline(y=0, xmin=0, xmax=1, color='k') sub2.plot(date, oriflux, 'b-', label='original') sub2.plot(date, storflux, 'r-', label='storage') sub2.plot(date, newflux, 'g-', label='new') plt.legend(loc='best') sub1.set_xlim(date[0], date[-1]) sub1.xaxis.set_major_locator(majticks) sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) sub2.set_xlim(date[0], date[-1]) sub2.xaxis.set_major_locator(majticks) sub2.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) fig1.autofmt_xdate() plt.show() fig1.savefig(pp1, format='pdf') pp1.close()
def means(date, dat, year=False, month=False, day=False, hour=False, half_hour=False, minute=False, meanday=False, meanmonth=False, seasonal=False, sum=False, max=False, min=False, onlydat=False): """ Calculate daily, monthly, yearly, etc. means of data depending on date stamp. date is Julian date. dat will be averaged over columns, i.e. first dimension. If no option is given, the mean will be over the whole first column. Returns centred dates, averages. Yearly dates are centred at June 15, 12:00h. Monthly dates are centred at 15th, 12:00h. Daily dates are centred at 12:00h. Hourly dates are centred at 30 min. Half hourly dates are centred at 15 and 45 min. Minutely dates are centred at 30 sec. Mean daily dates centred on 30 min of 01. January of first year. Mean monthly dates centred on 15th of month, 12:00h of first year. Seasonal dates are centred at 12:00h of each day of first (leap) year. Definition ---------- def means(date, dat, year=False, month=False, day=False, hour=False, half_hour=False, minute=False, meanday=False, meanmonth=False, seasonal=False, sum=False, max=False, min=False, onlydat=False): Input ----- date 1D array with Julian dates dat ND (masked-)array Optional Input -------------- year if True, yearly means. month if True, monthly means. day if True, daily means. hour if True, hourly means. half_hour if True, half-hourly means. minute if True, minutely means. meanday if True, mean daily cycle. meanmonth if True, mean monthly cycle. seasonal if True, seasonal cycle. sum if True, calculate sums instead of means. max if True, calculate maxima instead of means. min if True, calculate minima instead of means. onlydat if True, return only meandat, else return [outdate, meandat] Output ------ outdate centred date on year, month, day, etc. meandat averaged data in 1st dimension. If meanday==True, then all hours will be written; as a masked-array if hours are missing. If meanmonth==True, then all months will be written; as a masked-array if months are missing. If seasonal==True: input data has to be spaced <= days, otherwise consider meanmonth. If seasonal==True, then all days will be written; as a masked-array if days are missing. Note ---- If input date signifies the end of the time step, the user should remove half a time step before using the routine. Otherwise the routine would have to guess the time step, which is error prone. For example, remove 15 minutes in decimal days from time steps of half an hour: date = jams.date2dec(ascii=adate) - 15./(24.*60.) Examples -------- >>> from autostring import astr >>> from date2dec import date2dec >>> dates = ['01.01.1990 12:00:00', '01.02.1990 12:00:00', '01.03.1990 12:00:00', ... '01.01.1990 12:10:00', '01.01.1990 13:00:00', '01.01.1990 14:00:00'] >>> jdates = date2dec(ascii=dates) >>> x = np.arange(len(jdates))+1. >>> odates, xout = means(jdates, x) >>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True)) 2.448e+06 3.500 >>> odates, xout = means(jdates, x, year=True) >>> print(astr(xout, 3, pp=True)) ['3.500'] >>> odates, xout = means(jdates, x, month=True) >>> print(astr(xout, 3, pp=True)) ['4.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, day=True) >>> print(astr(xout, 3, pp=True)) ['4.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, hour=True) >>> print(astr(xout, 3, pp=True)) ['2.500' '5.000' '6.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, half_hour=True) >>> print(astr(xout, 3, pp=True)) ['2.500' '5.000' '6.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, meanday=True) >>> print(astr(xout[10:16], 3, pp=True)) ['-- ' '-- ' '2.500' '5.000' '6.000' '-- '] >>> odates, xout = means(jdates, x, meanmonth=True) >>> print(astr(xout[0:5], 3, pp=True)) ['4.000' '2.000' '3.000' '-- ' '-- '] >>> odates, xout = means(jdates, x, seasonal=True) >>> print(astr(xout[0:5], 3, pp=True)) ['4.000' '-- ' '-- ' '-- ' '-- '] >>> print(astr(means(jdates, x, month=True, onlydat=True), 3, pp=True)) ['4.000' '2.000' '3.000'] # Masked arrays >>> x = np.ma.array(x, mask=np.zeros(x.size, dtype=bool)) >>> x.mask[0] = True >>> odates, xout = means(jdates, x) >>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True)) 2.448e+06 4.000 >>> odates, xout = means(jdates, x, year=True) >>> print(astr(xout, 3, pp=True)) ['4.000'] >>> odates, xout = means(jdates, x, month=True) >>> print(astr(xout, 3, pp=True)) ['5.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, day=True) >>> print(astr(xout, 3, pp=True)) ['5.000' '2.000' '3.000'] # sum >>> odates, xout = means(jdates, x, sum=True) >>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True)) 2.448e+06 20.000 >>> odates, xout = means(jdates, x, year=True, sum=True) >>> print(astr(xout, 3, pp=True)) ['20.000'] >>> odates, xout = means(jdates, x, month=True, sum=True) >>> print(astr(xout, 3, pp=True)) ['15.000' ' 2.000' ' 3.000'] >>> odates, xout = means(jdates, x, day=True, sum=True) >>> print(astr(xout, 3, pp=True)) ['15.000' ' 2.000' ' 3.000'] # max >>> odates, xout = means(jdates, x, max=True) >>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True)) 2.448e+06 6.000 >>> odates, xout = means(jdates, x, year=True, max=True) >>> print(astr(xout, 3, pp=True)) ['6.000'] >>> odates, xout = means(jdates, x, month=True, max=True) >>> print(astr(xout, 3, pp=True)) ['6.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, day=True, max=True) >>> print(astr(xout, 3, pp=True)) ['6.000' '2.000' '3.000'] # min >>> odates, xout = means(jdates, x, min=True) >>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True)) 2.448e+06 2.000 >>> odates, xout = means(jdates, x, year=True, min=True) >>> print(astr(xout, 3, pp=True)) ['2.000'] >>> odates, xout = means(jdates, x, month=True, min=True) >>> print(astr(xout, 3, pp=True)) ['4.000' '2.000' '3.000'] >>> odates, xout = means(jdates, x, day=True, min=True) >>> print(astr(xout, 3, pp=True)) ['4.000' '2.000' '3.000'] # 2D and masked arrays >>> x = np.repeat(x,2).reshape((x.size,2)) >>> odates, xout = means(jdates, x) >>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True)) 2.448e+06 ['4.000' '4.000'] >>> odates, xout = means(jdates, x, year=True) >>> print(astr(xout, 3, pp=True)) [['4.000' '4.000']] >>> odates, xout = means(jdates, x, month=True) >>> print(astr(xout, 3, pp=True)) [['5.000' '5.000'] ['2.000' '2.000'] ['3.000' '3.000']] >>> odates, xout = means(jdates, x, day=True) >>> print(astr(xout, 3, pp=True)) [['5.000' '5.000'] ['2.000' '2.000'] ['3.000' '3.000']] License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2013-2018 Matthias Cuntz - mc (at) macu (dot) de Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, Matthias Cuntz, Jul 2013 Modified, Matthias Cuntz, Jul 2013 - meanday Matthias Cuntz, Apr 2014 - max, min Matthias Cuntz, Jun 2015 - onlydat, meanmonth Matthias Cuntz, Oct 2018 - seasonal Matthias Cuntz, Jun 2019 - bug in minute averaging: compred unique minutes to hours instead of minutes - half_hour inspired by Robin Leucht for level1 data - did not take keyword 'retrospective' from Robin Leucht but added a Note above for this case """ # Constants myundef = 9e33 ismasked = type(dat) == np.ma.core.MaskedArray # Assure array if not ismasked: dat = np.array(dat) # Assure ND-array isone = False if dat.ndim == 1: isone = True dat = dat[:, np.newaxis] # Check options allopts = day + month + year + hour + half_hour + minute + meanday + meanmonth + seasonal assert allopts <= 1, ( "only one averaging option day, month, year, etc. possible") # Check aggregation allaggs = sum + max + min assert allaggs <= 1, "only one aggregation option sum, min, max possible" # average 1st dim if allopts == 0: dout = 0.5 * (date[-1] + date[0]) if sum: out = np.ma.sum(dat, 0) elif max: out = np.ma.amax(dat, 0) elif min: out = np.ma.amin(dat, 0) else: out = np.ma.mean(dat, 0) if isone: if onlydat: return out[0] else: return dout, out[0] else: if onlydat: return out else: return dout, out else: yr, mo, dy, hr, mn, sc = dec2date(date) # year if year: yrs = np.unique(yr) nout = yrs.size dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in yrs: ii = np.where(yr == i)[0] if np.size(ii) > 0: dout[zahl] = date2dec(yr=i, mo=6, dy=15, hr=12) if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 # month if month: yrs = np.unique(yr) mos = np.unique(mo) nout = yrs.size * mos.size dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in yrs: for j in mos: ii = np.where((yr == i) & (mo == j))[0] if np.size(ii) > 0: dout[zahl] = date2dec(yr=i, mo=j, dy=15, hr=12) if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 # day if day: yrs = np.unique(yr) mos = np.unique(mo) dys = np.unique(dy) nout = yrs.size * mos.size * dys.size dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in yrs: for j in mos: for k in dys: ii = np.where((yr == i) & (mo == j) & (dy == k))[0] if np.size(ii) > 0: dout[zahl] = date2dec(yr=i, mo=j, dy=k, hr=12) if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 # hour if hour: yrs = np.unique(yr) mos = np.unique(mo) dys = np.unique(dy) hrs = np.unique(hr) nout = yrs.size * mos.size * dys.size * hrs.size dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in yrs: for j in mos: for k in dys: for l in hrs: ii = np.where((yr == i) & (mo == j) & (dy == k) & (hr == l))[0] if np.size(ii) > 0: dout[zahl] = date2dec(yr=i, mo=j, dy=k, hr=l, mi=30) if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 # half_hour if half_hour: yrs = np.unique(yr) mos = np.unique(mo) dys = np.unique(dy) hrs = np.unique(hr) nout = yrs.size * mos.size * dys.size * hrs.size * 2 dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in yrs: for j in mos: for k in dys: for l in hrs: for m in range(2): ii = np.where((yr == i) & (mo == j) & (dy == k) & (hr == l) & ((mn // 30) == m))[0] if np.size(ii) > 0: dout[zahl] = date2dec(yr=i, mo=j, dy=k, hr=l, mi=15 + m * 30) if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax( dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin( dat[ii, :], 0) else: out[zahl, :] = np.ma.mean( dat[ii, :], 0) zahl += 1 # minute if minute: yrs = np.unique(yr) mos = np.unique(mo) dys = np.unique(dy) hrs = np.unique(hr) mns = np.unique(mn) nout = yrs.size * mos.size * dys.size * hrs.size * mns.size dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in yrs: for j in mos: for k in dys: for l in hrs: for m in mns: ii = np.where((yr == i) & (mo == j) & (dy == k) & (hr == l) & (mn == m))[0] if np.size(ii) > 0: dout[zahl] = date2dec(yr=i, mo=j, dy=k, hr=l, mi=m, sc=30) if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax( dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin( dat[ii, :], 0) else: out[zahl, :] = np.ma.mean( dat[ii, :], 0) zahl += 1 # Mean daily if meanday: nout = 24 hrs = range(nout) dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in hrs: dout[zahl] = date2dec(yr=yr[0], mo=1, dy=1, hr=i, mi=30) ii = np.where(hr == i)[0] if np.size(ii) > 0: if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 if np.any(out == myundef): out = np.ma.array(out, mask=(out == myundef), keep_mask=True) # Mean monthly if meanmonth: nout = 12 mos = range(1, nout + 1) dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in mos: dout[zahl] = date2dec(yr=yr[0], mo=i, dy=15, hr=12, mi=0) ii = np.where(mo == i)[0] if np.size(ii) > 0: if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 if np.any(out == myundef): out = np.ma.array(out, mask=(out == myundef), keep_mask=True) # Seasonal if seasonal: dim = np.array( [[-9, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], [-9, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]]) # leaps = np.array((((yr%4)==0) & ((yr%100)!=0)) | # ((yr%400)==0)).astype(int) leaps = (((yr % 4) == 0) & ((yr % 100) != 0)) | ((yr % 400) == 0) leap = np.any(leaps) ileap = int(leap) if leap: iileap = np.argmax(leaps) nout = 366 else: iileap = 0 nout = 365 dys = range(1, nout + 1) dout = np.ones(nout) * myundef if ismasked: out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef else: out = np.ones([nout] + list(dat.shape[1:])) * myundef zahl = 0 for i in range(1, 13): for j in range(1, dim[ileap, i] + 1): dout[zahl] = date2dec(yr=yr[iileap], mo=i, dy=j, hr=12, mi=0) ii = np.where((mo == i) & (dy == j))[0] if np.size(ii) > 0: if sum: out[zahl, :] = np.ma.sum(dat[ii, :], 0) elif max: out[zahl, :] = np.ma.amax(dat[ii, :], 0) elif min: out[zahl, :] = np.ma.amin(dat[ii, :], 0) else: out[zahl, :] = np.ma.mean(dat[ii, :], 0) zahl += 1 if np.any(out == myundef): out = np.ma.array(out, mask=(out == myundef), keep_mask=True) ii = np.where(dout != myundef)[0] if len(ii) > 0: dout = dout[ii] out = out[ii, :] else: raise ValueError("all values undefined after mean.") if isone: if onlydat: return out[:, 0] else: return dout, out[:, 0] else: if onlydat: return out else: return dout, out
def fluxplot(infile, outfile, numheader=1, lineofvar=1, novalue=-9999, delimiter=',', format='ascii', format_str='%d-%m-%Y %H:%M', units=False, plot=False): ''' Plotting routine for Eddy Covariance data generated by EddyFlux (and basically any other ascii data file) with first column containing time steps in either 'asci' or 'eng' time format. Loads the file and plots each column in a separate page of a pdf file. Missing values are plotted in red. Definition ---------- fluxplot(infile, outfile, numheader=1, lineofvar=1, novalue=-9999, format='ascii', format_str_x='%d-%m-%Y %H:%M', units=False, plot=False): Input ----- infile str, path and name of the input file outfile str, path and name of the output pdf Optional Input -------------- numhead int, number of header lines in the input files (default: 1) lineofvar int, line in input file containing variable names (default: 1) novalue int/float, missing value in input file (default: -9999) delimiter str, delimiter of the input file format str, format of time stamps in input files. 'ascii' or 'eng' is possible (default: 'ascii') format_str str, format string for plot date formatting (default: '%d-%m-%Y %H:%M') units bool, if True, all units of a standard EddyFlux output file are added to each plot. Needs to be set False if other files are plotted (default: False) plot bool, only if True, the plot is performed (default: False) Output ------ outfile pdf with each page containing a plot of one column of the input file License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' if plot: import matplotlib matplotlib.use('PDF') import matplotlib.pyplot as plt import matplotlib.ticker as ticker from matplotlib.backends.backend_pdf import PdfPages from matplotlib import rcParams else: raise ValueError('fluxplot: plot must be True') ############################################################################ # set parameters for matplotlib rcParams['ps.papersize'] = 'a4' rcParams['savefig.orientation'] = 'landscape' rcParams['lines.linewidth'] = 0.75 rcParams['axes.linewidth'] = 1 rcParams['axes.titlesize'] = 12 rcParams['axes.labelsize'] = 8 rcParams['xtick.labelsize'] = 8 rcParams['ytick.labelsize'] = 8 rcParams['legend.fontsize'] = 8 rcParams['path.simplify'] = False ############################################################################ # reading input file d = np.loadtxt(infile, dtype='|S100', delimiter=delimiter) header = d[lineofvar - 1, 1:] date = d[numheader:, 0] value = d[numheader:, 1:] value[value == ''] = str(novalue) value = value.astype(np.float) missing = np.where(value == novalue, 0, np.NaN) value = np.where(value == novalue, np.NaN, value) length = len(date) if units: units = [ 'W m-2', 'W m-2', 'mmol m-2 s-1', 'mumol m-2 s-1', 'm s-1', 'kg m-1 s-2', 'm2 s-2', 'm2 s-2', 'm2 s-2', 'K2', 'mmol2 m-6', 'mmol2 m-6', 'm s-1', 'm s-1', 'm s-1', ' deg C', 'mmol m-3', 'mmol m-3', 'm s-1', 'deg', 'deg', 'deg', 'deg', 'samples', 'samples', 'm', '-', 'deg C', 'W m-2', 'W m-2', 'mmol m-2 s-1', 'mumol m-2 s-1', 'm', 'm', 'm', '-', '-', '-', '-', '-', '-', 'm2 s-2', 'm2 s-2', 'm2 s-2', '-', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'samples', 'hPa', 'deg C', '%', 'hPa', 'kg m-3', 'samples', 'samples', 'm', 'm' ] else: units = [''] * length ############################################################################ # convert input date in JD and array format date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) if format == 'ascii': jdarr = date2dec(ascii=date) - date01 elif format == 'eng': jdarr = date2dec(eng=date) - date01 else: raise ValueError('fluxplot: format unknown') ############################################################################ # plot majticks = matplotlib.dates.MonthLocator(bymonthday=1) pdf_pages = PdfPages(outfile) ############################################################################ for i in xrange(len(header)): #len(header) fig = plt.figure(i) ax = fig.add_subplot(111) ax.plot(jdarr, value[:, i], 'b-') ylim = ax.get_ylim() if (ylim[0] < 0) and (ylim[1] > 0): ax.axhline(y=0, linewidth=0.75, color='k') ax.plot(jdarr, missing[:, i], 'r-', lw=2) else: ax.plot(jdarr, np.where(missing[:, i] == 0, ylim[0], np.NaN), 'r-', lw=2) ax.set_xlabel('Date [DD-MM-YYYY]') ylabel = '%s \n %s' % (header[i], units[i]) ax.set_ylabel(ylabel) ax.set_xlim(jdarr[0], jdarr[-1]) ax.xaxis.set_major_locator(majticks) ax.xaxis.set_major_formatter( matplotlib.dates.DateFormatter(format_str)) fig.autofmt_xdate() pdf_pages.savefig(fig) plt.close() pdf_pages.close()
def fluxpart(fluxfile, metfile, outdir, swdr, tair, rh, method='local', nogppnight=False, delimiter=[',', ','], skiprows=[1, 1], format=['ascii', 'ascii'], undef=-9999, plot=False): ''' Wrapper function for nee2gpp with file management and plotting. Definition ---------- fluxpart(fluxfile, metfile, outdir, rg, tair, rh, method='local', delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag or fluxfill output file containing fluxes and flags metfile str, path and file name of the meteorology file (must be in sync with fluxfile) outdir str, path of the output folder swdr int, column number of short wave downward radiation [W m-2] in metfile, column number starts with 0 which is first data column. swdr is used for lasslopp and for swdr>0=isday tair int, column number of air temperature [deg C] in metfile, column number starts with 0 which is first data column. rh int, column number of relative humidity [%] in metfile, column number starts with 0 which is first data column. Optional Input -------------- method str, if 'global', fit of Reco vs. temperature to all nighttime data if 'local' | 'reichstein', method of Reichstein et al. (2005) if 'day' | 'lasslop', method of Lasslop et al. (2010) (default: 'local') nogppnight if True: Resp=NEE, GPP=0 at night, GPP always positive if False: Resp=lloyd_taylor, GPP=Resp-NEE at night (default) delimiter list of str, delimiters of fluxfile and metfile (default: [',',',']) skiprows list of int, lines to skip at the beginning of fluxfile and metfile, e.g. header lines (default: [1,1]) format list of str, time formats of fluxfile and metfile, 'ascii' and 'eng' possible (default: ['ascii','ascii']) undef int/float, missing value of fluxfile and metfile (default: -9999, np.nan is not possible) plot bool, if True performs plotting (default: False) Output ------ fluxpart.csv file containing fluxes with original flags and quality flags of the gap filling plus gpp and reco fluxes. gpp and reco get flags of c flux. fluxpart.pdf plot of c flux, gpp and reco License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' ########################################################################### # reading input files d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0], skiprows=skiprows[0]) m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1], skiprows=skiprows[1]) assert (d.shape[1] == 16) | ( d.shape[1] == 24 ), 'fluxfill: fluxfile must be from fluxfill and have 16 or 24 cols' if format[0] == 'ascii': datev = date2dec(ascii=d[:, 0]) elif format[0] == 'eng': datev = date2dec(eng=d[:, 0]) else: raise ValueError('fluxpart: unknown format') if format[1] == 'ascii': datem = date2dec(ascii=m[:, 0]) elif format[1] == 'eng': datem = date2dec(eng=m[:, 0]) else: raise ValueError('fluxpart: unknown format') val = np.where(d[:, 1:] == '', str(undef), d[:, 1:]).astype(np.float) met = np.where(m[:, 1:] == '', str(undef), m[:, 1:]).astype(np.float) ########################################################################### # assign variables if (d.shape[1] == 16): nee = val[:, 9] #corr. C else: nee = val[:, 12] # oder 13 #corr. C nee_stor = val[:, 13] # oder 13 #corr. C swdr = met[:, swdr] isday = swdr > 0. tair = met[:, tair] + 273.15 rh = met[:, rh] #vpd = (1.-rh/100.)*esat(tair) vpd = np.empty_like(tair) vpd[(tair == undef) | (rh == undef)] = undef vpd[~((tair == undef) | (rh == undef))] = (1. - rh[~((tair == undef) | (rh == undef))] / 100.) * esat(tair[~( (tair == undef) | (rh == undef))]) ########################################################################### # do partitioning if (d.shape[1] == 16): gpp, reco = nee2gpp.nee2gpp(datev, nee, tair, isday, rg=swdr, vpd=vpd, undef=undef, method=method, shape=False, masked=False, nogppnight=nogppnight) else: gpp, reco = nee2gpp.nee2gpp(datev, nee, tair, isday, rg=swdr, vpd=vpd, undef=undef, method=method, shape=False, masked=False, nogppnight=nogppnight) gpp_stor, reco_stor = nee2gpp.nee2gpp(datev, nee_stor, tair, isday, rg=swdr, vpd=vpd, undef=undef, method=method, shape=False, masked=False, nogppnight=nogppnight) ####################################################################### # plot if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf pp1 = pdf.PdfPages(outdir + '/fluxpart.pdf') majticks = mpl.dates.MonthLocator(bymonthday=1) format_str = '%d %m %Y %H:%M' date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) fig1 = plt.figure(1) if (d.shape[1] == 16): sub1 = fig1.add_subplot(111) else: sub1 = fig1.add_subplot(211) sub2 = fig1.add_subplot(212) l1 = sub1.plot(datev - date01, nee, '-k', label='nee') l2 = sub1.plot(datev - date01, gpp, '-g', label='gpp') l3 = sub1.plot(datev - date01, reco, '-r', label='reco') if (d.shape[1] != 16): l4 = sub2.plot(datev - date01, nee_stor, '-k', label='nee') l5 = sub2.plot(datev - date01, gpp_stor, '-g', label='gpp') l6 = sub2.plot(datev - date01, reco_stor, '-r', label='reco') sub1.set_xlim(datev[0] - date01, datev[-1] - date01) sub1.xaxis.set_major_locator(majticks) sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) if (d.shape[1] != 16): sub2.set_xlim(datev[0] - date01, datev[-1] - date01) sub2.xaxis.set_major_locator(majticks) sub2.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) fig1.autofmt_xdate() plt.legend(loc='best') plt.show() fig1.savefig(pp1, format='pdf') pp1.close() ########################################################################### # prepare output and save file if (d.shape[1] == 16): header = np.array([ ' H', ' Hflag', ' Hgf', ' LE', ' LEflag', ' LEgf', ' E', ' Eflag', ' Egf', ' C', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf', ' GPP', ' GPPflag', ' GPPgf', ' Reco', 'Recoflag', ' Recogf' ]) flux = np.vstack((gpp, reco)).transpose() flux_str = np.array([['%11.5f' % x for x in y] for y in flux]).repeat(3, axis=1) flux_str[:, [1, 2, 4, 5]] = np.tile(d[:, 11:13], 2) output = np.hstack((d[:, :], flux_str)) else: header = np.array([ ' H', ' H+sT', ' Hflag', ' Hgf', ' LE', ' LE+sLE', ' LEflag', ' LEgf', ' E', ' E+sE', ' Eflag', ' Egf', ' C', ' C+sC', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf', ' sT', ' sLE', ' sE', ' sC', ' GPP', ' GPP+sC', ' GPPflag', ' GPPfg', ' Reco', ' Reco+sC', 'Recoflag', ' Recofg' ]) flux = np.vstack((gpp, gpp_stor, reco, reco_stor)).transpose() flux_str = np.array([['%11.5f' % x for x in y] for y in flux]) flux_str = np.insert(flux_str, [2, 2, 4, 4], flux_str, axis=1) flux_str[:, [2, 3, 6, 7]] = np.tile(d[:, 11:13], 2) output = np.hstack((d[:, :], flux_str)) np.savetxt('%s/fluxpart.csv' % outdir, np.vstack((np.concatenate( ([' date'], header))[np.newaxis, :], output)), '%s', delimiter=',')
def fluxflag(fluxfile, metfile, outdir, swdr, T, lat, delimiter=[',', ','], skiprows=[1, 1], format=['ascii', 'ascii'], limit=0.3, analyzer='LI7000', spike_f=True, itc_f=True, ustar_f=True, novalue=-9999, plot=False): ''' Quality flag calculation for Eddy Covariance data originating from EddyFlux. Including stationarity, exceeding limits, exceeding change rates, exceeding variances, integral turbulence characteristics, spike detection and ustar thresholds. Missing values are flagged with 2. Ustar thresholding works ONLY for a data set of ONE FULL year. Definition ---------- fluxflag(fluxfile, metfile, outdir, Rg, T, lat, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], limit=0.3, analyzer='LI7000', spike_f=True, itc_f=True, ustar_f=True, novalue=-9999, plot=False): Input ----- fluxfile str, path and file name of EddyFlux output file metfile str, path and file name of the meteorology file (must be in sync with fluxfile) outdir str, path of the output folder swdr int, column number of short wave incoming radiation in metfile, column number starts with 0 which is first data column. Used for swdr>0=isday T int, column number of air temperature in metfile, column number starts with 0 which is first data column. Used for ustarflag lat latitude of tower position in decimal degrees. Used for itcflag Optional Input -------------- delimiter list of str, delimiters of fluxfile and metfile (default: [',',',']) skiprows list of int, lines to skip at the beginning of fluxfile and metfile, e.g. header lines (default: [1,1]) format list of str, time formats of fluxfile and metfile, 'ascii' and 'eng' possible (default: ['ascii','ascii']) limit float, relative deviation limit from the itc model above which values are flagged with itcflag. e.g. 0.3 means 30% above and 30% below the model (default: 0.3) analyzer str, analyzer type. 'LI7000' and 'LI7500' possible. Used for exceeding variance limits. (default: 'LI7000') spike_f bool, if True spike detection and flagging is performed itc_f bool, if True itc calculation and flagging is performed ustar_f bool, if True ustar threshold calculation and flagging is performed novalue int/float, missing value of fluxfile and metfile (default: -9999) plot bool, if True subroutines (itc, spike and ustar) perform plotting (default: False) Output ------ flags.csv file containing all calculated flags fluxflags.csv file containing fluxes with the sum of all respective flags for each flux. Where flag>2, flag=2. flags.log file containing statistics for each flag Restrictions ------------ - ustar flagging works ONLY for a data set of ONE FULL year - ustar flagging works ONLY for half hourly time steps License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' ########################################################################### # reading input files d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0], skiprows=skiprows[0]) m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1], skiprows=skiprows[1]) if format[0] == 'ascii': datev = date2dec(ascii=d[:, 0]) elif format[0] == 'eng': datev = date2dec(eng=d[:, 0]) else: raise ValueError('fluxflag: unknown format') if format[1] == 'ascii': datem = date2dec(ascii=m[:, 0]) elif format[1] == 'eng': datem = date2dec(eng=m[:, 0]) else: raise ValueError('fluxflag: unknown format') val = np.where(d[:, 1:] == '', str(novalue), d[:, 1:]).astype(np.float) val = np.where(val == novalue, np.NaN, val) met = np.where(m[:, 1:] == '', str(novalue), m[:, 1:]).astype(np.float) met = np.where(met == novalue, np.NaN, met) print('LOADING FILES COMPLETED') ########################################################################### # calculate isday isday = met[:, swdr] > 0. fluxes = val[:, [28, 29, 30, 31, 5]] # corr. H, corr. LE, corr. E, corr. C, tau zeta = val[:, 26] varu = val[:, 41] ustar = val[:, 4] varw = val[:, 8] vart = val[:, 9] rho = val[:, 62] H = val[:, 28] T = met[:, T] ########################################################################### # calculate standard flags sttest2 = np.where(np.isnan(val[:, 37:38]), np.NaN, np.unpackbits(val[:, 37:38].astype('uint8'), axis=1)) h2oexli = np.where(np.isnan(val[:, 56]), np.NaN, np.where(val[:, 56] > 100, 2, 0)) c02exli = np.where(np.isnan(val[:, 55]), np.NaN, np.where(val[:, 55] > 100, 2, 0)) h2oexcr = np.where(np.isnan(val[:, 50]), np.NaN, np.where(val[:, 50] > 50, 2, 0)) co2excr = np.where(np.isnan(val[:, 49]), np.NaN, np.where(val[:, 49] > 50, 2, 0)) texcr = np.where(np.isnan(val[:, 48]), np.NaN, np.where(val[:, 48] > 50, 2, 0)) tauexcr = np.where( np.isnan(val[:, 45]) ^ np.isnan(val[:, 47]), np.NaN, np.where((val[:, 45] > 50) ^ (val[:, 47] > 50), 2, 0)) zetaexcr = np.where( np.isnan(val[:, 45]) ^ np.isnan(val[:, 47]) ^ np.isnan(val[:, 48]), np.NaN, np.where((val[:, 45] > 50) ^ (val[:, 47] > 50) ^ (val[:, 48] > 50), 2, 0)) wexvar = np.where(np.isnan(val[:, 8]), np.NaN, np.where(val[:, 8] > 3, 1, 0)) texvar = np.where(np.isnan(val[:, 9]), np.NaN, np.where(val[:, 9] > 3, 1, 0)) if analyzer == 'LI7000': h2oexvar3 = np.where(np.isnan(val[:, 11]), np.NaN, np.where(val[:, 11] > 3, 1, 0)) h2oexvar10 = np.where(np.isnan(val[:, 11]), np.NaN, np.where(val[:, 11] > 10, 1, 0)) co2exvar3 = np.where(np.isnan(val[:, 10]), np.NaN, np.where(val[:, 10] > 3, 1, 0)) elif analyzer == 'LI7500': h2oexvar3 = np.where(np.isnan(val[:, 11]), np.NaN, np.where(val[:, 11] > 5000, 1, 0)) h2oexvar10 = np.where(np.isnan(val[:, 11]), np.NaN, np.where(val[:, 11] > 17300, 1, 0)) co2exvar3 = np.where(np.isnan(val[:, 10]), np.NaN, np.where(val[:, 10] > 3.5, 1, 0)) else: raise ValueError('fluxflag: unknown analyzer') wvar0 = np.where(np.isnan(val[:, 8]), np.NaN, np.where(val[:, 8] < 0.00001, 2, 0)) h2ovar0 = np.where(np.isnan(val[:, 11]), np.NaN, np.where(val[:, 11] < 0.00001, 2, 0)) co2var0 = np.where(np.isnan(val[:, 10]), np.NaN, np.where(val[:, 10] < 0.00001, 2, 0)) print('CALCULATE STANDARD FLAGS COMPLETE') ########################################################################### # itc flag calculation if itc_f: itcu, itcw, itct = itc.itc(H, zeta, ustar, varu, varw, vart, rho, lat, limit, '%s/itc' % outdir, plot=plot) print('ITC CALCUATION COMPLETED') else: itcu = np.zeros_like(H, dtype=np.int) itcw = np.zeros_like(H, dtype=np.int) itct = np.zeros_like(H, dtype=np.int) ########################################################################### # summing flags for each flux Hflag = np.nansum( np.vstack( (sttest2[:, 6], itcw, texcr, wexvar, texvar, wvar0)).transpose(), 1).astype(int) Hflag[Hflag > 1] = 2 LEflag = np.nansum( np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3, wvar0, h2ovar0)).transpose(), 1).astype(int) LEflag[LEflag > 1] = 2 Eflag = np.nansum( np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3, wvar0, h2ovar0)).transpose(), 1).astype(int) Eflag[Eflag > 1] = 2 Cflag = np.nansum( np.vstack((sttest2[:, 4], itcw, c02exli, co2excr, wexvar, h2oexvar10, wvar0, co2var0)).transpose(), 1).astype(int) #, co2exvar3 Cflag[Cflag > 1] = 2 Tauflag = np.nansum( np.vstack( (sttest2[:, 3], itcu, itcw, tauexcr, wexvar, wvar0)).transpose(), 1).astype(int) Tauflag[Tauflag > 1] = 2 ########################################################################### # spike detection #inflag = np.zeros_like(fluxes, dtype=np.int) inflag = np.vstack((Hflag, LEflag, Eflag, Cflag, Tauflag)).transpose() if spike_f: spikef = spikeflag.spikeflag(datev, fluxes, inflag, isday, '%s/spike' % outdir, z=7, deriv=1, udef=np.NaN, spike_v=2, plot=plot) print('SPIKE DETECTION COMPLETED') else: spikef = np.zeros_like(inflag, dtype=np.int) inflag += spikef ########################################################################### # ustar flagging if ustar_f: C_ustar_T = np.vstack((fluxes[:, 3], ustar, T)).transpose() C_ustar_T_flags = np.isnan(C_ustar_T).astype(np.int) C_ustar_T_flags[:, 0] += inflag[:, 3] ustarf = ustarflag.ustarflag(datev, C_ustar_T, C_ustar_T_flags, isday, '%s/ustar' % outdir, ustar_v=2, plot=plot) print('USTAR FLAGGING COMPLETED') else: ustarf = np.zeros_like(datev, dtype=np.int) ########################################################################### # the big gathering :-D header = np.array([ 'sttest2_H', ' sttest2_LE/E', ' sttest2_C', ' sttest2_tau', ' h2oexli', ' c02exli', ' h2oexcr', ' co2excr', ' texcr', ' tauexcr', ' zetaexcr', ' wexvar', ' texvar', ' h2oexvar3', ' h2oexvar10', ' co2exvar3', ' wvar0', ' h2ovar0', ' co2var0', ' itcu', ' itcw', ' itct', ' spike_H', ' spike_LE', ' spike_E', ' spike_C', ' spike_tau', ' ustar' ]) maxlen = [len(x) for x in header] flags = np.vstack( (sttest2[:, 6], sttest2[:, 5], sttest2[:, 4], sttest2[:, 3], h2oexli, c02exli, h2oexcr, co2excr, texcr, tauexcr, zetaexcr, wexvar, texvar, h2oexvar3, h2oexvar10, co2exvar3, wvar0, h2ovar0, co2var0, itcu, itcw, itct, spikef[:, 0], spikef[:, 1], spikef[:, 2], spikef[:, 3], spikef[:, 4], ustarf)).transpose() flags = np.where(np.isnan(flags), 2, flags).astype(np.int) flags_str = np.zeros_like(flags, dtype='|S100') for i, item in enumerate(maxlen): flags_str[:, i] = astr(flags[:, i], prec=item) # save flag file output = np.hstack((d[:, 0:1], flags_str)) np.savetxt('%s/flags.csv' % outdir, np.vstack((np.concatenate( ([' date'], header))[np.newaxis, :], output)), '%s', delimiter=',') ########################################################################### # summing flags for each flux Hflag = np.nansum( np.vstack((sttest2[:, 6], itcw, texcr, wexvar, texvar, wvar0, spikef[:, 0])).transpose(), 1).astype(int) Hflag[Hflag > 1] = 2 LEflag = np.nansum( np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3, wvar0, h2ovar0, spikef[:, 1])).transpose(), 1).astype(int) LEflag[LEflag > 1] = 2 Eflag = np.nansum( np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3, wvar0, h2ovar0, spikef[:, 2])).transpose(), 1).astype(int) Eflag[Eflag > 1] = 2 Cflag = np.nansum( np.vstack((sttest2[:, 4], itcw, c02exli, co2excr, wexvar, h2oexvar10, co2exvar3, wvar0, co2var0, spikef[:, 3], ustarf)).transpose(), 1).astype(int) ### why h20exvar10 Cflag[Cflag > 1] = 2 Tauflag = np.nansum( np.vstack((sttest2[:, 3], itcu, itcw, tauexcr, wexvar, wvar0, spikef[:, 4])).transpose(), 1).astype(int) Tauflag[Tauflag > 1] = 2 header2 = np.array([ ' H', ' Hflag', ' LE', ' LEflag', ' E', ' Eflag', ' C', ' Cflag', ' TAU', ' TAUflag' ]) fluxes[np.isnan(fluxes)] = novalue fluxes_str = np.array([['%11.5f' % x for x in y] for y in fluxes]) fluxes_str = np.where(fluxes_str == '%11.5f' % novalue, ' ' * (11 - len(str(novalue))) + str(novalue), fluxes_str) Hflag_str = astr(Hflag, prec=8) LEflag_str = astr(LEflag, prec=8) Eflag_str = astr(Eflag, prec=8) Cflag_str = astr(Cflag, prec=8) Tauflag_str = astr(Tauflag, prec=8) output = np.hstack((d[:, 0:1], fluxes_str.repeat(2, axis=1))) output[:, 2::2] = np.vstack((Hflag_str, LEflag_str, Eflag_str, Cflag_str, Tauflag_str)).transpose() np.savetxt('%s/fluxflags.csv' % outdir, np.vstack((np.concatenate( ([' date'], header2))[np.newaxis, :], output)), '%s', delimiter=',') ########################################################################### # write log file with stats log = open('%s/flags.log' % outdir, 'w') log.write('flag, 0, 1, 2\n') for i, item in enumerate(header): hist = tuple(np.histogram(flags[:, i], [-0.5, 0.5, 1.5, 2.5])[0]) log.write(item.strip() + ', %i, %i, %i\n' % hist) log.close()
def meteo4slt(sltdir, metfile, p_t_rh, outfile, pat='[a-zA-Z0-9]*.slt|[a-zA-Z0-9]*.SLT', delimiter=',', skiprows=1, format='ascii'): """ To supply EddyFlux (Kolle & Rebmann, 2007) with meteorological data, it is necessary to extract for each available *.slt file the corresponding air pressure, air temperature and air relative humidity. The module looks in sltdir for available *.slt files and uses the metfile for syncronisation Definition ---------- meteo4slt(sltdir, metfile, p_t_rh, outfile, pat='[a-zA-Z0-9]*.slt|[a-zA-Z0-9]*.SLT', delimiter=',', skiprows=1, format='ascii'): Input ----- sltdir str, path of the folder containing the *.slt files metfile str, path of the meteo file p_t_rh list, column number of Pressure, T, rH outfile str, path of the output file Optional Input -------------- pat str, regular expression, describing the name pattern of the *.slt files in the indir folder delimiter str, column delimiter of the meteo file (default=',') skiprows int, number of rows to skip at the beginning of the met file e.g. header lines (default=1) format str, time format of the meteo file (default='ascii') Output ------ outfile file, containing Pressure, T and rH values of the meteo file for each *.slt file Restrictions ------------ - assumes site name in slt fielname is only one character (e.g. W20133652300.slt AND NOT WULF20133652300.slt) - currently only supports format='ascii', nice would be 'eng' - does not work for data containning multiple years because of ugly doy calculation - works only for half hourly *.stl and half hourly meteo data License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Jul 2014 Modified, MC, Aug 2014 - clean up and Python 3 """ # half hour in jd halfhjd = 1800./86400. ########################################################################### # reading slt directory dirlist = os.listdir(sltdir) sltdates = np.array([]) ########################################################################### # remove all files and folders from list which are not *.slt files and get size pat = re.compile(pat) for item in dirlist: if re.search(pat, item): sltdates = np.append(sltdates, item[1:-4]) # replace time steps which dont fit in half hour interval mi = np.array([x[-2:] for x in sltdates]) mi = np.where(mi.astype(int)<30, '00', '30') sltdates = np.array([sltdates[i][:-2]+mi[i] for i in range(mi.size)]) ########################################################################### # load meteo data metdata = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter, skiprows=skiprows, usecols=[0]+p_t_rh) # get the metdate if format == 'ascii': # shift met dates one half hour back since slt time stamp marks half # hour start but meteo date mark half hour end jd = date2dec(ascii=metdata[:,0]) - halfhjd fulldate = dec2date(jd, fulldate=True) adate = dec2date(jd, ascii=True) doy = np.ceil(date2dec(ascii=adate)- date2dec(yr=fulldate[0][0], mo=1,dy=1,hr=0,mi=0, sc=0)).astype(int) doy = np.where((fulldate[3]==0) & (fulldate[4]==0), doy+1, doy) metdates = np.array(['%04i%03i%02i%02i'%(fulldate[0][i], doy[i], fulldate[3][i], fulldate[4][i]) for i in range(jd.size)]) elif format == 'eng': # shift met dates one half hour back since slt time stamp marks half # hour start but meteo date mark half hour end jd = date2dec(eng=metdata[:,0]) - halfhjd fulldate = dec2date(jd, fulldate=True) adate = dec2date(jd, ascii=True) doibegin = np.array([date2dec(yr=x, mo=1,dy=1,hr=0,mi=0, sc=0) for x in fulldate[0]]) doy = np.ceil(date2dec(ascii=adate)-doibegin).astype(int) doy = np.where((fulldate[3]==0) & (fulldate[4]==0), doy+1, doy) metdates = np.array(['%04i%03i%02i%02i'%(fulldate[0][i], doy[i], fulldate[3][i], fulldate[4][i]) for i in range(jd.size)]) else: raise ValueError('meteo4slt: unknown format!') ########################################################################### # sync both dates mask = np.in1d(metdates, sltdates) ########################################################################### # write output np.savetxt(outfile, metdata[mask,1:], '%s', ',')
def eddyspec(indir, cfile, hfile, rawfile, sltdir, tspfile='34_specmean.csv', cspfile='35_specmean.csv', hspfile='36_specmean.csv', novalue=-9999, plot=False): ''' Provides plots with carbon and water fluxes from the rawfile together with the respective time lags in the cfile and hfile for the user to select 'golden days' with sufficient flux and constant lags (about 6 half hour values during midday) to perform spectrum analysis with EddySpec and SpecMean (Kolle & Rebmann, 2007). When user selected desired time period, the respective *.slt files are printed to the console. The user can than use EddySpec and SpecMean with these files and the script continues after user is finished. Inductances for water and carbon are fitted and saved together with spectrum plots. It is recommended to repeat the spectrum analysis for different times within the year. Definition ---------- eddyspec(indir, cfile, hfile, rawfile, sltdir, tspfile='34_specmean.csv', cspfile='35_specmean.csv', hspfile='36_specmean.csv', novalue=-9999): Input ----- indir str, path of the folder where results will be saved cfile str, path of the carbon lag file hfile str, path of the water lag file rawfile str, path of the raw flux file sltdir str, path of the folder containing the *.slt files (will contain EddySpec and SpecMean files after use) Optional Input -------------- tspfile str, name of the average temperature spectrum file (default: '34_specmean.csv') cspfile str, name of the average carbon spectrum file (default: '35_specmean.csv') hspfile str, name of the average water spectrum file (default: '36_specmean.csv') novalue int, novalue in rawfile (default=-9999) Output ------ (for each run in the year, a new folder is created in indir and EddySpec and SpecMean files are moved here from sltdir) c_specw.pdf plot of the carbon spectrum h_specw.pdf plot of the water spectrum spec_X_X.log log file containing used *.slt files and inductances License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Jul 2014 Modified, MC, Aug 2014 - clean up and Python 3 ''' ############################################################################ # reading input files #lags lagdate = np.array(sread('%s' % (cfile), nc=1, skip=1), dtype='|S16') day = np.array([x[5:8] for x in lagdate.flatten()], dtype='|S7').astype(float) hour = np.array([x[8:10] for x in lagdate.flatten()], dtype='|S2').astype(float) min = np.array([x[10:12] for x in lagdate.flatten()], dtype='|S2').astype(float) doysfloat = day + (hour + min / 60.) / 24. cl = np.array(fread('%s' % (cfile), skip=1, cskip=1)) hl = np.array(fread('%s' % (hfile), skip=1, cskip=1)) # fluxes fluxdate = date2dec( ascii=np.array(sread('%s' % (rawfile), nc=1, skip=1), dtype='|S16')) year = np.array(sread('%s' % (rawfile), nc=1, skip=1), dtype='|S16') year = np.array([x[6:10] for x in year.flatten()], dtype='|S4').astype(int) fluxdate = fluxdate.flatten() - date2dec( yr=year, mo=1, dy=1, hr=0, mi=0, sc=0) cf = np.array(fread('%s' % (rawfile), skip=1, cskip=4, nc=1)) cf = np.where(cf == novalue, np.nan, cf).flatten() hf = np.array(fread('%s' % (rawfile), skip=1, cskip=1, nc=1)) hf = np.where(hf == novalue, np.nan, hf).flatten() if plot: import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf ############################################################################ # plot lag c plt.figure(1) plt.plot(cl[:, 0], cl[:, 2], 'bo', label='minlag (sam)') plt.plot(cl[:, 0], cl[:, 5], 'ro', label='maxlag (sam)') plt.xlabel('DOY') plt.ylabel('Lags [Samples]') plt.title('c lags') plt.axis('auto') plt.grid('on') plt.legend() ############################################################################ # plot flux c plt.figure(2) plt.plot(fluxdate, cf, 'r-', label='cflux') plt.xlabel('DOY') plt.ylabel('cflux [?]') plt.title('c fluxes') plt.axis('auto') plt.grid('on') plt.legend() ############################################################################ # plot lag h plt.figure(3) plt.plot(hl[:, 0], hl[:, 2], 'bo', label='minlag (sam)') plt.plot(hl[:, 0], hl[:, 5], 'ro', label='maxlag (sam)') plt.xlabel('DOY') plt.ylabel('Lags [Samples]') plt.title('h lags') plt.axis('auto') plt.grid('on') plt.legend() ############################################################################ # plot flux h plt.figure(4) plt.plot(fluxdate, hf, 'b-', label='hflux') plt.xlabel('DOY') plt.ylabel('hflux [?]') plt.title('h fluxes') plt.axis('auto') plt.grid('on') plt.legend() plt.show() interval = [] inp = True while inp: inpfrom = raw_input("From doy [ddd.ddd]: ") inpto = raw_input("To doy [ddd.ddd]: ") try: interval += [ np.min(np.where(np.abs(doysfloat - float(inpfrom)) < 0.02083)) ] interval += [ np.min(np.where(np.abs(doysfloat - float(inpto)) < 0.02083)) ] inp = False except (ValueError, IndexError): print( 'EddySpecWarning: type in floats(with . not ,), nothing else!') inp = True inpclag = raw_input("C lag: ") inphlag = raw_input("H lag: ") print(lagdate[interval[0]:interval[1] + 1]) print('\nDO EDDYSPEC AND SPECMEAN NOW!\n') inp = raw_input("Mean spectrums ready? [y/n]: ").lower() if inp == "y": pass else: import sys sys.exit() ############################################################################ # move spectrum files from slt folder to spec folder if len(os.listdir(indir)) == 0: os.mkdir(indir + '/1') indir = indir + '/1' else: os.mkdir(indir + '/%i' % (int(os.listdir(indir)[-1]) + 1)) indir = indir + '/' + os.listdir(indir)[-1] specpat = re.compile('[0-9]*_specmean.csv') specpat2 = re.compile('[a-zA-Z0-9]*_[0-9]*_spec.csv') sltdirlist = os.listdir(sltdir) for file in sltdirlist: if bool(re.search(specpat, file)) | bool(re.search(specpat2, file)): sh.move('%s/%s' % (sltdir, file), indir) ############################################################################ # conductance fitting # reading mean spec files tsp = np.array(fread('%s/%s' % (indir, tspfile), skip=1, nc=2)) csp = np.array(fread('%s/%s' % (indir, cspfile), skip=1, nc=2)) hsp = np.array(fread('%s/%s' % (indir, hspfile), skip=1, nc=2)) # filter input specs for log10(input) != NaN global fcoc, fcoh fcoc = csp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1])))) & (np.invert(np.isnan(np.log10(csp[:, 1])))))[0], 1] fcoh = hsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1])))) & (np.invert(np.isnan(np.log10(hsp[:, 1])))))[0], 1] xc = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1])))) & (np.invert(np.isnan(np.log10(csp[:, 1])))))[0], 0] yc = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1])))) & (np.invert(np.isnan(np.log10(csp[:, 1])))))[0], 1] xh = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1])))) & (np.invert(np.isnan(np.log10(hsp[:, 1])))))[0], 0] yh = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1])))) & (np.invert(np.isnan(np.log10(hsp[:, 1])))))[0], 1] # calculate deviations from t spec for different inductences and select the one with the smallest residual con = np.arange(0, 5, 0.01) devc = np.empty_like(con) devh = np.empty_like(con) for i in xrange(np.shape(con)[0]): devc[i] = np.nansum(np.log10(yc) - modc(xc, con[i])) devh[i] = np.nansum(np.log10(yh) - modh(xh, con[i])) conc = con[np.argmin(np.abs(devc))] conh = con[np.argmin(np.abs(devh))] if plot: ############################################################################ # plot c spec fig1 = plt.figure(5) sub = fig1.add_subplot(111) sub.plot(np.log10(tsp[:, 0]), np.log10(tsp[:, 1]), 'ro-', label='T Spec') sub.plot(np.log10(csp[:, 0]), np.log10(csp[:, 1]), 'bo-', label='C Spec') sub.plot(np.log10(xc), modc(xc, conc), 'go-', label='C Spec with %.2f Induc.' % (conc)) plt.xlabel('F[Hz]') plt.ylabel('F[Hz]*Co(w,X)') sub.set_title('c Spectrum') sub.axis('auto') sub.grid('on') sub.legend(loc='best') ############################################################################ # plot h spec fig2 = plt.figure(6) sub = fig2.add_subplot(111) sub.plot(np.log10(tsp[:, 0]), np.log10(tsp[:, 1]), 'ro-', label='T Spec') sub.plot(np.log10(hsp[:, 0]), np.log10(hsp[:, 1]), 'bo-', label='H Spec') sub.plot(np.log10(xh), modh(xh, conh), 'go-', label='H Spec %.2f Induc.' % (conh)) plt.xlabel('F[Hz]') plt.ylabel('F[Hz]*Co(w,X)') sub.set_title('h Spectrum') sub.axis('auto') sub.grid('on') sub.legend(loc='best') plt.show() ############################################################################ # save figures pp1 = pdf.PdfPages('%s/c_spec%s.pdf' % (indir, rawfile[6:-4])) pp2 = pdf.PdfPages('%s/h_spec%s.pdf' % (indir, rawfile[6:-4])) fig1.savefig(pp1, format='pdf') fig2.savefig(pp2, format='pdf') pp1.close() pp2.close() ################################################################################ # writing log file log = open( '%s/spec_%i_%02i_%02i_%02i_%02i_%02i.log' % ( indir, t.localtime()[0], t.localtime()[1], t.localtime()[2], t.localtime()[3], t.localtime()[4], t.localtime()[5], ), 'w') log.write('Inductence determination for the mean of:\n') for item in lagdate[interval[0]:interval[1] + 1]: log.write('%s\n' % (item[0])) log.write('\nC lag: %s\n' % (inpclag)) log.write('H lag: %s\n' % (inphlag)) log.write('\nC Inductence: %.2f\n' % (conc)) log.write('H Inductence: %.2f\n' % (conh))
def energyclosure(fluxfile, metfile, outdir, Rn, G, swdr, Ts=None, theta=None, depths=None, por=None, method='year', force0=True, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): ''' Calculation of energy balance closure and correction of Eddy Covariance data with different methods. Possible calculation of soil heat flux from soil temperature and moisture if not given. Definition ---------- energyclosure(fluxfile, metfile, outdir, Rn, G, swdr, Ts=None, theta=None, depths=None, por=None, method='year', force0=True, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag or fluxfill or fluxpart output file containing fluxes and flags metfile str, path and file name of the meteorology file (must be in sync with fluxfile) outdir str, path of the output folder Rn int, column number of net radiation [W m-2] in metfile, column number starts with 0 which is first data column. G int, column number of soil heat flux [W m-2] in metfile, column number starts with 0 which is first data column. If soil heat flux not available, set to False swdr int, column number of short wave downward radiation [W m-2] in metfile, column number starts with 0 which is first data column. swdr is used for swdr>0=isday Optional Input -------------- Ts list(M) of int, if G is not given, column numbers of soil temperatures in certain depths in metfile, column number starts with 0 which is first data column. theta list(M) of int, if G is not given, column numbers of soil moistures in certain depths in metfile, column number starts with 0 which is first data column. depths list(M) or array(M) of int, if G is not given, depths of Ts and theta measurements [m], positively increasing with depths e.g. [0.05, 0.30, 0.60] por float, if G is not given, porosity of the soil [-], must be bigger or equal than np.amax(theta) method str, method of how energy balance closure is calculated and applied. if 'year', fit of whole year daytime flag=0 data Rn-G vs. H+Le and application of the fraction to all H, Le and E values implement new methods here implement new methods here implement new methods here (default: 'year') force0 bool, if method='year', True forces fit through origin, if False fit is allowed to have intercept (default: True) delimiter list of str, delimiters of fluxfile and metfile (default: [',',',']) skiprows list of int, lines to skip at the beginning of fluxfile and metfile, e.g. header lines (default: [1,1]) format list of str, time formats of fluxfile and metfile, 'ascii' and 'eng' possible (default: ['ascii','ascii']) undef int/float, missing value of fluxfile and metfile (default: -9999, np.nan is not possible) plot bool, if True performs plotting (default: False) Output ------ fluxclosed.csv file containing fluxes and flags where depending on the method chosen fluxes are corrected for energy balance closure fluxclosed.pdf if method='year', plot of whole year daytime flag=0 data Rn-G vs. H+Le fluxclosed_stor.pdf if method='year', plot of whole year daytime flag=0 data Rn-G vs. H+Le including storages Restrictions ------------ Works only with half hourly time steps License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Sep 2014 ''' ########################################################################### # reading input files d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0]) m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1]) assert (d.shape[1]==22) | (d.shape[1]==32), 'energyclosure: fluxfile must be from fluxpart and have 22 or 32 cols' if format[0]=='ascii': datev = date2dec(ascii=d[skiprows[0]:,0]) elif format[0]=='eng': datev = date2dec(eng=d[skiprows[0]:,0]) else: raise ValueError('energyclosure: unknown format') if format[1]=='ascii': datem = date2dec(ascii=m[skiprows[1]:,0]) elif format[1]=='eng': datem = date2dec(eng=m[skiprows[1]:,0]) else: raise ValueError('energyclosure: unknown format') val = np.where(d[skiprows[0]:,1:]=='', str(undef), d[skiprows[0]:,1:]).astype(np.float) met = np.where(m[skiprows[1]:,1:]=='', str(undef), m[skiprows[1]:,1:]).astype(np.float) ########################################################################### # assign variables if (d.shape[1]==22): H = val[:,0] Hflag = val[:,1] Le = val[:,3] Leflag = val[:,4] E = val[:,6] Eflag = val[:,7] else: H = val[:,0] Hflag = val[:,2] Le = val[:,4] Leflag = val[:,6] E = val[:,8] Eflag = val[:,10] H_stor = val[:,1] Hflag_stor = val[:,2] Le_stor = val[:,5] Leflag_stor = val[:,6] E_stor = val[:,9] Eflag_stor = val[:,10] Rn = met[:,Rn] Ts = met[:,Ts] swdr = met[:,swdr] isday = swdr>0. ########################################################################### # check if soil heat flux is given or needs to be calculated if not G: if Ts==None or theta==None or depths==None or por==None: raise ValueError('energyclosure: if G is not given, Ts, theta, depths and rhos are needed to calculate G') else: G=soilheatflux(Ts, theta, depths, rhos) else: G = met[:,G] ########################################################################### # apply energy balance closure methods # yearly approach if (method.lower() == 'year'): closure = energyclosure_year(H, Hflag, Le, Leflag, Rn, G, isday, outdir, force0=force0, undef=undef, plot='energyclosure.pdf') if force0: H_closed, Le_closed, E_closed = H/closure, Le/closure, E/closure else: H_closed, Le_closed, E_closed = H/closure[0], Le/closure[0], E/closure[0] if (d.shape[1]==32): closure_stor = energyclosure_year(H_stor, Hflag_stor, Le_stor, Leflag_stor,# Rn, G, isday, outdir, force0=force0, undef=undef, plot='energyclosure_stor.pdf') if force0: H_stor_closed, Le_stor_closed, E_stor_closed = H_stor/closure, Le_stor/closure, E_stor/closure else: H_stor_closed, Le_stor_closed, E_stor_closed = H_stor/closure[0], Le_stor/closure[0], E_stor/closure[0] # Include new methods here # Include new methods here # Include new methods here else: raise ValueError('energyclosure: method not implemented yet.') H_closed[H==undef] = undef Le_closed[Le==undef] = undef E_closed[E==undef] = undef if (d.shape[1]==32): H_stor_closed[H_stor==undef] = undef Le_stor_closed[Le_stor==undef] = undef E_stor_closed[E_stor==undef] = undef ########################################################################### # prepare and write output if (d.shape[1]==22): flux = np.vstack((H_closed,Le_closed,E_closed)).transpose() else: flux = np.vstack((H_closed,H_stor_closed,Le_closed,Le_stor_closed,E_closed,E_stor_closed)).transpose() flux_str = np.array([['%11.5f'%x for x in y] for y in flux]) flux_str = np.where(flux_str=='%11.5f'%undef, ' '*(11-len(str(undef)))+str(undef), flux_str) if (d.shape[1]==22): d[skiprows[0]:,[1,4,7]] = flux_str else: d[skiprows[0]:,[1,2,5,6,9,10]] = flux_str np.savetxt('%s/fluxclosed.csv'%outdir, d, '%s', delimiter=',')
def t2sap(date, data, swd=None, undef=-9999.): """ Conversion of temperature difference measured with sap flow sensors (Granier type) in (mV) to sap flux density (cm^3 cm^-2 h^-1) (Granier, 1987). In addition, the correction according to Clearwater (1999) is possible. Definition ---------- def t2sap(date, data, swd=None, undef=-9999.): Input ----- date 1D array (n,) of ascii times in format DD.MM.YYYY hh:mm:ss data ND array (n,...) of the raw data in mV. First dimension is time Optional Input -------------- undef values are excluded from calculations (default: -9999) swd if None: sapflux calculation according to original Granier calibration function. if not None: sapwood depth in cm for Clearwater correction T = (T - b*Tmax)/a with a = active sapwood = swd/2 b = inactive sapwood = 1-a Output ------ SFD 2D array (n,m) with sap flux density in [cm^3 cm^-2 h^-1] according to the original Granier calibration function: SFD = 0.0119*K**1.231*3600 [Granier, 1985] References ---------- Granier, A. Evaluation of transpiration in a Douglas-fir stand by means of sap flow measurements, Tree Physiology 3, 309-320, 1987 Clearwater, M. J., Meinzer, F. C., Andrade, J. L., Goldstein, G., Holbrook, N. M. Potential errors in measurement of nonuniform sap flow using heat dissipation probes, Tree Physiology 19, 681-687, 1999 Examples -------- # normal sapflux conversion >>> data = np.array([0.434, 0.433, 0.432, 0.431, 0.431, 0.432]) >>> date = ['18.05.2013 08:00', '18.05.2013 08:10', '18.05.2013 08:20', ... '18.05.2013 08:30', '18.05.2013 08:40', '18.05.2013 08:50'] >>> SFD = t2sap(date, data) >>> print(np.round(SFD,3)) [0. 0.024 0.057 0.095 0.095 0.057] >>> # sapflux conversion including clearwater correction >>> data = np.array([0.434, 0.433, 0.432, 0.431, 0.431, 0.432]) >>> date = ['18.05.2013 08:00', '18.05.2013 08:10', '18.05.2013 08:20', ... '18.05.2013 08:30', '18.05.2013 08:40', '18.05.2013 08:50'] >>> SFD = t2sap(date, data, swd=1.5) >>> print(np.round(SFD,3)) [0. 0.035 0.082 0.135 0.135 0.082] License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014-2016 Andreas Wiedemann, Matthias Cuntz - mc (at) macu (dot) de Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AW, Jun 2014 Modified, MC, Jun 2014 - sap_app -> t2sap, incl. undef, first date then data - squeeze 1D array, fill undef if not masked Modified, MC, Nov 2016 - ported to Python 3 """ isvec = False if data.ndim == 1: isvec = True data = data.reshape((-1, 1)) # mask data isnotmask = True if type(data) == np.ma.core.MaskedArray: isnotmask = False data_masked = np.ma.array(data, mask=(data == undef)) # julian day jd = np.array(date2dec(ascii=date)) # integer julian day jd_int = jd.astype(np.int) # unique days jd_uni = np.unique(jd_int) # Tmax per day tmax_day = np.ma.ones((jd_uni.size, data.shape[1])) * undef # Time of Tmax per day jdmax_day = np.ma.zeros((jd_uni.size, data.shape[1]), dtype=np.int) # Determine Tmax per day for count, i in enumerate(jd_uni): # where is given day ii = np.where(jd_int == i)[0] # index of Tmax of day jj = np.ma.argmax(data_masked[ii, :], axis=0) # time at Tmax of day jdmax_day[count, :] = jd[ii[jj]] # Tmax per day tmax_day[count, :] = data_masked[ii[jj], :] # Tmax for every record tmax = np.empty(data.shape) for i in range(data.shape[1]): # time points at Tmax per day xx = jdmax_day[:, i] # Tmax per day yy = tmax_day[:, i] ii = ~yy.mask # Tmax per time point tmax[:, i] = np.interp(jd, xx[ii], yy[ii]) if swd == None: Tsw = data_masked else: # sapwood depth in cm / 2cm for needle lenght = portion of sensor in sapwood a = 0.5 * swd # portion of sensor in inactive xylem b = 1.0 - a # define weighted mean of T in the sapwood (a) and T in the inactive xylem (b) Tsw = (data_masked - (b * tmax)) / a # converts raw data [mV] into Sap Flux density [cm3*cm-2*h-1] SFD = (0.0119 * ((tmax - Tsw) / Tsw)**1.231) * 3600. # if it was not masked then the original undef will be undef if isnotmask: SFD = SFD.filled(undef) # 1D in = 1D out if isvec: SFD = np.squeeze(SFD) return SFD
def fluxfill(fluxfile, metfile, outdir, rg, tair, rh, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): ''' Wrapper function for gapfill with file management and plotting. Definition ---------- fluxfill(fluxfile, metfile, outdir, rg, tair, rh, delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag output file containing fluxes and flags metfile str, path and file name of the meteorology file (must be in sync with fluxfile) outdir str, path of the output folder rg int, column number of global radiation [W m-2] in metfile, column number starts with 0 which is first data column. tair int, column number of air temperature [deg C] in metfile, column number starts with 0 which is first data column. rh int, column number of relative humidity [%] in metfile, column number starts with 0 which is first data column. Optional Input -------------- delimiter list of str, delimiters of fluxfile and metfile (default: [',',',']) skiprows list of int, lines to skip at the beginning of fluxfile and metfile, e.g. header lines (default: [1,1]) format list of str, time formats of fluxfile and metfile, 'ascii' and 'eng' possible (default: ['ascii','ascii']) undef int/float, missing value of fluxfile and metfile (default: -9999, np.nan not possible) plot bool, if True performs plotting (default: False) Output ------ fluxfilled.csv file containing fluxes with original flags and quality flags of the gap filling. Fluxes are filled where flag>1. fluxfilled.pdf plot of each gap filled flux. License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' ########################################################################### # reading input files d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0], skiprows=skiprows[0]) m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1], skiprows=skiprows[1]) assert (d.shape[1]==11) | (d.shape[1]==19), 'fluxfill: fluxfile must be from fluxflag or profile2storage and have 11 or 19 cols' if format[0]=='ascii': datev = date2dec(ascii=d[:,0]) elif format[0]=='eng': datev = date2dec(eng=d[:,0]) else: raise ValueError('fluxfill: unknown format') if format[1]=='ascii': datem = date2dec(ascii=m[:,0]) elif format[1]=='eng': datem = date2dec(eng=m[:,0]) else: raise ValueError('fluxfill: unknown format') val = np.where(d[:,1:]=='', str(undef), d[:,1:]).astype(np.float) met = np.where(m[:,1:]=='', str(undef), m[:,1:]).astype(np.float) ########################################################################### # assign variables if (d.shape[1]==11): data = val[:,0::2] # corr. H, corr. LE, corr. E, corr. C, tau data_flag = (val[:,1::2]>1) | (val[:,0::2]==undef) else: data = val[:,[0,1, 3,4, 6,7, 9,10, 12]] # corr. H, corr. H+s, corr. LE, corr. LE+s, corr. E, corr. E+s, corr. C, corr. C+s, tau data_flag = np.repeat((val[:,[2,5,8,11,13]]>1) | (val[:,[0,1, 3,4, 6,7, 9,10, 12]]==undef), 2, axis=1)[:,:-1] rg = met[:,rg] rg_flag = rg==undef tair = met[:,tair] tair_flag = tair==undef rh = met[:,rh] rh_flag = rh==undef vpd = np.empty_like(tair) vpd[tair_flag | rh_flag] = undef vpd[~(tair_flag | rh_flag)] = (1.-rh[~(tair_flag | rh_flag)]/100.)*esat(tair[~(tair_flag | rh_flag)]+273.14) vpd_flag = vpd==undef flux = np.zeros_like(data) flag = np.zeros_like(data_flag, dtype=np.int) ########################################################################### if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf pp1 = pdf.PdfPages(outdir+'/fluxfilled.pdf') ########################################################################### # do gapfill for i in range(data.shape[1]): flux[:,i], flag[:,i] = gapfill.gapfill(datev, data[:,i], rg, tair, vpd, data_flag[:,i], rg_flag, tair_flag, vpd_flag, rg_dev=50., tair_dev=2.5, vpd_dev=5., longgap=60, fullday=False, undef=undef, ddof=1, err=False, shape=False) ####################################################################### # plot if plot: majticks = mpl.dates.MonthLocator(bymonthday=1) format_str='%d %m %Y %H:%M' date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) fig1 = plt.figure(1) sub1 = fig1.add_subplot(111) l1 =sub1.plot(datev-date01, flux[:,i], '-g') l2 =sub1.plot(datev-date01, np.ma.array(data[:,i],mask=data_flag[:,i]), '-b') sub1.set_xlim(datev[0]-date01,datev[-1]-date01) sub1.xaxis.set_major_locator(majticks) sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) fig1.autofmt_xdate() plt.show() fig1.savefig(pp1, format='pdf') if plot: pp1.close() ########################################################################### # prepare output and save file flux_str = np.array([['%11.5f'%x for x in y] for y in flux]) if (d.shape[1]==11): header = np.array([' H', ' Hflag', ' Hgf', ' LE', ' LEflag', ' LEgf', ' E', ' Eflag', ' Egf', ' C', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf']) output = np.hstack((d[:,0:1], flux_str.repeat(3, axis=1))) output[:,2::3] = astr(val[:,1::2].astype(int), prec=8) output[:,3::3] = astr(flag, prec=8) else: header = np.array([' H', ' H+sT', ' Hflag', ' Hgf', ' LE', ' LE+sLE', ' LEflag', ' LEgf', ' E', ' E+sE', ' Eflag', ' Egf', ' C', ' C+sC', ' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf', ' sT', ' sLE', ' sE', ' sC']) output = np.hstack((d[:,0:1], np.insert(flux_str, [2,2,4,4,6,6,8,8], flux_str[:,:-1], axis=1), flux_str[:,-1:].repeat(2, axis=1), d[:,-4:])) output[:,[3,7,11,15,18]] = astr(val[:,[2,5,8,11,13]].astype(int), prec=8) output[:,[4,8,12,16,19]] = astr(flag[:,[0,2,4,6,8]], prec=8) np.savetxt('%s/fluxfilled.csv'%outdir, np.vstack((np.concatenate(([' date'], header))[np.newaxis,:], output)), '%s', delimiter=',')
def profile2storage(fluxfile, fluxfile2, profilefile, outdir, heights, CO2=None, H2O=None, T=None, rH=None, delimiter=[',', ',', ','], skiprows=[1, 1, 1], format=['ascii', 'ascii', 'ascii'], undef=-9999, plot=False): ''' Calculates storage fluxes for changes in CO2, H2O, air temperature and air moisture from profile data or meteorological data to correct Eddy Covariance fluxes. FLux files from EddySoft and from fluxflag are needed as well as a file with the profile or meteo data. Fluxes will be updated with the respective storage fluxes and saved in a new file. Multiple application of this routine with different profile or meteo files are possible to correct e.g. the CO2, H2O and latent heat fluxes with profile data of CO2 and H2O concentrations and afterwards the H flux with temperature data from another file. Definition ---------- profile2storage(fluxfile, fluxfile2, profilefile, outdir, heights, CO2=None, H2O=None, T=None, rH=None, delimiter=[',',',',','], skiprows=[1,1,1], format=['ascii','ascii','ascii'], undef=-9999, plot=False): Input ----- fluxfile str, path and file name of fluxflag output file containing fluxes and flags. These fluxes will be updated by the storage fluxes and saved as a new file fluxfile2 str, path and file name of EddyFlux output file (timestep checked) containing original fluxes profilefile str, path and file name of the profile file or meteorology file containing CO2, H2O, T or rH values to compute the profile storage from outdir str, path of the output folder heights list of floats, observation heights of the profile [m], increasing e.g. [0.5,1.0,10.0,20.0]. CO2 list of int, column numbers of CO2 concentrations for the different heights (in the same order) [mumol/mol] in profilefile, column number starts with 0 which is first data column. H2O list of int, column numbers of H2O concentrations for the different heights (in the same order) [mmol/mol] in profilefile, column number starts with 0 which is first data column. T list of int, column numbers of air temperatures for the different heights (in the same order) [degC] in profilefile, column number starts with 0 which is first data column. rH list of int, column numbers of relative humidity for the different heights (in the same order) [%] in profilefile, column number starts with 0 which is first data column. The calculation of air vapour energy storage change within the profile works only when T is given as well. Optional Input -------------- delimiter list of str, delimiters of fluxfile, fluxfile and profilefile (default: [',',',',',']) skiprows list of int, lines to skip at the beginning of fluxfile, fluxfile and profilefile, e.g. header lines (default: [1,1,1]) format list of str, time formats of fluxfile, fluxfile and profilefile, 'ascii' and 'eng' possible (default: ['ascii','ascii','ascii']) undef int/float, missing value of fluxfile, fluxfile and profilefile (default: -9999, np.nan is not possible) plot bool, if True performs plotting (default: False) Output ------ flux+stor.csv file containing fluxes and flags where storage fluxes are added in an additional column and storage fluxes are appended to the end of the file Restrictions ------------ Works only with half hourly time steps, all files in sync License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Sep 2014 ''' ########################################################################### # time interval int = 30. dt = int * 60. if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf ########################################################################### # reading input files # fluxes to correct for storage changes d1 = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0]) # original flux file from EddyFlux containing air density rho_a d2 = np.loadtxt(fluxfile2, dtype='|S100', delimiter=delimiter[1]) # file containing profile data (can be meteo file if no profile available) d3 = np.loadtxt(profilefile, dtype='|S100', delimiter=delimiter[2]) assert (d1.shape[1] == 11) | ( d1.shape[1] == 19 ), 'profile2storage: fluxfile must be from fluxflag or profiletostorage and have 11 or 19 cols' assert d2.shape[ 1] == 68, 'profile2storage: fluxfile2 must be from EddyFlux and have 68 cols' assert d1.shape[0] == d2.shape[ 0], 'profile2storage: fluxfile and fluxfile2 must be in sync' assert d1.shape[0] == d3.shape[ 0], 'profile2storage: fluxfile and profilefile must be in sync' assert ( ((H2O == None) & (rH == None)) ^ ((H2O != None) ^ (rH != None)) ), 'profile2storage: give either H2O or rH, both would be double correction' if format[0] == 'ascii': datev = date2dec(ascii=d1[skiprows[0]:, 0]) elif format[0] == 'eng': datev = date2dec(eng=d1[skiprows[0]:, 0]) else: raise ValueError('profile2storage: unknown format') if format[2] == 'ascii': datem = date2dec(ascii=d2[skiprows[2]:, 0]) elif format[2] == 'eng': datem = date2dec(eng=d2[skiprows[2]:, 0]) else: raise ValueError('profile2storage: unknown format') flux1 = np.where(d1[skiprows[0]:, 1:] == '', str(undef), d1[skiprows[0]:, 1:]).astype(np.float) flux2 = np.where(d2[skiprows[1]:, 1:] == '', str(undef), d2[skiprows[1]:, 1:]).astype(np.float) prof = np.where(d3[skiprows[2]:, 1:] == '', str(undef), d3[skiprows[2]:, 1:]).astype(np.float) flux1 = np.ma.array(flux1, mask=flux1 == undef, hard_mask=True) flux2 = np.ma.array(flux2, mask=flux2 == undef) prof = np.ma.array(prof, mask=prof == undef) ########################################################################### # assign variables if d1.shape[1] == 11: H, Hflag = flux1[:, 0], flux1[:, 1] Le, Leflag = flux1[:, 2], flux1[:, 3] E, Eflag = flux1[:, 4], flux1[:, 5] C, Cflag = flux1[:, 6], flux1[:, 7] else: H, Hflag = flux1[:, 0], flux1[:, 2] Le, Leflag = flux1[:, 3], flux1[:, 5] E, Eflag = flux1[:, 6], flux1[:, 8] C, Cflag = flux1[:, 9], flux1[:, 11] p = flux2[:, 58] # [hPa] rho = flux2[:, 62] # [kg/m3] ########################################################################### # prepare output array d4 = np.copy(d1) if d1.shape[1] == 11: temp = np.empty((d1.shape[0], 4), dtype='|S100') temp[:] = ' ' * (11 - len(str(undef))) + str(undef) temp[0, :] = [ ' H+sT', ' LE+sLE', ' E+sE', ' C+sC' ] d4 = np.insert(d4, [2, 4, 6, 8], temp, axis=1) temp[0, :] = [ ' sT', ' sLE', ' sE', ' sC' ] d4 = np.append(d4, temp, axis=1) ########################################################################### # calls if CO2: CO2 = prof[:, CO2] assert CO2.shape[1] == len( heights), 'profile2storage: number of CO2 cols must equal heights' # calculate storage flux and storage flux flag sfCO2 = stor2flux(CO2, rho, heights, dt, 'CO2') sfCO2flag = sfCO2.mask.astype(np.int) # add to eddy flux newC = C + np.ma.filled(sfCO2, 0) # format and write into output array newC_str = np.array(['%11.5f' % x for x in np.ma.filled(newC, undef)]) newC_str = np.where(newC_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newC_str) sfCO2_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfCO2, undef)]) sfCO2_str = np.where(sfCO2_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfCO2_str) d4[skiprows[0]:, 11] = newC_str d4[skiprows[0]:, 18] = sfCO2_str if plot: storplot(CO2, datev, heights, C, sfCO2, newC, 'storageCO2.pdf', pdf, plt, mpl, outdir) if H2O: H2O = prof[:, H2O] assert H2O.shape[1] == len( heights), 'profile2storage: number of H2O cols must equal heights' # calculate storage flux and storage flux flag sfH2O = stor2flux(H2O, rho, heights, dt, 'H2O') sfH2O_Wm2 = sfH2O * mmol_h2o * latentheat_vaporization / 1.e6 sfH2Oflag = sfH2O.mask.astype(np.int) # add to eddy flux newE = E + np.ma.filled(sfH2O, 0) newLe = Le + np.ma.filled(sfH2O_Wm2, 0) # format and write into output array newE_str = np.array(['%11.5f' % x for x in np.ma.filled(newE, undef)]) newLe_str = np.array( ['%11.5f' % x for x in np.ma.filled(newLe, undef)]) sfH2O_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfH2O, undef)]) sfH2O_Wm2_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfH2O_Wm2, undef)]) newE_str = np.where(newE_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newE_str) newLe_str = np.where(newLe_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newLe_str) sfH2O_str = np.where(sfH2O_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfH2O_str) sfH2O_Wm2_str = np.where(sfH2O_Wm2_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfH2O_Wm2_str) d4[skiprows[0]:, 8] = newE_str d4[skiprows[0]:, 17] = sfH2O_str d4[skiprows[0]:, 5] = newLe_str d4[skiprows[0]:, 16] = sfH2O_Wm2_str if plot: storplot(H2O, datev, heights, E, sfH2O, newE, 'storageH2O.pdf', pdf, plt, mpl, outdir) if T: T = prof[:, T] assert T.shape[1] == len( heights), 'profile2storage: number of T cols must equal heights' # calculate storage flux and storage flux flag sfT = stor2flux(T, rho, heights, dt, 'T') sfTflag = sfT.mask.astype(np.int) # add to eddy flux newH = H + np.ma.filled(sfT, 0) # format and write into output array newH_str = np.array(['%11.5f' % x for x in np.ma.filled(newH, undef)]) newH_str = np.where(newH_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newH_str) sfT_str = np.array(['%11.5f' % x for x in np.ma.filled(sfT, undef)]) sfT_str = np.where(sfT_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfT_str) d4[skiprows[0]:, 2] = newH_str d4[skiprows[0]:, 15] = sfT_str if plot: storplot(T, datev, heights, H, sfT, newH, 'storageT.pdf', pdf, plt, mpl, outdir) if rH: rH = prof[:, rH] assert rH.shape[1] == len( heights), 'profile2storage: number of rH cols must equal heights' # calculate specific humidity vapourpressure = esat(T + T0) * (rH / 100.) / 100. #[hPa] specifichumidity = (mmol_h2o / mmol_air * vapourpressure) / ( p - (1. - mmol_h2o / mmol_air) * vapourpressure) # calculate storage flux and storage flux flag sfrH_Wm2 = stor2flux(specifichumidity, rho, heights, dt, 'rH') sfrH = sfrH_Wm2 * 1.e6 / (mmol_h2o * latentheat_vaporization) sfrHflag = sfrH.mask.astype(np.int) # add to eddy flux newE = E + np.ma.filled(sfrH, 0) newLe = Le + np.ma.filled(sfrH_Wm2, 0) # format and write into output array newE_str = np.array(['%11.5f' % x for x in np.ma.filled(newE, undef)]) newLe_str = np.array( ['%11.5f' % x for x in np.ma.filled(newLe, undef)]) sfrH_str = np.array(['%11.5f' % x for x in np.ma.filled(sfrH, undef)]) sfrH_Wm2_str = np.array( ['%11.5f' % x for x in np.ma.filled(sfrH_Wm2, undef)]) newE_str = np.where(newE_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newE_str) newLe_str = np.where(newLe_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), newLe_str) sfrH_str = np.where(sfrH_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfrH_str) sfrH_Wm2_str = np.where(sfrH_Wm2_str == '%11.5f' % undef, ' ' * (11 - len(str(undef))) + str(undef), sfrH_Wm2_str) d4[skiprows[0]:, 8] = newE_str d4[skiprows[0]:, 17] = sfrH_str d4[skiprows[0]:, 5] = newLe_str d4[skiprows[0]:, 16] = sfrH_Wm2_str if plot: storplot(rH, datev, heights, E, sfH2O, newE, 'storagerH.pdf', pdf, plt, mpl, outdir) ########################################################################### # write output np.savetxt('%s/flux+stor.csv' % outdir, d4, '%s', delimiter=',')
def timestepcheck(indir, pat, outfile, begin, end, numhead=1, timeint=30, format='ascii', empty='-9999', delimiter=',', skiprows=0): ''' Checks ascii data files with first column being a time stamp in ascii of eng style for correctness. If for a given time interval time steps are missing, they will will be filled with the correct time stamp and empty values. A beginning and end for the output files can be set to which data should be kept or filled. Pat defines the file name pattern to consider in the check. Multiple files, which match the pat are concatenated. Definition ---------- timestepcheck(indir, pat, outfile, begin, end, numhead=1, timeint=30, format='ascii', empty='-9999', delimiter=',', skiprows=0): Input ----- indir str, path of the folder where the input files are pat str, name or regular expression of the input files outfile str, path and name of the output file begin str, start time of the output file, must be in the same format as time stamps in the input files end str, end time of the output file, must be in the same format as time stamps in the input files Optional Input -------------- numhead int, number of header lines in the input files (default: 1) timeint int, time interval of the input file in minutes (default: 30) format str, format of time stamps in input files. 'ascii' or 'eng' is possible (default: 'ascii') empty str, value for missing values (default: '-9999') delimiter str, delimiter of the input files (default: ',') skiprows int, rows to skip in input files, e.g. logger fuzzle before actual data header starts (default: 0) Output ------ outfile file with missing time steps filled with empty values cut from begin to end Restrictions ------------ TODO: tested thoroughly only for timeint=30 TODO: more bad value checks can be included, see sternchen and tuedelchen License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' ########################################################################### # time interval list interval = range(0,60,timeint) #jdint = date2dec(yr=-4712,mo=1,dy=1,hr=12,mi=timeint) #jdmin = date2dec(yr=-4712,mo=1,dy=1,hr=12,mi=0,sc=30)# (=precision) jdint = date2dec(yr=1,mo=1,dy=1,hr=12,mi=timeint) % 1 jdmin = date2dec(yr=1,mo=1,dy=1,hr=12,mi=0,sc=30) % 1# (=precision) if format == 'ascii': jdbegin = date2dec(ascii = np.array([begin])) jdend = date2dec(ascii = np.array([end])) elif format == 'eng': jdbegin = date2dec(eng = np.array([begin])) jdend = date2dec(eng = np.array([end])) ########################################################################### # reading input directory pat = re.compile(pat) new = True filelist = os.listdir(indir) for file in filelist: if re.search(pat, file): if new: data = np.loadtxt('./%s/%s'%(indir, file), dtype='|S21',\ delimiter=delimiter, skiprows=skiprows) if data.shape[0] == 0: print('Warning: File %s is empty!' %(file)) else: if np.shape(data.shape)[0] == 1: data = data.reshape((1,-1)) new = False else: add_data = np.loadtxt('./%s/%s'%(indir, file), dtype='|S21', delimiter=delimiter, skiprows=numhead+skiprows) if add_data.shape[0] == 0: print('Warning: File %s is empty!' %(file)) elif np.shape(add_data.shape)[0] == 1: add_data = add_data.reshape((1,-1)) data = np.append(data, add_data, 0) else: data = np.append(data, add_data, 0) ########################################################################### # sternchen check :-D # replace with regular expression check data[data=='***********'] = empty #!!! uberprufen auf lange und re data[data=='********'] = empty #!!! uberprufen auf lange und re data[data=='*********'] = empty ########################################################################### # tuedelchen check :-D # replace with regular expression check if data[numhead,0][0] == '"': data[numhead:,0] = np.array([x[1:-1] for x in data[numhead:,0]]) ########################################################################### # "NAN" check :-D # replace with regular expression check data[data=='"NAN"'] = empty #!!! uberprufen auf lange und re ########################################################################### # leerzeilen check blankline = np.where(data[0:2,0]=='')[0] data = np.delete(data, blankline, 0) ########################################################################### # missing values check data[data==''] = empty data[data=='""'] = empty columns = np.shape(data)[1]-1 ########################################################################### # calculate julian date if format == 'ascii': import time jd = date2dec(ascii = data[numhead:,0]) elif format == 'eng': jd = date2dec(eng = data[numhead:,0]) ########################################################################### # wrong time stamp check diff = jd[1:] - jd[:-1] minute = np.array([x.split()[1][3:5] for x in data[numhead:,0]]).astype(int) nii = np.nonzero(~np.in1d(minute, interval))[0] ts = np.nonzero(np.less(diff, jdint-jdmin))[0] wrong = np.unique(np.append(nii, ts)) if data.shape[0]-numhead-2 in wrong: wrong = np.append(wrong, [data.shape[0]-numhead-1], 0) delete = [] for i in wrong: print('\nHERE IS SOMETHING WRONG:\n') print('BOF' if numhead+i-2<0 else data[numhead+i-2,:4]) print('BOF' if numhead+i-1<0 else data[numhead+i-1,:4]) print('-----------------------------------------------') print(data[numhead+i,:4]) print('-----------------------------------------------') print('EOF' if numhead+i+1>=np.shape(data)[0] else data[numhead+i+1,:4]) print('EOF' if numhead+i+2>=np.shape(data)[0] else data[numhead+i+2,:4]) do = raw_input("\n(d)elete entry, (s)et to empty, (t)ype in date, (i)gnore: ") if do == 'd': delete += [numhead+i] elif do == 's': data[numhead+i,1:] = empty elif do == 't': newdate = str(raw_input("\nreplace with: ")) data[numhead+i,0] = newdate # newmin = str(raw_input("\n%s"%(data[numhead+i,0][:-2]))) # data[numhead+i,0] = data[numhead+i,0][:-2] + newmin elif do == 'i': pass data = np.delete(data, delete, 0) ########################################################################### # calculate julian date again if format == 'ascii': jd = date2dec(ascii = data[numhead:,0]) elif format == 'eng': jd = date2dec(eng = data[numhead:,0]) ########################################################################### # check time step diff = jd[1:] - jd[:-1] ingap = np.where(np.greater(diff, jdint+jdmin))[0] nugap = np.rint((diff[ingap]/jdint)-1) ########################################################################### # insert missing time steps for i in range(np.size(ingap))[::-1]: where = np.ones(int(nugap[i]), dtype=int)*(ingap[i]+1+numhead) if format == 'ascii': span = np.arange(1,nugap[i]+1)*jdint + jd[ingap[i]] what = dec2date(span.astype('|S16').astype(float), ascii=True) elif format == 'eng': span = np.arange(1,nugap[i]+1)*jdint + jd[ingap[i]] what = dec2date(span.astype('|S16').astype(float), eng=True) what = np.array([x[:-3] for x in what]) miss = np.empty((int(nugap[i]),columns), dtype='|S11') miss[:] = empty what = np.append(np.reshape(what, (-1,1)), miss, 1) data = np.insert(data, where, what, 0) ########################################################################### # fill/cut up/off beginning and end start = np.where(data[:,0]==begin)[0] if start == numhead: pass elif start > numhead: data = np.delete(data, np.arange(numhead, start), 0) else: if format == 'ascii': tofill = int((date2dec(ascii = data[numhead,0]) - jdbegin)/jdint) span = np.arange(0,tofill)*jdint + jdbegin # tofill+1 weggenommen!!! what = dec2date(span.astype('|S16').astype(float), ascii=True) elif format == 'eng': tofill = int((date2dec(eng = data[numhead,0]) - jdbegin)/jdint)-1 span = np.arange(0,tofill)*jdint + jdbegin # tofill+1 weggenommen!!! what = dec2date(span.astype('|S16').astype(float), eng=True) what = np.array([x[:-3] for x in what]) where = np.ones(tofill, dtype=int)*numhead # tofill+1 weggenommen!!! #if doidate: # miss = np.empty((tofill,columns-2), dtype='|S11') # tofill+1 weggenommen!!! #else: miss = np.empty((tofill,columns), dtype='|S11') # tofill+1 weggenommen!!! miss[:] = empty what = np.append(np.reshape(what, (-1,1)), miss, 1) data = np.insert(data, where, what, 0) stop = np.where(data[:,0]==end)[0] maxind = np.shape(data)[0]-1 if stop == maxind: pass elif stop < maxind: #data = data[:stop+1,:] data = data[:stop[0]+1,:] else: if format == 'ascii': tofill = int((jdend - date2dec(ascii = data[-1,0]))/jdint) span = np.arange(1,tofill+1)*jdint + date2dec(ascii = data[-1,0]) what = dec2date(span.astype('|S16').astype(float), ascii=True) elif format == 'eng': tofill = int((jdend - date2dec(eng = data[-1,0]))/jdint) span = np.arange(1,tofill+1)*jdint + date2dec(eng = data[-1,0]) what = dec2date(span.astype('|S16').astype(float), eng=True) what = np.array([x[:-3] for x in what]) #if doidate: # miss = np.empty((tofill,columns-2), dtype='|S11') #else: miss = np.empty((tofill,columns), dtype='|S11') miss[:] = empty what = np.append(np.reshape(what, (-1,1)), miss, 1) data = np.append(data, what, 0) ########################################################################### # save data to txt file np.savetxt('%s'%outfile, data, fmt='%s', delimiter=delimiter)
def spikeflag(date, data, inflag, isday, outdir, window=13, iter=1, fill_days=1, t_int=48, z=7, deriv=0, udef=-9999, spike_v=2, plot=False): ''' Spike detection for Eddy Covariance data (and basically all other data) using a moving median absolute difference filter. Multiple iterations possible. Originally coded by Tino Rau. Definition ---------- spikeflag(date, data, inflag, isday, window=13, iter=1, fill_days=1, t_int=48, z=5.5, deriv=0, udef=-9999, spike_v=2, plot=False): Input ----- date np.array(N), julian date (used only for plotting) data np.array(N,M), data array where spike detection is applied on each column (M) inflag np.array(N,M), dtype=int, quality flag of data, spike detection is only applied where inflag=0, all other data is ignored isday np.array(N), dtype=bool, True where it is day and False where it is night outdir path where plots are saved Optional Input -------------- window int, size of the moving window where mad is calculated in days (default: 13) iter int, how often the running window mad shall be applied (default: 1) fill_days int, number of days where mad is applied within moving window (default: 1) t_int int, number of data points within one day (default: 48) z int/float, data is allowed to deviate maximum z standard deviations from the median (default: 7) deriv int, 0: Act on raw data; 1: use first derivatives; 2: use 2nd derivatives (default: 0) udef int/float, missing value of data (default: -9999) NaN values are excluded from computations anyhow. spike_v int, spike value which shall be returned when a spike is detected (default: 2) plot bool, if True data and spikes are plotted (default: False) Output ------ flag np.array(N), flag array where everything is 0 except where spikes were detected, there it is spike_v. License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2014 Arndt Piayda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written, AP, Aug 2014 ''' rows, cols = np.shape(data) flag = np.zeros_like(inflag).astype(np.int) # mad window length and flag window length period = np.int(window*t_int)/2 fill_win = np.int(fill_days*t_int)/2 # calculate dusk and dawn times and separate in day and night isdawn = np.zeros(rows,dtype=np.bool) isdusk = np.zeros(rows,dtype=np.bool) dis = (isday.astype(int) - np.roll(isday,-1).astype(int)).astype(bool) isdawn[:-1] = np.where(dis[:-1] == -1, True, False) isdusk[:-1] = np.where(dis[:-1] == 1, True, False) isddday = isdawn tmp = np.roll(isdusk,1) isddday[1:] += tmp[1:] isddnight = isdusk tmp = np.roll(isdawn,1) isddnight[1:] += tmp[1:] # iterate over each column of data for col in xrange(cols): # iterate as much as iter for i in xrange(iter): # get day and night data# day_data = np.where((isday | isddday) & (inflag[:,col]==0) & ((data[:,col]!=udef) | (~np.isnan(data[:,col]))), data[:,col], np.nan) night_data = np.where((~isday | isddnight) & (inflag[:,col]==0) & ((data[:,col]!=udef) | (~np.isnan(data[:,col]))), data[:,col], np.nan) # iterate over flag window fill_points = xrange(fill_win, isday.size-1, 2*fill_win) for j in fill_points: j1 = np.max([ j - period - 1,0]) j2 = np.min([ j + period + 1,isday.size]) fill_start = np.max([ j - fill_win,1]) fill_end = np.min([ j + fill_win,isday.size-1]) day_flag = mad(np.ma.masked_array(data=day_data[j1:j2], mask=(np.isnan(day_data[j1:j2]))), z=z, deriv=deriv) flag[fill_start:fill_end,col] += np.where(day_flag[fill_start-j1-1:fill_end-j1-1], spike_v, 0) night_flag = mad(np.ma.masked_array(data=night_data[j1:j2], mask=(np.isnan(night_data[j1:j2]))), z=z, deriv=deriv) flag[fill_start:fill_end,col] += np.where(night_flag[fill_start-j1-1:fill_end-j1-1], spike_v, 0) if plot: import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.backends.backend_pdf as pdf majticks = mpl.dates.MonthLocator(bymonthday=1) format_str='%d %m %Y %H:%M' date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0) fig1 = plt.figure(1) sub1 = fig1.add_subplot(111) valid = (inflag[:,col]==0) & ((data[:,col]!=udef) | (~np.isnan(data[:,col]))) l1 =sub1.plot(date[valid]-date01, data[valid,col], '-b') l2 =sub1.plot(date[flag[:,col]!=0]-date01, data[flag[:,col]!=0,col], 'or') sub1.xaxis.set_major_locator(majticks) sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str)) fig1.autofmt_xdate() plt.show() pp1 = pdf.PdfPages(outdir+'/spike_%i.pdf'%col) fig1.savefig(pp1, format='pdf') pp1.close() return flag