def storplot(conc, date, heights, oriflux, storflux, newflux, name, pdf, plt,
mpl, outdir):
'''
'''
majticks = mpl.dates.MonthLocator(bymonthday=1)
format_str = '%d %m %Y %H:%M'
date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0)
conc = np.ma.copy(conc.transpose())
date = np.ma.copy(date - date01)
pp1 = pdf.PdfPages(outdir + '/' + name)
fig1 = plt.figure(name)
sub1 = fig1.add_subplot(211)
for i, item in enumerate(conc):
sub1.plot(date, item, label='%2.1f m' % (heights[i]))
plt.legend(loc='best')
sub2 = fig1.add_subplot(212)
sub2.axhline(y=0, xmin=0, xmax=1, color='k')
sub2.plot(date, oriflux, 'b-', label='original')
sub2.plot(date, storflux, 'r-', label='storage')
sub2.plot(date, newflux, 'g-', label='new')
plt.legend(loc='best')
sub1.set_xlim(date[0], date[-1])
sub1.xaxis.set_major_locator(majticks)
sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str))
sub2.set_xlim(date[0], date[-1])
sub2.xaxis.set_major_locator(majticks)
sub2.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str))
fig1.autofmt_xdate()
plt.show()
fig1.savefig(pp1, format='pdf')
pp1.close()
def means(date,
dat,
year=False,
month=False,
day=False,
hour=False,
half_hour=False,
minute=False,
meanday=False,
meanmonth=False,
seasonal=False,
sum=False,
max=False,
min=False,
onlydat=False):
"""
Calculate daily, monthly, yearly, etc. means of data depending on date
stamp.
date is Julian date. dat will be averaged over columns, i.e. first
dimension.
If no option is given, the mean will be over the whole first column.
Returns centred dates, averages.
Yearly dates are centred at June 15, 12:00h.
Monthly dates are centred at 15th, 12:00h.
Daily dates are centred at 12:00h.
Hourly dates are centred at 30 min.
Half hourly dates are centred at 15 and 45 min.
Minutely dates are centred at 30 sec.
Mean daily dates centred on 30 min of 01. January of first year.
Mean monthly dates centred on 15th of month, 12:00h of first year.
Seasonal dates are centred at 12:00h of each day of first (leap) year.
Definition
----------
def means(date, dat, year=False, month=False, day=False, hour=False,
half_hour=False, minute=False,
meanday=False, meanmonth=False, seasonal=False,
sum=False, max=False, min=False, onlydat=False):
Input
-----
date 1D array with Julian dates
dat ND (masked-)array
Optional Input
--------------
year if True, yearly means.
month if True, monthly means.
day if True, daily means.
hour if True, hourly means.
half_hour if True, half-hourly means.
minute if True, minutely means.
meanday if True, mean daily cycle.
meanmonth if True, mean monthly cycle.
seasonal if True, seasonal cycle.
sum if True, calculate sums instead of means.
max if True, calculate maxima instead of means.
min if True, calculate minima instead of means.
onlydat if True, return only meandat, else return [outdate, meandat]
Output
------
outdate centred date on year, month, day, etc.
meandat averaged data in 1st dimension.
If meanday==True, then all hours will be written;
as a masked-array if hours are missing.
If meanmonth==True, then all months will be written;
as a masked-array if months are missing.
If seasonal==True: input data has to be spaced <= days,
otherwise consider meanmonth.
If seasonal==True, then all days will be written;
as a masked-array if days are missing.
Note
----
If input date signifies the end of the time step, the user should remove
half a time step before using the routine. Otherwise the routine would have
to guess the time step, which is error prone.
For example, remove 15 minutes in decimal days from time steps of
half an hour:
date = jams.date2dec(ascii=adate) - 15./(24.*60.)
Examples
--------
>>> from autostring import astr
>>> from date2dec import date2dec
>>> dates = ['01.01.1990 12:00:00', '01.02.1990 12:00:00', '01.03.1990 12:00:00',
... '01.01.1990 12:10:00', '01.01.1990 13:00:00', '01.01.1990 14:00:00']
>>> jdates = date2dec(ascii=dates)
>>> x = np.arange(len(jdates))+1.
>>> odates, xout = means(jdates, x)
>>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True))
2.448e+06 3.500
>>> odates, xout = means(jdates, x, year=True)
>>> print(astr(xout, 3, pp=True))
['3.500']
>>> odates, xout = means(jdates, x, month=True)
>>> print(astr(xout, 3, pp=True))
['4.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, day=True)
>>> print(astr(xout, 3, pp=True))
['4.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, hour=True)
>>> print(astr(xout, 3, pp=True))
['2.500' '5.000' '6.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, half_hour=True)
>>> print(astr(xout, 3, pp=True))
['2.500' '5.000' '6.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, meanday=True)
>>> print(astr(xout[10:16], 3, pp=True))
['-- ' '-- ' '2.500' '5.000' '6.000' '-- ']
>>> odates, xout = means(jdates, x, meanmonth=True)
>>> print(astr(xout[0:5], 3, pp=True))
['4.000' '2.000' '3.000' '-- ' '-- ']
>>> odates, xout = means(jdates, x, seasonal=True)
>>> print(astr(xout[0:5], 3, pp=True))
['4.000' '-- ' '-- ' '-- ' '-- ']
>>> print(astr(means(jdates, x, month=True, onlydat=True), 3, pp=True))
['4.000' '2.000' '3.000']
# Masked arrays
>>> x = np.ma.array(x, mask=np.zeros(x.size, dtype=bool))
>>> x.mask[0] = True
>>> odates, xout = means(jdates, x)
>>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True))
2.448e+06 4.000
>>> odates, xout = means(jdates, x, year=True)
>>> print(astr(xout, 3, pp=True))
['4.000']
>>> odates, xout = means(jdates, x, month=True)
>>> print(astr(xout, 3, pp=True))
['5.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, day=True)
>>> print(astr(xout, 3, pp=True))
['5.000' '2.000' '3.000']
# sum
>>> odates, xout = means(jdates, x, sum=True)
>>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True))
2.448e+06 20.000
>>> odates, xout = means(jdates, x, year=True, sum=True)
>>> print(astr(xout, 3, pp=True))
['20.000']
>>> odates, xout = means(jdates, x, month=True, sum=True)
>>> print(astr(xout, 3, pp=True))
['15.000' ' 2.000' ' 3.000']
>>> odates, xout = means(jdates, x, day=True, sum=True)
>>> print(astr(xout, 3, pp=True))
['15.000' ' 2.000' ' 3.000']
# max
>>> odates, xout = means(jdates, x, max=True)
>>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True))
2.448e+06 6.000
>>> odates, xout = means(jdates, x, year=True, max=True)
>>> print(astr(xout, 3, pp=True))
['6.000']
>>> odates, xout = means(jdates, x, month=True, max=True)
>>> print(astr(xout, 3, pp=True))
['6.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, day=True, max=True)
>>> print(astr(xout, 3, pp=True))
['6.000' '2.000' '3.000']
# min
>>> odates, xout = means(jdates, x, min=True)
>>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True))
2.448e+06 2.000
>>> odates, xout = means(jdates, x, year=True, min=True)
>>> print(astr(xout, 3, pp=True))
['2.000']
>>> odates, xout = means(jdates, x, month=True, min=True)
>>> print(astr(xout, 3, pp=True))
['4.000' '2.000' '3.000']
>>> odates, xout = means(jdates, x, day=True, min=True)
>>> print(astr(xout, 3, pp=True))
['4.000' '2.000' '3.000']
# 2D and masked arrays
>>> x = np.repeat(x,2).reshape((x.size,2))
>>> odates, xout = means(jdates, x)
>>> print(astr(odates, 3, pp=True), astr(xout, 3, pp=True))
2.448e+06 ['4.000' '4.000']
>>> odates, xout = means(jdates, x, year=True)
>>> print(astr(xout, 3, pp=True))
[['4.000' '4.000']]
>>> odates, xout = means(jdates, x, month=True)
>>> print(astr(xout, 3, pp=True))
[['5.000' '5.000']
['2.000' '2.000']
['3.000' '3.000']]
>>> odates, xout = means(jdates, x, day=True)
>>> print(astr(xout, 3, pp=True))
[['5.000' '5.000']
['2.000' '2.000']
['3.000' '3.000']]
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python
library, Department of Computational Hydrosystems, Helmholtz Centre for
Environmental Research - UFZ, Leipzig, Germany.
Copyright (c) 2013-2018 Matthias Cuntz - mc (at) macu (dot) de
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
History
-------
Written, Matthias Cuntz, Jul 2013
Modified, Matthias Cuntz, Jul 2013 - meanday
Matthias Cuntz, Apr 2014 - max, min
Matthias Cuntz, Jun 2015 - onlydat, meanmonth
Matthias Cuntz, Oct 2018 - seasonal
Matthias Cuntz, Jun 2019
- bug in minute averaging: compred unique minutes to hours
instead of minutes
- half_hour inspired by Robin Leucht for level1 data
- did not take keyword 'retrospective' from Robin Leucht but
added a Note above for this case
"""
# Constants
myundef = 9e33
ismasked = type(dat) == np.ma.core.MaskedArray
# Assure array
if not ismasked:
dat = np.array(dat)
# Assure ND-array
isone = False
if dat.ndim == 1:
isone = True
dat = dat[:, np.newaxis]
# Check options
allopts = day + month + year + hour + half_hour + minute + meanday + meanmonth + seasonal
assert allopts <= 1, (
"only one averaging option day, month, year, etc. possible")
# Check aggregation
allaggs = sum + max + min
assert allaggs <= 1, "only one aggregation option sum, min, max possible"
# average 1st dim
if allopts == 0:
dout = 0.5 * (date[-1] + date[0])
if sum:
out = np.ma.sum(dat, 0)
elif max:
out = np.ma.amax(dat, 0)
elif min:
out = np.ma.amin(dat, 0)
else:
out = np.ma.mean(dat, 0)
if isone:
if onlydat:
return out[0]
else:
return dout, out[0]
else:
if onlydat:
return out
else:
return dout, out
else:
yr, mo, dy, hr, mn, sc = dec2date(date)
# year
if year:
yrs = np.unique(yr)
nout = yrs.size
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in yrs:
ii = np.where(yr == i)[0]
if np.size(ii) > 0:
dout[zahl] = date2dec(yr=i, mo=6, dy=15, hr=12)
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
# month
if month:
yrs = np.unique(yr)
mos = np.unique(mo)
nout = yrs.size * mos.size
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in yrs:
for j in mos:
ii = np.where((yr == i) & (mo == j))[0]
if np.size(ii) > 0:
dout[zahl] = date2dec(yr=i, mo=j, dy=15, hr=12)
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
# day
if day:
yrs = np.unique(yr)
mos = np.unique(mo)
dys = np.unique(dy)
nout = yrs.size * mos.size * dys.size
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in yrs:
for j in mos:
for k in dys:
ii = np.where((yr == i) & (mo == j) & (dy == k))[0]
if np.size(ii) > 0:
dout[zahl] = date2dec(yr=i, mo=j, dy=k, hr=12)
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
# hour
if hour:
yrs = np.unique(yr)
mos = np.unique(mo)
dys = np.unique(dy)
hrs = np.unique(hr)
nout = yrs.size * mos.size * dys.size * hrs.size
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in yrs:
for j in mos:
for k in dys:
for l in hrs:
ii = np.where((yr == i) & (mo == j) & (dy == k)
& (hr == l))[0]
if np.size(ii) > 0:
dout[zahl] = date2dec(yr=i,
mo=j,
dy=k,
hr=l,
mi=30)
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
# half_hour
if half_hour:
yrs = np.unique(yr)
mos = np.unique(mo)
dys = np.unique(dy)
hrs = np.unique(hr)
nout = yrs.size * mos.size * dys.size * hrs.size * 2
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in yrs:
for j in mos:
for k in dys:
for l in hrs:
for m in range(2):
ii = np.where((yr == i) & (mo == j) & (dy == k)
& (hr == l)
& ((mn // 30) == m))[0]
if np.size(ii) > 0:
dout[zahl] = date2dec(yr=i,
mo=j,
dy=k,
hr=l,
mi=15 + m * 30)
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(
dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(
dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(
dat[ii, :], 0)
zahl += 1
# minute
if minute:
yrs = np.unique(yr)
mos = np.unique(mo)
dys = np.unique(dy)
hrs = np.unique(hr)
mns = np.unique(mn)
nout = yrs.size * mos.size * dys.size * hrs.size * mns.size
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in yrs:
for j in mos:
for k in dys:
for l in hrs:
for m in mns:
ii = np.where((yr == i) & (mo == j) & (dy == k)
& (hr == l) & (mn == m))[0]
if np.size(ii) > 0:
dout[zahl] = date2dec(yr=i,
mo=j,
dy=k,
hr=l,
mi=m,
sc=30)
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(
dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(
dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(
dat[ii, :], 0)
zahl += 1
# Mean daily
if meanday:
nout = 24
hrs = range(nout)
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in hrs:
dout[zahl] = date2dec(yr=yr[0], mo=1, dy=1, hr=i, mi=30)
ii = np.where(hr == i)[0]
if np.size(ii) > 0:
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
if np.any(out == myundef):
out = np.ma.array(out, mask=(out == myundef), keep_mask=True)
# Mean monthly
if meanmonth:
nout = 12
mos = range(1, nout + 1)
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in mos:
dout[zahl] = date2dec(yr=yr[0], mo=i, dy=15, hr=12, mi=0)
ii = np.where(mo == i)[0]
if np.size(ii) > 0:
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
if np.any(out == myundef):
out = np.ma.array(out, mask=(out == myundef), keep_mask=True)
# Seasonal
if seasonal:
dim = np.array(
[[-9, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31],
[-9, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]])
# leaps = np.array((((yr%4)==0) & ((yr%100)!=0)) |
# ((yr%400)==0)).astype(int)
leaps = (((yr % 4) == 0) & ((yr % 100) != 0)) | ((yr % 400) == 0)
leap = np.any(leaps)
ileap = int(leap)
if leap:
iileap = np.argmax(leaps)
nout = 366
else:
iileap = 0
nout = 365
dys = range(1, nout + 1)
dout = np.ones(nout) * myundef
if ismasked:
out = np.ma.ones([nout] + list(dat.shape[1:])) * myundef
else:
out = np.ones([nout] + list(dat.shape[1:])) * myundef
zahl = 0
for i in range(1, 13):
for j in range(1, dim[ileap, i] + 1):
dout[zahl] = date2dec(yr=yr[iileap],
mo=i,
dy=j,
hr=12,
mi=0)
ii = np.where((mo == i) & (dy == j))[0]
if np.size(ii) > 0:
if sum:
out[zahl, :] = np.ma.sum(dat[ii, :], 0)
elif max:
out[zahl, :] = np.ma.amax(dat[ii, :], 0)
elif min:
out[zahl, :] = np.ma.amin(dat[ii, :], 0)
else:
out[zahl, :] = np.ma.mean(dat[ii, :], 0)
zahl += 1
if np.any(out == myundef):
out = np.ma.array(out, mask=(out == myundef), keep_mask=True)
ii = np.where(dout != myundef)[0]
if len(ii) > 0:
dout = dout[ii]
out = out[ii, :]
else:
raise ValueError("all values undefined after mean.")
if isone:
if onlydat:
return out[:, 0]
else:
return dout, out[:, 0]
else:
if onlydat:
return out
else:
return dout, out
def fluxplot(infile,
outfile,
numheader=1,
lineofvar=1,
novalue=-9999,
delimiter=',',
format='ascii',
format_str='%d-%m-%Y %H:%M',
units=False,
plot=False):
'''
Plotting routine for Eddy Covariance data generated by EddyFlux (and
basically any other ascii data file) with first column containing time steps
in either 'asci' or 'eng' time format. Loads the file and plots each column
in a separate page of a pdf file. Missing values are plotted in red.
Definition
----------
fluxplot(infile, outfile, numheader=1, lineofvar=1, novalue=-9999,
format='ascii', format_str_x='%d-%m-%Y %H:%M', units=False,
plot=False):
Input
-----
infile str, path and name of the input file
outfile str, path and name of the output pdf
Optional Input
--------------
numhead int, number of header lines in the input files (default: 1)
lineofvar int, line in input file containing variable names (default: 1)
novalue int/float, missing value in input file (default: -9999)
delimiter str, delimiter of the input file
format str, format of time stamps in input files. 'ascii' or 'eng' is
possible (default: 'ascii')
format_str str, format string for plot date formatting
(default: '%d-%m-%Y %H:%M')
units bool, if True, all units of a standard EddyFlux output file
are added to each plot. Needs to be set False if other files
are plotted (default: False)
plot bool, only if True, the plot is performed (default: False)
Output
------
outfile pdf with each page containing a plot of one column of the input
file
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Aug 2014
'''
if plot:
import matplotlib
matplotlib.use('PDF')
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib import rcParams
else:
raise ValueError('fluxplot: plot must be True')
############################################################################
# set parameters for matplotlib
rcParams['ps.papersize'] = 'a4'
rcParams['savefig.orientation'] = 'landscape'
rcParams['lines.linewidth'] = 0.75
rcParams['axes.linewidth'] = 1
rcParams['axes.titlesize'] = 12
rcParams['axes.labelsize'] = 8
rcParams['xtick.labelsize'] = 8
rcParams['ytick.labelsize'] = 8
rcParams['legend.fontsize'] = 8
rcParams['path.simplify'] = False
############################################################################
# reading input file
d = np.loadtxt(infile, dtype='|S100', delimiter=delimiter)
header = d[lineofvar - 1, 1:]
date = d[numheader:, 0]
value = d[numheader:, 1:]
value[value == ''] = str(novalue)
value = value.astype(np.float)
missing = np.where(value == novalue, 0, np.NaN)
value = np.where(value == novalue, np.NaN, value)
length = len(date)
if units:
units = [
'W m-2', 'W m-2', 'mmol m-2 s-1', 'mumol m-2 s-1', 'm s-1',
'kg m-1 s-2', 'm2 s-2', 'm2 s-2', 'm2 s-2', 'K2', 'mmol2 m-6',
'mmol2 m-6', 'm s-1', 'm s-1', 'm s-1', ' deg C', 'mmol m-3',
'mmol m-3', 'm s-1', 'deg', 'deg', 'deg', 'deg', 'samples',
'samples', 'm', '-', 'deg C', 'W m-2', 'W m-2', 'mmol m-2 s-1',
'mumol m-2 s-1', 'm', 'm', 'm', '-', '-', '-', '-', '-', '-',
'm2 s-2', 'm2 s-2', 'm2 s-2', '-', 'samples', 'samples', 'samples',
'samples', 'samples', 'samples', 'samples', 'samples', 'samples',
'samples', 'samples', 'samples', 'samples', 'hPa', 'deg C', '%',
'hPa', 'kg m-3', 'samples', 'samples', 'm', 'm'
]
else:
units = [''] * length
############################################################################
# convert input date in JD and array format
date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0)
if format == 'ascii':
jdarr = date2dec(ascii=date) - date01
elif format == 'eng':
jdarr = date2dec(eng=date) - date01
else:
raise ValueError('fluxplot: format unknown')
############################################################################
# plot
majticks = matplotlib.dates.MonthLocator(bymonthday=1)
pdf_pages = PdfPages(outfile)
############################################################################
for i in xrange(len(header)): #len(header)
fig = plt.figure(i)
ax = fig.add_subplot(111)
ax.plot(jdarr, value[:, i], 'b-')
ylim = ax.get_ylim()
if (ylim[0] < 0) and (ylim[1] > 0):
ax.axhline(y=0, linewidth=0.75, color='k')
ax.plot(jdarr, missing[:, i], 'r-', lw=2)
else:
ax.plot(jdarr,
np.where(missing[:, i] == 0, ylim[0], np.NaN),
'r-',
lw=2)
ax.set_xlabel('Date [DD-MM-YYYY]')
ylabel = '%s \n %s' % (header[i], units[i])
ax.set_ylabel(ylabel)
ax.set_xlim(jdarr[0], jdarr[-1])
ax.xaxis.set_major_locator(majticks)
ax.xaxis.set_major_formatter(
matplotlib.dates.DateFormatter(format_str))
fig.autofmt_xdate()
pdf_pages.savefig(fig)
plt.close()
pdf_pages.close()
def fluxpart(fluxfile,
metfile,
outdir,
swdr,
tair,
rh,
method='local',
nogppnight=False,
delimiter=[',', ','],
skiprows=[1, 1],
format=['ascii', 'ascii'],
undef=-9999,
plot=False):
'''
Wrapper function for nee2gpp with file management and plotting.
Definition
----------
fluxpart(fluxfile, metfile, outdir, rg, tair, rh, method='local',
delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'],
undef=-9999, plot=False):
Input
-----
fluxfile str, path and file name of fluxflag or fluxfill output file
containing fluxes and flags
metfile str, path and file name of the meteorology file (must be in
sync with fluxfile)
outdir str, path of the output folder
swdr int, column number of short wave downward radiation [W m-2] in
metfile, column number starts with 0 which is first data column.
swdr is used for lasslopp and for swdr>0=isday
tair int, column number of air temperature [deg C] in metfile, column
number starts with 0 which is first data column.
rh int, column number of relative humidity [%] in metfile, column
number starts with 0 which is first data column.
Optional Input
--------------
method str, if 'global', fit of Reco vs. temperature to all nighttime data
if 'local' | 'reichstein', method of Reichstein et al. (2005)
if 'day' | 'lasslop', method of Lasslop et al. (2010)
(default: 'local')
nogppnight if True: Resp=NEE, GPP=0 at night, GPP always positive
if False: Resp=lloyd_taylor, GPP=Resp-NEE at night (default)
delimiter list of str, delimiters of fluxfile and metfile
(default: [',',','])
skiprows list of int, lines to skip at the beginning of fluxfile and
metfile, e.g. header lines (default: [1,1])
format list of str, time formats of fluxfile and metfile, 'ascii' and
'eng' possible (default: ['ascii','ascii'])
undef int/float, missing value of fluxfile and metfile
(default: -9999, np.nan is not possible)
plot bool, if True performs plotting (default: False)
Output
------
fluxpart.csv file containing fluxes with original flags and quality flags
of the gap filling plus gpp and reco fluxes. gpp and reco get
flags of c flux.
fluxpart.pdf plot of c flux, gpp and reco
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Aug 2014
'''
###########################################################################
# reading input files
d = np.loadtxt(fluxfile,
dtype='|S100',
delimiter=delimiter[0],
skiprows=skiprows[0])
m = np.loadtxt(metfile,
dtype='|S100',
delimiter=delimiter[1],
skiprows=skiprows[1])
assert (d.shape[1] == 16) | (
d.shape[1] == 24
), 'fluxfill: fluxfile must be from fluxfill and have 16 or 24 cols'
if format[0] == 'ascii':
datev = date2dec(ascii=d[:, 0])
elif format[0] == 'eng':
datev = date2dec(eng=d[:, 0])
else:
raise ValueError('fluxpart: unknown format')
if format[1] == 'ascii':
datem = date2dec(ascii=m[:, 0])
elif format[1] == 'eng':
datem = date2dec(eng=m[:, 0])
else:
raise ValueError('fluxpart: unknown format')
val = np.where(d[:, 1:] == '', str(undef), d[:, 1:]).astype(np.float)
met = np.where(m[:, 1:] == '', str(undef), m[:, 1:]).astype(np.float)
###########################################################################
# assign variables
if (d.shape[1] == 16):
nee = val[:, 9] #corr. C
else:
nee = val[:, 12] # oder 13 #corr. C
nee_stor = val[:, 13] # oder 13 #corr. C
swdr = met[:, swdr]
isday = swdr > 0.
tair = met[:, tair] + 273.15
rh = met[:, rh]
#vpd = (1.-rh/100.)*esat(tair)
vpd = np.empty_like(tair)
vpd[(tair == undef) | (rh == undef)] = undef
vpd[~((tair == undef) |
(rh == undef))] = (1. - rh[~((tair == undef) |
(rh == undef))] / 100.) * esat(tair[~(
(tair == undef) | (rh == undef))])
###########################################################################
# do partitioning
if (d.shape[1] == 16):
gpp, reco = nee2gpp.nee2gpp(datev,
nee,
tair,
isday,
rg=swdr,
vpd=vpd,
undef=undef,
method=method,
shape=False,
masked=False,
nogppnight=nogppnight)
else:
gpp, reco = nee2gpp.nee2gpp(datev,
nee,
tair,
isday,
rg=swdr,
vpd=vpd,
undef=undef,
method=method,
shape=False,
masked=False,
nogppnight=nogppnight)
gpp_stor, reco_stor = nee2gpp.nee2gpp(datev,
nee_stor,
tair,
isday,
rg=swdr,
vpd=vpd,
undef=undef,
method=method,
shape=False,
masked=False,
nogppnight=nogppnight)
#######################################################################
# plot
if plot:
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.backends.backend_pdf as pdf
pp1 = pdf.PdfPages(outdir + '/fluxpart.pdf')
majticks = mpl.dates.MonthLocator(bymonthday=1)
format_str = '%d %m %Y %H:%M'
date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0)
fig1 = plt.figure(1)
if (d.shape[1] == 16):
sub1 = fig1.add_subplot(111)
else:
sub1 = fig1.add_subplot(211)
sub2 = fig1.add_subplot(212)
l1 = sub1.plot(datev - date01, nee, '-k', label='nee')
l2 = sub1.plot(datev - date01, gpp, '-g', label='gpp')
l3 = sub1.plot(datev - date01, reco, '-r', label='reco')
if (d.shape[1] != 16):
l4 = sub2.plot(datev - date01, nee_stor, '-k', label='nee')
l5 = sub2.plot(datev - date01, gpp_stor, '-g', label='gpp')
l6 = sub2.plot(datev - date01, reco_stor, '-r', label='reco')
sub1.set_xlim(datev[0] - date01, datev[-1] - date01)
sub1.xaxis.set_major_locator(majticks)
sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str))
if (d.shape[1] != 16):
sub2.set_xlim(datev[0] - date01, datev[-1] - date01)
sub2.xaxis.set_major_locator(majticks)
sub2.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str))
fig1.autofmt_xdate()
plt.legend(loc='best')
plt.show()
fig1.savefig(pp1, format='pdf')
pp1.close()
###########################################################################
# prepare output and save file
if (d.shape[1] == 16):
header = np.array([
' H', ' Hflag', ' Hgf', ' LE', ' LEflag',
' LEgf', ' E', ' Eflag', ' Egf', ' C',
' Cflag', ' Cgf', ' TAU', ' TAUflag', ' TAUgf',
' GPP', ' GPPflag', ' GPPgf', ' Reco', 'Recoflag',
' Recogf'
])
flux = np.vstack((gpp, reco)).transpose()
flux_str = np.array([['%11.5f' % x for x in y]
for y in flux]).repeat(3, axis=1)
flux_str[:, [1, 2, 4, 5]] = np.tile(d[:, 11:13], 2)
output = np.hstack((d[:, :], flux_str))
else:
header = np.array([
' H', ' H+sT', ' Hflag', ' Hgf',
' LE', ' LE+sLE', ' LEflag', ' LEgf',
' E', ' E+sE', ' Eflag', ' Egf',
' C', ' C+sC', ' Cflag', ' Cgf',
' TAU', ' TAUflag', ' TAUgf', ' sT',
' sLE', ' sE', ' sC', ' GPP',
' GPP+sC', ' GPPflag', ' GPPfg', ' Reco',
' Reco+sC', 'Recoflag', ' Recofg'
])
flux = np.vstack((gpp, gpp_stor, reco, reco_stor)).transpose()
flux_str = np.array([['%11.5f' % x for x in y] for y in flux])
flux_str = np.insert(flux_str, [2, 2, 4, 4], flux_str, axis=1)
flux_str[:, [2, 3, 6, 7]] = np.tile(d[:, 11:13], 2)
output = np.hstack((d[:, :], flux_str))
np.savetxt('%s/fluxpart.csv' % outdir,
np.vstack((np.concatenate(
([' date'], header))[np.newaxis, :], output)),
'%s',
delimiter=',')
def fluxflag(fluxfile,
metfile,
outdir,
swdr,
T,
lat,
delimiter=[',', ','],
skiprows=[1, 1],
format=['ascii', 'ascii'],
limit=0.3,
analyzer='LI7000',
spike_f=True,
itc_f=True,
ustar_f=True,
novalue=-9999,
plot=False):
'''
Quality flag calculation for Eddy Covariance data originating from EddyFlux.
Including stationarity, exceeding limits, exceeding change rates, exceeding
variances, integral turbulence characteristics, spike detection and ustar
thresholds. Missing values are flagged with 2. Ustar thresholding works
ONLY for a data set of ONE FULL year.
Definition
----------
fluxflag(fluxfile, metfile, outdir, Rg, T, lat, delimiter=[',',','],
skiprows=[1,1], format=['ascii','ascii'], limit=0.3,
analyzer='LI7000', spike_f=True, itc_f=True, ustar_f=True,
novalue=-9999, plot=False):
Input
-----
fluxfile str, path and file name of EddyFlux output file
metfile str, path and file name of the meteorology file (must be in
sync with fluxfile)
outdir str, path of the output folder
swdr int, column number of short wave incoming radiation in metfile,
column number starts with 0 which is first data column.
Used for swdr>0=isday
T int, column number of air temperature in metfile, column number
starts with 0 which is first data column. Used for ustarflag
lat latitude of tower position in decimal degrees. Used for itcflag
Optional Input
--------------
delimiter list of str, delimiters of fluxfile and metfile
(default: [',',','])
skiprows list of int, lines to skip at the beginning of fluxfile and
metfile, e.g. header lines (default: [1,1])
format list of str, time formats of fluxfile and metfile, 'ascii' and
'eng' possible (default: ['ascii','ascii'])
limit float, relative deviation limit from the itc model above which
values are flagged with itcflag. e.g. 0.3 means 30% above and
30% below the model (default: 0.3)
analyzer str, analyzer type. 'LI7000' and 'LI7500' possible. Used for
exceeding variance limits. (default: 'LI7000')
spike_f bool, if True spike detection and flagging is performed
itc_f bool, if True itc calculation and flagging is performed
ustar_f bool, if True ustar threshold calculation and flagging is
performed
novalue int/float, missing value of fluxfile and metfile
(default: -9999)
plot bool, if True subroutines (itc, spike and ustar) perform
plotting (default: False)
Output
------
flags.csv file containing all calculated flags
fluxflags.csv file containing fluxes with the sum of all respective flags
for each flux. Where flag>2, flag=2.
flags.log file containing statistics for each flag
Restrictions
------------
- ustar flagging works ONLY for a data set of ONE FULL year
- ustar flagging works ONLY for half hourly time steps
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Aug 2014
'''
###########################################################################
# reading input files
d = np.loadtxt(fluxfile,
dtype='|S100',
delimiter=delimiter[0],
skiprows=skiprows[0])
m = np.loadtxt(metfile,
dtype='|S100',
delimiter=delimiter[1],
skiprows=skiprows[1])
if format[0] == 'ascii':
datev = date2dec(ascii=d[:, 0])
elif format[0] == 'eng':
datev = date2dec(eng=d[:, 0])
else:
raise ValueError('fluxflag: unknown format')
if format[1] == 'ascii':
datem = date2dec(ascii=m[:, 0])
elif format[1] == 'eng':
datem = date2dec(eng=m[:, 0])
else:
raise ValueError('fluxflag: unknown format')
val = np.where(d[:, 1:] == '', str(novalue), d[:, 1:]).astype(np.float)
val = np.where(val == novalue, np.NaN, val)
met = np.where(m[:, 1:] == '', str(novalue), m[:, 1:]).astype(np.float)
met = np.where(met == novalue, np.NaN, met)
print('LOADING FILES COMPLETED')
###########################################################################
# calculate isday
isday = met[:, swdr] > 0.
fluxes = val[:, [28, 29, 30, 31,
5]] # corr. H, corr. LE, corr. E, corr. C, tau
zeta = val[:, 26]
varu = val[:, 41]
ustar = val[:, 4]
varw = val[:, 8]
vart = val[:, 9]
rho = val[:, 62]
H = val[:, 28]
T = met[:, T]
###########################################################################
# calculate standard flags
sttest2 = np.where(np.isnan(val[:, 37:38]), np.NaN,
np.unpackbits(val[:, 37:38].astype('uint8'), axis=1))
h2oexli = np.where(np.isnan(val[:, 56]), np.NaN,
np.where(val[:, 56] > 100, 2, 0))
c02exli = np.where(np.isnan(val[:, 55]), np.NaN,
np.where(val[:, 55] > 100, 2, 0))
h2oexcr = np.where(np.isnan(val[:, 50]), np.NaN,
np.where(val[:, 50] > 50, 2, 0))
co2excr = np.where(np.isnan(val[:, 49]), np.NaN,
np.where(val[:, 49] > 50, 2, 0))
texcr = np.where(np.isnan(val[:, 48]), np.NaN,
np.where(val[:, 48] > 50, 2, 0))
tauexcr = np.where(
np.isnan(val[:, 45]) ^ np.isnan(val[:, 47]), np.NaN,
np.where((val[:, 45] > 50) ^ (val[:, 47] > 50), 2, 0))
zetaexcr = np.where(
np.isnan(val[:, 45]) ^ np.isnan(val[:, 47]) ^ np.isnan(val[:, 48]),
np.NaN,
np.where((val[:, 45] > 50) ^ (val[:, 47] > 50) ^ (val[:, 48] > 50), 2,
0))
wexvar = np.where(np.isnan(val[:, 8]), np.NaN,
np.where(val[:, 8] > 3, 1, 0))
texvar = np.where(np.isnan(val[:, 9]), np.NaN,
np.where(val[:, 9] > 3, 1, 0))
if analyzer == 'LI7000':
h2oexvar3 = np.where(np.isnan(val[:, 11]), np.NaN,
np.where(val[:, 11] > 3, 1, 0))
h2oexvar10 = np.where(np.isnan(val[:, 11]), np.NaN,
np.where(val[:, 11] > 10, 1, 0))
co2exvar3 = np.where(np.isnan(val[:, 10]), np.NaN,
np.where(val[:, 10] > 3, 1, 0))
elif analyzer == 'LI7500':
h2oexvar3 = np.where(np.isnan(val[:, 11]), np.NaN,
np.where(val[:, 11] > 5000, 1, 0))
h2oexvar10 = np.where(np.isnan(val[:, 11]), np.NaN,
np.where(val[:, 11] > 17300, 1, 0))
co2exvar3 = np.where(np.isnan(val[:, 10]), np.NaN,
np.where(val[:, 10] > 3.5, 1, 0))
else:
raise ValueError('fluxflag: unknown analyzer')
wvar0 = np.where(np.isnan(val[:, 8]), np.NaN,
np.where(val[:, 8] < 0.00001, 2, 0))
h2ovar0 = np.where(np.isnan(val[:, 11]), np.NaN,
np.where(val[:, 11] < 0.00001, 2, 0))
co2var0 = np.where(np.isnan(val[:, 10]), np.NaN,
np.where(val[:, 10] < 0.00001, 2, 0))
print('CALCULATE STANDARD FLAGS COMPLETE')
###########################################################################
# itc flag calculation
if itc_f:
itcu, itcw, itct = itc.itc(H,
zeta,
ustar,
varu,
varw,
vart,
rho,
lat,
limit,
'%s/itc' % outdir,
plot=plot)
print('ITC CALCUATION COMPLETED')
else:
itcu = np.zeros_like(H, dtype=np.int)
itcw = np.zeros_like(H, dtype=np.int)
itct = np.zeros_like(H, dtype=np.int)
###########################################################################
# summing flags for each flux
Hflag = np.nansum(
np.vstack(
(sttest2[:, 6], itcw, texcr, wexvar, texvar, wvar0)).transpose(),
1).astype(int)
Hflag[Hflag > 1] = 2
LEflag = np.nansum(
np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3,
wvar0, h2ovar0)).transpose(), 1).astype(int)
LEflag[LEflag > 1] = 2
Eflag = np.nansum(
np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3,
wvar0, h2ovar0)).transpose(), 1).astype(int)
Eflag[Eflag > 1] = 2
Cflag = np.nansum(
np.vstack((sttest2[:, 4], itcw, c02exli, co2excr, wexvar, h2oexvar10,
wvar0, co2var0)).transpose(), 1).astype(int) #, co2exvar3
Cflag[Cflag > 1] = 2
Tauflag = np.nansum(
np.vstack(
(sttest2[:, 3], itcu, itcw, tauexcr, wexvar, wvar0)).transpose(),
1).astype(int)
Tauflag[Tauflag > 1] = 2
###########################################################################
# spike detection
#inflag = np.zeros_like(fluxes, dtype=np.int)
inflag = np.vstack((Hflag, LEflag, Eflag, Cflag, Tauflag)).transpose()
if spike_f:
spikef = spikeflag.spikeflag(datev,
fluxes,
inflag,
isday,
'%s/spike' % outdir,
z=7,
deriv=1,
udef=np.NaN,
spike_v=2,
plot=plot)
print('SPIKE DETECTION COMPLETED')
else:
spikef = np.zeros_like(inflag, dtype=np.int)
inflag += spikef
###########################################################################
# ustar flagging
if ustar_f:
C_ustar_T = np.vstack((fluxes[:, 3], ustar, T)).transpose()
C_ustar_T_flags = np.isnan(C_ustar_T).astype(np.int)
C_ustar_T_flags[:, 0] += inflag[:, 3]
ustarf = ustarflag.ustarflag(datev,
C_ustar_T,
C_ustar_T_flags,
isday,
'%s/ustar' % outdir,
ustar_v=2,
plot=plot)
print('USTAR FLAGGING COMPLETED')
else:
ustarf = np.zeros_like(datev, dtype=np.int)
###########################################################################
# the big gathering :-D
header = np.array([
'sttest2_H', ' sttest2_LE/E', ' sttest2_C', ' sttest2_tau', ' h2oexli',
' c02exli', ' h2oexcr', ' co2excr', ' texcr', ' tauexcr', ' zetaexcr',
' wexvar', ' texvar', ' h2oexvar3', ' h2oexvar10', ' co2exvar3',
' wvar0', ' h2ovar0', ' co2var0', ' itcu', ' itcw', ' itct',
' spike_H', ' spike_LE', ' spike_E', ' spike_C', ' spike_tau', ' ustar'
])
maxlen = [len(x) for x in header]
flags = np.vstack(
(sttest2[:, 6], sttest2[:, 5], sttest2[:, 4], sttest2[:, 3], h2oexli,
c02exli, h2oexcr, co2excr, texcr, tauexcr, zetaexcr, wexvar, texvar,
h2oexvar3, h2oexvar10, co2exvar3, wvar0, h2ovar0, co2var0, itcu, itcw,
itct, spikef[:, 0], spikef[:, 1], spikef[:, 2], spikef[:, 3],
spikef[:, 4], ustarf)).transpose()
flags = np.where(np.isnan(flags), 2, flags).astype(np.int)
flags_str = np.zeros_like(flags, dtype='|S100')
for i, item in enumerate(maxlen):
flags_str[:, i] = astr(flags[:, i], prec=item)
# save flag file
output = np.hstack((d[:, 0:1], flags_str))
np.savetxt('%s/flags.csv' % outdir,
np.vstack((np.concatenate(
([' date'], header))[np.newaxis, :], output)),
'%s',
delimiter=',')
###########################################################################
# summing flags for each flux
Hflag = np.nansum(
np.vstack((sttest2[:, 6], itcw, texcr, wexvar, texvar, wvar0,
spikef[:, 0])).transpose(), 1).astype(int)
Hflag[Hflag > 1] = 2
LEflag = np.nansum(
np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3,
wvar0, h2ovar0, spikef[:, 1])).transpose(), 1).astype(int)
LEflag[LEflag > 1] = 2
Eflag = np.nansum(
np.vstack((sttest2[:, 5], itcw, h2oexli, h2oexcr, wexvar, h2oexvar3,
wvar0, h2ovar0, spikef[:, 2])).transpose(), 1).astype(int)
Eflag[Eflag > 1] = 2
Cflag = np.nansum(
np.vstack((sttest2[:, 4], itcw, c02exli, co2excr, wexvar, h2oexvar10,
co2exvar3, wvar0, co2var0, spikef[:,
3], ustarf)).transpose(),
1).astype(int) ### why h20exvar10
Cflag[Cflag > 1] = 2
Tauflag = np.nansum(
np.vstack((sttest2[:, 3], itcu, itcw, tauexcr, wexvar, wvar0,
spikef[:, 4])).transpose(), 1).astype(int)
Tauflag[Tauflag > 1] = 2
header2 = np.array([
' H', ' Hflag', ' LE', ' LEflag', ' E',
' Eflag', ' C', ' Cflag', ' TAU', ' TAUflag'
])
fluxes[np.isnan(fluxes)] = novalue
fluxes_str = np.array([['%11.5f' % x for x in y] for y in fluxes])
fluxes_str = np.where(fluxes_str == '%11.5f' % novalue,
' ' * (11 - len(str(novalue))) + str(novalue),
fluxes_str)
Hflag_str = astr(Hflag, prec=8)
LEflag_str = astr(LEflag, prec=8)
Eflag_str = astr(Eflag, prec=8)
Cflag_str = astr(Cflag, prec=8)
Tauflag_str = astr(Tauflag, prec=8)
output = np.hstack((d[:, 0:1], fluxes_str.repeat(2, axis=1)))
output[:, 2::2] = np.vstack((Hflag_str, LEflag_str, Eflag_str, Cflag_str,
Tauflag_str)).transpose()
np.savetxt('%s/fluxflags.csv' % outdir,
np.vstack((np.concatenate(
([' date'], header2))[np.newaxis, :], output)),
'%s',
delimiter=',')
###########################################################################
# write log file with stats
log = open('%s/flags.log' % outdir, 'w')
log.write('flag, 0, 1, 2\n')
for i, item in enumerate(header):
hist = tuple(np.histogram(flags[:, i], [-0.5, 0.5, 1.5, 2.5])[0])
log.write(item.strip() + ', %i, %i, %i\n' % hist)
log.close()
def meteo4slt(sltdir, metfile, p_t_rh, outfile,
pat='[a-zA-Z0-9]*.slt|[a-zA-Z0-9]*.SLT', delimiter=',',
skiprows=1, format='ascii'):
"""
To supply EddyFlux (Kolle & Rebmann, 2007) with meteorological
data, it is necessary to extract for each available *.slt file the
corresponding air pressure, air temperature and air relative humidity.
The module looks in sltdir for available *.slt files and uses the metfile
for syncronisation
Definition
----------
meteo4slt(sltdir, metfile, p_t_rh, outfile,
pat='[a-zA-Z0-9]*.slt|[a-zA-Z0-9]*.SLT', delimiter=',',
skiprows=1, format='ascii'):
Input
-----
sltdir str, path of the folder containing the *.slt files
metfile str, path of the meteo file
p_t_rh list, column number of Pressure, T, rH
outfile str, path of the output file
Optional Input
--------------
pat str, regular expression, describing the name pattern of
the *.slt files in the indir folder
delimiter str, column delimiter of the meteo file (default=',')
skiprows int, number of rows to skip at the beginning of the met file
e.g. header lines (default=1)
format str, time format of the meteo file (default='ascii')
Output
------
outfile file, containing Pressure, T and rH values of the meteo
file for each *.slt file
Restrictions
------------
- assumes site name in slt fielname is only one character
(e.g. W20133652300.slt AND NOT WULF20133652300.slt)
- currently only supports format='ascii', nice would be 'eng'
- does not work for data containning multiple years because of ugly
doy calculation
- works only for half hourly *.stl and half hourly meteo data
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Jul 2014
Modified, MC, Aug 2014 - clean up and Python 3
"""
# half hour in jd
halfhjd = 1800./86400.
###########################################################################
# reading slt directory
dirlist = os.listdir(sltdir)
sltdates = np.array([])
###########################################################################
# remove all files and folders from list which are not *.slt files and get size
pat = re.compile(pat)
for item in dirlist:
if re.search(pat, item):
sltdates = np.append(sltdates, item[1:-4])
# replace time steps which dont fit in half hour interval
mi = np.array([x[-2:] for x in sltdates])
mi = np.where(mi.astype(int)<30, '00', '30')
sltdates = np.array([sltdates[i][:-2]+mi[i] for i in range(mi.size)])
###########################################################################
# load meteo data
metdata = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter,
skiprows=skiprows, usecols=[0]+p_t_rh)
# get the metdate
if format == 'ascii':
# shift met dates one half hour back since slt time stamp marks half
# hour start but meteo date mark half hour end
jd = date2dec(ascii=metdata[:,0]) - halfhjd
fulldate = dec2date(jd, fulldate=True)
adate = dec2date(jd, ascii=True)
doy = np.ceil(date2dec(ascii=adate)-
date2dec(yr=fulldate[0][0],
mo=1,dy=1,hr=0,mi=0, sc=0)).astype(int)
doy = np.where((fulldate[3]==0) & (fulldate[4]==0), doy+1, doy)
metdates = np.array(['%04i%03i%02i%02i'%(fulldate[0][i], doy[i],
fulldate[3][i], fulldate[4][i])
for i in range(jd.size)])
elif format == 'eng':
# shift met dates one half hour back since slt time stamp marks half
# hour start but meteo date mark half hour end
jd = date2dec(eng=metdata[:,0]) - halfhjd
fulldate = dec2date(jd, fulldate=True)
adate = dec2date(jd, ascii=True)
doibegin = np.array([date2dec(yr=x, mo=1,dy=1,hr=0,mi=0, sc=0) for x in fulldate[0]])
doy = np.ceil(date2dec(ascii=adate)-doibegin).astype(int)
doy = np.where((fulldate[3]==0) & (fulldate[4]==0), doy+1, doy)
metdates = np.array(['%04i%03i%02i%02i'%(fulldate[0][i], doy[i],
fulldate[3][i], fulldate[4][i])
for i in range(jd.size)])
else:
raise ValueError('meteo4slt: unknown format!')
###########################################################################
# sync both dates
mask = np.in1d(metdates, sltdates)
###########################################################################
# write output
np.savetxt(outfile, metdata[mask,1:], '%s', ',')
def eddyspec(indir,
cfile,
hfile,
rawfile,
sltdir,
tspfile='34_specmean.csv',
cspfile='35_specmean.csv',
hspfile='36_specmean.csv',
novalue=-9999,
plot=False):
'''
Provides plots with carbon and water fluxes from the rawfile together
with the respective time lags in the cfile and hfile for the user to select
'golden days' with sufficient flux and constant lags (about 6 half hour
values during midday) to perform spectrum analysis with EddySpec and
SpecMean (Kolle & Rebmann, 2007). When user selected desired time period,
the respective *.slt files are printed to the console. The user can than
use EddySpec and SpecMean with these files and the script continues after
user is finished. Inductances for water and carbon are fitted and saved
together with spectrum plots. It is recommended to repeat the spectrum
analysis for different times within the year.
Definition
----------
eddyspec(indir, cfile, hfile, rawfile, sltdir, tspfile='34_specmean.csv',
cspfile='35_specmean.csv', hspfile='36_specmean.csv', novalue=-9999):
Input
-----
indir str, path of the folder where results will be saved
cfile str, path of the carbon lag file
hfile str, path of the water lag file
rawfile str, path of the raw flux file
sltdir str, path of the folder containing the *.slt files (will contain
EddySpec and SpecMean files after use)
Optional Input
--------------
tspfile str, name of the average temperature spectrum file
(default: '34_specmean.csv')
cspfile str, name of the average carbon spectrum file
(default: '35_specmean.csv')
hspfile str, name of the average water spectrum file
(default: '36_specmean.csv')
novalue int, novalue in rawfile (default=-9999)
Output
------
(for each run in the year, a new folder is created in indir and EddySpec and
SpecMean files are moved here from sltdir)
c_specw.pdf plot of the carbon spectrum
h_specw.pdf plot of the water spectrum
spec_X_X.log log file containing used *.slt files and inductances
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Jul 2014
Modified, MC, Aug 2014 - clean up and Python 3
'''
############################################################################
# reading input files
#lags
lagdate = np.array(sread('%s' % (cfile), nc=1, skip=1), dtype='|S16')
day = np.array([x[5:8] for x in lagdate.flatten()],
dtype='|S7').astype(float)
hour = np.array([x[8:10] for x in lagdate.flatten()],
dtype='|S2').astype(float)
min = np.array([x[10:12] for x in lagdate.flatten()],
dtype='|S2').astype(float)
doysfloat = day + (hour + min / 60.) / 24.
cl = np.array(fread('%s' % (cfile), skip=1, cskip=1))
hl = np.array(fread('%s' % (hfile), skip=1, cskip=1))
# fluxes
fluxdate = date2dec(
ascii=np.array(sread('%s' % (rawfile), nc=1, skip=1), dtype='|S16'))
year = np.array(sread('%s' % (rawfile), nc=1, skip=1), dtype='|S16')
year = np.array([x[6:10] for x in year.flatten()], dtype='|S4').astype(int)
fluxdate = fluxdate.flatten() - date2dec(
yr=year, mo=1, dy=1, hr=0, mi=0, sc=0)
cf = np.array(fread('%s' % (rawfile), skip=1, cskip=4, nc=1))
cf = np.where(cf == novalue, np.nan, cf).flatten()
hf = np.array(fread('%s' % (rawfile), skip=1, cskip=1, nc=1))
hf = np.where(hf == novalue, np.nan, hf).flatten()
if plot:
import matplotlib.pyplot as plt
import matplotlib.backends.backend_pdf as pdf
############################################################################
# plot lag c
plt.figure(1)
plt.plot(cl[:, 0], cl[:, 2], 'bo', label='minlag (sam)')
plt.plot(cl[:, 0], cl[:, 5], 'ro', label='maxlag (sam)')
plt.xlabel('DOY')
plt.ylabel('Lags [Samples]')
plt.title('c lags')
plt.axis('auto')
plt.grid('on')
plt.legend()
############################################################################
# plot flux c
plt.figure(2)
plt.plot(fluxdate, cf, 'r-', label='cflux')
plt.xlabel('DOY')
plt.ylabel('cflux [?]')
plt.title('c fluxes')
plt.axis('auto')
plt.grid('on')
plt.legend()
############################################################################
# plot lag h
plt.figure(3)
plt.plot(hl[:, 0], hl[:, 2], 'bo', label='minlag (sam)')
plt.plot(hl[:, 0], hl[:, 5], 'ro', label='maxlag (sam)')
plt.xlabel('DOY')
plt.ylabel('Lags [Samples]')
plt.title('h lags')
plt.axis('auto')
plt.grid('on')
plt.legend()
############################################################################
# plot flux h
plt.figure(4)
plt.plot(fluxdate, hf, 'b-', label='hflux')
plt.xlabel('DOY')
plt.ylabel('hflux [?]')
plt.title('h fluxes')
plt.axis('auto')
plt.grid('on')
plt.legend()
plt.show()
interval = []
inp = True
while inp:
inpfrom = raw_input("From doy [ddd.ddd]: ")
inpto = raw_input("To doy [ddd.ddd]: ")
try:
interval += [
np.min(np.where(np.abs(doysfloat - float(inpfrom)) < 0.02083))
]
interval += [
np.min(np.where(np.abs(doysfloat - float(inpto)) < 0.02083))
]
inp = False
except (ValueError, IndexError):
print(
'EddySpecWarning: type in floats(with . not ,), nothing else!')
inp = True
inpclag = raw_input("C lag: ")
inphlag = raw_input("H lag: ")
print(lagdate[interval[0]:interval[1] + 1])
print('\nDO EDDYSPEC AND SPECMEAN NOW!\n')
inp = raw_input("Mean spectrums ready? [y/n]: ").lower()
if inp == "y":
pass
else:
import sys
sys.exit()
############################################################################
# move spectrum files from slt folder to spec folder
if len(os.listdir(indir)) == 0:
os.mkdir(indir + '/1')
indir = indir + '/1'
else:
os.mkdir(indir + '/%i' % (int(os.listdir(indir)[-1]) + 1))
indir = indir + '/' + os.listdir(indir)[-1]
specpat = re.compile('[0-9]*_specmean.csv')
specpat2 = re.compile('[a-zA-Z0-9]*_[0-9]*_spec.csv')
sltdirlist = os.listdir(sltdir)
for file in sltdirlist:
if bool(re.search(specpat, file)) | bool(re.search(specpat2, file)):
sh.move('%s/%s' % (sltdir, file), indir)
############################################################################
# conductance fitting
# reading mean spec files
tsp = np.array(fread('%s/%s' % (indir, tspfile), skip=1, nc=2))
csp = np.array(fread('%s/%s' % (indir, cspfile), skip=1, nc=2))
hsp = np.array(fread('%s/%s' % (indir, hspfile), skip=1, nc=2))
# filter input specs for log10(input) != NaN
global fcoc, fcoh
fcoc = csp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1]))))
& (np.invert(np.isnan(np.log10(csp[:, 1])))))[0], 1]
fcoh = hsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1]))))
& (np.invert(np.isnan(np.log10(hsp[:, 1])))))[0], 1]
xc = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1]))))
& (np.invert(np.isnan(np.log10(csp[:, 1])))))[0], 0]
yc = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1]))))
& (np.invert(np.isnan(np.log10(csp[:, 1])))))[0], 1]
xh = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1]))))
& (np.invert(np.isnan(np.log10(hsp[:, 1])))))[0], 0]
yh = tsp[np.where((np.invert(np.isnan(np.log10(tsp[:, 1]))))
& (np.invert(np.isnan(np.log10(hsp[:, 1])))))[0], 1]
# calculate deviations from t spec for different inductences and select the one with the smallest residual
con = np.arange(0, 5, 0.01)
devc = np.empty_like(con)
devh = np.empty_like(con)
for i in xrange(np.shape(con)[0]):
devc[i] = np.nansum(np.log10(yc) - modc(xc, con[i]))
devh[i] = np.nansum(np.log10(yh) - modh(xh, con[i]))
conc = con[np.argmin(np.abs(devc))]
conh = con[np.argmin(np.abs(devh))]
if plot:
############################################################################
# plot c spec
fig1 = plt.figure(5)
sub = fig1.add_subplot(111)
sub.plot(np.log10(tsp[:, 0]),
np.log10(tsp[:, 1]),
'ro-',
label='T Spec')
sub.plot(np.log10(csp[:, 0]),
np.log10(csp[:, 1]),
'bo-',
label='C Spec')
sub.plot(np.log10(xc),
modc(xc, conc),
'go-',
label='C Spec with %.2f Induc.' % (conc))
plt.xlabel('F[Hz]')
plt.ylabel('F[Hz]*Co(w,X)')
sub.set_title('c Spectrum')
sub.axis('auto')
sub.grid('on')
sub.legend(loc='best')
############################################################################
# plot h spec
fig2 = plt.figure(6)
sub = fig2.add_subplot(111)
sub.plot(np.log10(tsp[:, 0]),
np.log10(tsp[:, 1]),
'ro-',
label='T Spec')
sub.plot(np.log10(hsp[:, 0]),
np.log10(hsp[:, 1]),
'bo-',
label='H Spec')
sub.plot(np.log10(xh),
modh(xh, conh),
'go-',
label='H Spec %.2f Induc.' % (conh))
plt.xlabel('F[Hz]')
plt.ylabel('F[Hz]*Co(w,X)')
sub.set_title('h Spectrum')
sub.axis('auto')
sub.grid('on')
sub.legend(loc='best')
plt.show()
############################################################################
# save figures
pp1 = pdf.PdfPages('%s/c_spec%s.pdf' % (indir, rawfile[6:-4]))
pp2 = pdf.PdfPages('%s/h_spec%s.pdf' % (indir, rawfile[6:-4]))
fig1.savefig(pp1, format='pdf')
fig2.savefig(pp2, format='pdf')
pp1.close()
pp2.close()
################################################################################
# writing log file
log = open(
'%s/spec_%i_%02i_%02i_%02i_%02i_%02i.log' % (
indir,
t.localtime()[0],
t.localtime()[1],
t.localtime()[2],
t.localtime()[3],
t.localtime()[4],
t.localtime()[5],
), 'w')
log.write('Inductence determination for the mean of:\n')
for item in lagdate[interval[0]:interval[1] + 1]:
log.write('%s\n' % (item[0]))
log.write('\nC lag: %s\n' % (inpclag))
log.write('H lag: %s\n' % (inphlag))
log.write('\nC Inductence: %.2f\n' % (conc))
log.write('H Inductence: %.2f\n' % (conh))
def energyclosure(fluxfile, metfile, outdir, Rn, G, swdr, Ts=None, theta=None,
depths=None, por=None, method='year', force0=True,
delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'],
undef=-9999, plot=False):
'''
Calculation of energy balance closure and correction of Eddy Covariance data
with different methods. Possible calculation of soil heat flux from soil
temperature and moisture if not given.
Definition
----------
energyclosure(fluxfile, metfile, outdir, Rn, G, swdr, Ts=None, theta=None,
depths=None, por=None, method='year', force0=True,
delimiter=[',',','], skiprows=[1,1], format=['ascii','ascii'],
undef=-9999, plot=False):
Input
-----
fluxfile str, path and file name of fluxflag or fluxfill or fluxpart
output file containing fluxes and flags
metfile str, path and file name of the meteorology file (must be in
sync with fluxfile)
outdir str, path of the output folder
Rn int, column number of net radiation [W m-2] in metfile, column
number starts with 0 which is first data column.
G int, column number of soil heat flux [W m-2] in metfile, column
number starts with 0 which is first data column. If soil heat
flux not available, set to False
swdr int, column number of short wave downward radiation [W m-2] in
metfile, column number starts with 0 which is first data column.
swdr is used for swdr>0=isday
Optional Input
--------------
Ts list(M) of int, if G is not given, column numbers of soil
temperatures in certain depths in metfile, column number starts
with 0 which is first data column.
theta list(M) of int, if G is not given, column numbers of soil
moistures in certain depths in metfile, column number starts
with 0 which is first data column.
depths list(M) or array(M) of int, if G is not given, depths of Ts and
theta measurements [m], positively increasing with depths
e.g. [0.05, 0.30, 0.60]
por float, if G is not given, porosity of the soil [-], must be
bigger or equal than np.amax(theta)
method str, method of how energy balance closure is calculated and
applied.
if 'year', fit of whole year daytime flag=0 data
Rn-G vs. H+Le and application of the fraction to all H, Le
and E values
implement new methods here
implement new methods here
implement new methods here
(default: 'year')
force0 bool, if method='year', True forces fit through origin, if False
fit is allowed to have intercept (default: True)
delimiter list of str, delimiters of fluxfile and metfile
(default: [',',','])
skiprows list of int, lines to skip at the beginning of fluxfile and
metfile, e.g. header lines (default: [1,1])
format list of str, time formats of fluxfile and metfile, 'ascii' and
'eng' possible (default: ['ascii','ascii'])
undef int/float, missing value of fluxfile and metfile
(default: -9999, np.nan is not possible)
plot bool, if True performs plotting (default: False)
Output
------
fluxclosed.csv file containing fluxes and flags where depending on the
method chosen fluxes are corrected for energy balance
closure
fluxclosed.pdf if method='year', plot of whole year daytime flag=0 data
Rn-G vs. H+Le
fluxclosed_stor.pdf if method='year', plot of whole year daytime flag=0 data
Rn-G vs. H+Le including storages
Restrictions
------------
Works only with half hourly time steps
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Sep 2014
'''
###########################################################################
# reading input files
d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0])
m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1])
assert (d.shape[1]==22) | (d.shape[1]==32), 'energyclosure: fluxfile must be from fluxpart and have 22 or 32 cols'
if format[0]=='ascii':
datev = date2dec(ascii=d[skiprows[0]:,0])
elif format[0]=='eng':
datev = date2dec(eng=d[skiprows[0]:,0])
else:
raise ValueError('energyclosure: unknown format')
if format[1]=='ascii':
datem = date2dec(ascii=m[skiprows[1]:,0])
elif format[1]=='eng':
datem = date2dec(eng=m[skiprows[1]:,0])
else:
raise ValueError('energyclosure: unknown format')
val = np.where(d[skiprows[0]:,1:]=='', str(undef), d[skiprows[0]:,1:]).astype(np.float)
met = np.where(m[skiprows[1]:,1:]=='', str(undef), m[skiprows[1]:,1:]).astype(np.float)
###########################################################################
# assign variables
if (d.shape[1]==22):
H = val[:,0]
Hflag = val[:,1]
Le = val[:,3]
Leflag = val[:,4]
E = val[:,6]
Eflag = val[:,7]
else:
H = val[:,0]
Hflag = val[:,2]
Le = val[:,4]
Leflag = val[:,6]
E = val[:,8]
Eflag = val[:,10]
H_stor = val[:,1]
Hflag_stor = val[:,2]
Le_stor = val[:,5]
Leflag_stor = val[:,6]
E_stor = val[:,9]
Eflag_stor = val[:,10]
Rn = met[:,Rn]
Ts = met[:,Ts]
swdr = met[:,swdr]
isday = swdr>0.
###########################################################################
# check if soil heat flux is given or needs to be calculated
if not G:
if Ts==None or theta==None or depths==None or por==None:
raise ValueError('energyclosure: if G is not given, Ts, theta, depths and rhos are needed to calculate G')
else:
G=soilheatflux(Ts, theta, depths, rhos)
else:
G = met[:,G]
###########################################################################
# apply energy balance closure methods
# yearly approach
if (method.lower() == 'year'):
closure = energyclosure_year(H, Hflag, Le, Leflag, Rn, G, isday, outdir,
force0=force0, undef=undef, plot='energyclosure.pdf')
if force0:
H_closed, Le_closed, E_closed = H/closure, Le/closure, E/closure
else:
H_closed, Le_closed, E_closed = H/closure[0], Le/closure[0], E/closure[0]
if (d.shape[1]==32):
closure_stor = energyclosure_year(H_stor, Hflag_stor, Le_stor, Leflag_stor,#
Rn, G, isday, outdir, force0=force0,
undef=undef, plot='energyclosure_stor.pdf')
if force0:
H_stor_closed, Le_stor_closed, E_stor_closed = H_stor/closure, Le_stor/closure, E_stor/closure
else:
H_stor_closed, Le_stor_closed, E_stor_closed = H_stor/closure[0], Le_stor/closure[0], E_stor/closure[0]
# Include new methods here
# Include new methods here
# Include new methods here
else:
raise ValueError('energyclosure: method not implemented yet.')
H_closed[H==undef] = undef
Le_closed[Le==undef] = undef
E_closed[E==undef] = undef
if (d.shape[1]==32):
H_stor_closed[H_stor==undef] = undef
Le_stor_closed[Le_stor==undef] = undef
E_stor_closed[E_stor==undef] = undef
###########################################################################
# prepare and write output
if (d.shape[1]==22):
flux = np.vstack((H_closed,Le_closed,E_closed)).transpose()
else:
flux = np.vstack((H_closed,H_stor_closed,Le_closed,Le_stor_closed,E_closed,E_stor_closed)).transpose()
flux_str = np.array([['%11.5f'%x for x in y] for y in flux])
flux_str = np.where(flux_str=='%11.5f'%undef, ' '*(11-len(str(undef)))+str(undef), flux_str)
if (d.shape[1]==22):
d[skiprows[0]:,[1,4,7]] = flux_str
else:
d[skiprows[0]:,[1,2,5,6,9,10]] = flux_str
np.savetxt('%s/fluxclosed.csv'%outdir, d, '%s', delimiter=',')
def t2sap(date, data, swd=None, undef=-9999.):
"""
Conversion of temperature difference measured with sap flow sensors
(Granier type) in (mV) to sap flux density (cm^3 cm^-2 h^-1) (Granier, 1987).
In addition, the correction according to Clearwater (1999) is possible.
Definition
----------
def t2sap(date, data, swd=None, undef=-9999.):
Input
-----
date 1D array (n,) of ascii times in format DD.MM.YYYY hh:mm:ss
data ND array (n,...) of the raw data in mV. First dimension is time
Optional Input
--------------
undef values are excluded from calculations (default: -9999)
swd if None: sapflux calculation according to original Granier
calibration function.
if not None: sapwood depth in cm for Clearwater correction
T = (T - b*Tmax)/a
with a = active sapwood = swd/2
b = inactive sapwood = 1-a
Output
------
SFD 2D array (n,m) with sap flux density in [cm^3 cm^-2 h^-1] according
to the original Granier calibration function:
SFD = 0.0119*K**1.231*3600 [Granier, 1985]
References
----------
Granier, A.
Evaluation of transpiration in a Douglas-fir stand by means of sap flow measurements,
Tree Physiology 3, 309-320, 1987
Clearwater, M. J., Meinzer, F. C., Andrade, J. L., Goldstein, G., Holbrook, N. M.
Potential errors in measurement of nonuniform sap flow using heat dissipation probes,
Tree Physiology 19, 681-687, 1999
Examples
--------
# normal sapflux conversion
>>> data = np.array([0.434, 0.433, 0.432, 0.431, 0.431, 0.432])
>>> date = ['18.05.2013 08:00', '18.05.2013 08:10', '18.05.2013 08:20',
... '18.05.2013 08:30', '18.05.2013 08:40', '18.05.2013 08:50']
>>> SFD = t2sap(date, data)
>>> print(np.round(SFD,3))
[0. 0.024 0.057 0.095 0.095 0.057]
>>> # sapflux conversion including clearwater correction
>>> data = np.array([0.434, 0.433, 0.432, 0.431, 0.431, 0.432])
>>> date = ['18.05.2013 08:00', '18.05.2013 08:10', '18.05.2013 08:20',
... '18.05.2013 08:30', '18.05.2013 08:40', '18.05.2013 08:50']
>>> SFD = t2sap(date, data, swd=1.5)
>>> print(np.round(SFD,3))
[0. 0.035 0.082 0.135 0.135 0.082]
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014-2016 Andreas Wiedemann, Matthias Cuntz - mc (at) macu (dot) de
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AW, Jun 2014
Modified, MC, Jun 2014 - sap_app -> t2sap, incl. undef, first date then data
- squeeze 1D array, fill undef if not masked
Modified, MC, Nov 2016 - ported to Python 3
"""
isvec = False
if data.ndim == 1:
isvec = True
data = data.reshape((-1, 1))
# mask data
isnotmask = True
if type(data) == np.ma.core.MaskedArray:
isnotmask = False
data_masked = np.ma.array(data, mask=(data == undef))
# julian day
jd = np.array(date2dec(ascii=date))
# integer julian day
jd_int = jd.astype(np.int)
# unique days
jd_uni = np.unique(jd_int)
# Tmax per day
tmax_day = np.ma.ones((jd_uni.size, data.shape[1])) * undef
# Time of Tmax per day
jdmax_day = np.ma.zeros((jd_uni.size, data.shape[1]), dtype=np.int)
# Determine Tmax per day
for count, i in enumerate(jd_uni):
# where is given day
ii = np.where(jd_int == i)[0]
# index of Tmax of day
jj = np.ma.argmax(data_masked[ii, :], axis=0)
# time at Tmax of day
jdmax_day[count, :] = jd[ii[jj]]
# Tmax per day
tmax_day[count, :] = data_masked[ii[jj], :]
# Tmax for every record
tmax = np.empty(data.shape)
for i in range(data.shape[1]):
# time points at Tmax per day
xx = jdmax_day[:, i]
# Tmax per day
yy = tmax_day[:, i]
ii = ~yy.mask
# Tmax per time point
tmax[:, i] = np.interp(jd, xx[ii], yy[ii])
if swd == None:
Tsw = data_masked
else:
# sapwood depth in cm / 2cm for needle lenght = portion of sensor in sapwood
a = 0.5 * swd
# portion of sensor in inactive xylem
b = 1.0 - a
# define weighted mean of T in the sapwood (a) and T in the inactive xylem (b)
Tsw = (data_masked - (b * tmax)) / a
# converts raw data [mV] into Sap Flux density [cm3*cm-2*h-1]
SFD = (0.0119 * ((tmax - Tsw) / Tsw)**1.231) * 3600.
# if it was not masked then the original undef will be undef
if isnotmask:
SFD = SFD.filled(undef)
# 1D in = 1D out
if isvec:
SFD = np.squeeze(SFD)
return SFD
def fluxfill(fluxfile, metfile, outdir, rg, tair, rh, delimiter=[',',','],
skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False):
'''
Wrapper function for gapfill with file management and plotting.
Definition
----------
fluxfill(fluxfile, metfile, outdir, rg, tair, rh, delimiter=[',',','],
skiprows=[1,1], format=['ascii','ascii'], undef=-9999, plot=False):
Input
-----
fluxfile str, path and file name of fluxflag output file containing
fluxes and flags
metfile str, path and file name of the meteorology file (must be in
sync with fluxfile)
outdir str, path of the output folder
rg int, column number of global radiation [W m-2] in metfile,
column number starts with 0 which is first data column.
tair int, column number of air temperature [deg C] in metfile, column
number starts with 0 which is first data column.
rh int, column number of relative humidity [%] in metfile, column
number starts with 0 which is first data column.
Optional Input
--------------
delimiter list of str, delimiters of fluxfile and metfile
(default: [',',','])
skiprows list of int, lines to skip at the beginning of fluxfile and
metfile, e.g. header lines (default: [1,1])
format list of str, time formats of fluxfile and metfile, 'ascii' and
'eng' possible (default: ['ascii','ascii'])
undef int/float, missing value of fluxfile and metfile
(default: -9999, np.nan not possible)
plot bool, if True performs plotting (default: False)
Output
------
fluxfilled.csv file containing fluxes with original flags and quality flags
of the gap filling. Fluxes are filled where flag>1.
fluxfilled.pdf plot of each gap filled flux.
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Aug 2014
'''
###########################################################################
# reading input files
d = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0], skiprows=skiprows[0])
m = np.loadtxt(metfile, dtype='|S100', delimiter=delimiter[1], skiprows=skiprows[1])
assert (d.shape[1]==11) | (d.shape[1]==19), 'fluxfill: fluxfile must be from fluxflag or profile2storage and have 11 or 19 cols'
if format[0]=='ascii':
datev = date2dec(ascii=d[:,0])
elif format[0]=='eng':
datev = date2dec(eng=d[:,0])
else:
raise ValueError('fluxfill: unknown format')
if format[1]=='ascii':
datem = date2dec(ascii=m[:,0])
elif format[1]=='eng':
datem = date2dec(eng=m[:,0])
else:
raise ValueError('fluxfill: unknown format')
val = np.where(d[:,1:]=='', str(undef), d[:,1:]).astype(np.float)
met = np.where(m[:,1:]=='', str(undef), m[:,1:]).astype(np.float)
###########################################################################
# assign variables
if (d.shape[1]==11):
data = val[:,0::2] # corr. H, corr. LE, corr. E, corr. C, tau
data_flag = (val[:,1::2]>1) | (val[:,0::2]==undef)
else:
data = val[:,[0,1, 3,4, 6,7, 9,10, 12]] # corr. H, corr. H+s, corr. LE, corr. LE+s, corr. E, corr. E+s, corr. C, corr. C+s, tau
data_flag = np.repeat((val[:,[2,5,8,11,13]]>1) | (val[:,[0,1, 3,4, 6,7, 9,10, 12]]==undef), 2, axis=1)[:,:-1]
rg = met[:,rg]
rg_flag = rg==undef
tair = met[:,tair]
tair_flag = tair==undef
rh = met[:,rh]
rh_flag = rh==undef
vpd = np.empty_like(tair)
vpd[tair_flag | rh_flag] = undef
vpd[~(tair_flag | rh_flag)] = (1.-rh[~(tair_flag | rh_flag)]/100.)*esat(tair[~(tair_flag | rh_flag)]+273.14)
vpd_flag = vpd==undef
flux = np.zeros_like(data)
flag = np.zeros_like(data_flag, dtype=np.int)
###########################################################################
if plot:
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.backends.backend_pdf as pdf
pp1 = pdf.PdfPages(outdir+'/fluxfilled.pdf')
###########################################################################
# do gapfill
for i in range(data.shape[1]):
flux[:,i], flag[:,i] = gapfill.gapfill(datev, data[:,i], rg, tair, vpd,
data_flag[:,i], rg_flag, tair_flag,
vpd_flag, rg_dev=50., tair_dev=2.5,
vpd_dev=5., longgap=60, fullday=False,
undef=undef, ddof=1, err=False,
shape=False)
#######################################################################
# plot
if plot:
majticks = mpl.dates.MonthLocator(bymonthday=1)
format_str='%d %m %Y %H:%M'
date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0)
fig1 = plt.figure(1)
sub1 = fig1.add_subplot(111)
l1 =sub1.plot(datev-date01, flux[:,i], '-g')
l2 =sub1.plot(datev-date01, np.ma.array(data[:,i],mask=data_flag[:,i]), '-b')
sub1.set_xlim(datev[0]-date01,datev[-1]-date01)
sub1.xaxis.set_major_locator(majticks)
sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str))
fig1.autofmt_xdate()
plt.show()
fig1.savefig(pp1, format='pdf')
if plot:
pp1.close()
###########################################################################
# prepare output and save file
flux_str = np.array([['%11.5f'%x for x in y] for y in flux])
if (d.shape[1]==11):
header = np.array([' H', ' Hflag', ' Hgf',
' LE', ' LEflag', ' LEgf',
' E', ' Eflag', ' Egf',
' C', ' Cflag', ' Cgf',
' TAU', ' TAUflag', ' TAUgf'])
output = np.hstack((d[:,0:1], flux_str.repeat(3, axis=1)))
output[:,2::3] = astr(val[:,1::2].astype(int), prec=8)
output[:,3::3] = astr(flag, prec=8)
else:
header = np.array([' H', ' H+sT', ' Hflag', ' Hgf',
' LE', ' LE+sLE', ' LEflag', ' LEgf',
' E', ' E+sE', ' Eflag', ' Egf',
' C', ' C+sC', ' Cflag', ' Cgf',
' TAU', ' TAUflag', ' TAUgf',
' sT', ' sLE', ' sE', ' sC'])
output = np.hstack((d[:,0:1], np.insert(flux_str, [2,2,4,4,6,6,8,8], flux_str[:,:-1], axis=1),
flux_str[:,-1:].repeat(2, axis=1), d[:,-4:]))
output[:,[3,7,11,15,18]] = astr(val[:,[2,5,8,11,13]].astype(int), prec=8)
output[:,[4,8,12,16,19]] = astr(flag[:,[0,2,4,6,8]], prec=8)
np.savetxt('%s/fluxfilled.csv'%outdir,
np.vstack((np.concatenate(([' date'], header))[np.newaxis,:],
output)), '%s', delimiter=',')
def profile2storage(fluxfile,
fluxfile2,
profilefile,
outdir,
heights,
CO2=None,
H2O=None,
T=None,
rH=None,
delimiter=[',', ',', ','],
skiprows=[1, 1, 1],
format=['ascii', 'ascii', 'ascii'],
undef=-9999,
plot=False):
'''
Calculates storage fluxes for changes in CO2, H2O, air temperature and air
moisture from profile data or meteorological data to correct Eddy
Covariance fluxes. FLux files from EddySoft and from fluxflag are needed as
well as a file with the profile or meteo data. Fluxes will be updated with
the respective storage fluxes and saved in a new file. Multiple application
of this routine with different profile or meteo files are possible to
correct e.g. the CO2, H2O and latent heat fluxes with profile data of CO2
and H2O concentrations and afterwards the H flux with temperature data from
another file.
Definition
----------
profile2storage(fluxfile, fluxfile2, profilefile, outdir, heights, CO2=None,
H2O=None, T=None, rH=None, delimiter=[',',',',','],
skiprows=[1,1,1], format=['ascii','ascii','ascii'],
undef=-9999, plot=False):
Input
-----
fluxfile str, path and file name of fluxflag output file containing
fluxes and flags. These fluxes will be updated by the storage
fluxes and saved as a new file
fluxfile2 str, path and file name of EddyFlux output file (timestep
checked) containing original fluxes
profilefile str, path and file name of the profile file or meteorology file
containing CO2, H2O, T or rH values to compute the profile
storage from
outdir str, path of the output folder
heights list of floats, observation heights of the profile [m],
increasing e.g. [0.5,1.0,10.0,20.0].
CO2 list of int, column numbers of CO2 concentrations for the
different heights (in the same order) [mumol/mol] in profilefile,
column number starts with 0 which is first data column.
H2O list of int, column numbers of H2O concentrations for the
different heights (in the same order) [mmol/mol] in profilefile,
column number starts with 0 which is first data column.
T list of int, column numbers of air temperatures for the
different heights (in the same order) [degC] in profilefile,
column number starts with 0 which is first data column.
rH list of int, column numbers of relative humidity for the
different heights (in the same order) [%] in profilefile,
column number starts with 0 which is first data column. The
calculation of air vapour energy storage change within the
profile works only when T is given as well.
Optional Input
--------------
delimiter list of str, delimiters of fluxfile, fluxfile and profilefile
(default: [',',',',','])
skiprows list of int, lines to skip at the beginning of fluxfile,
fluxfile and profilefile, e.g. header lines (default: [1,1,1])
format list of str, time formats of fluxfile, fluxfile and profilefile,
'ascii' and 'eng' possible (default: ['ascii','ascii','ascii'])
undef int/float, missing value of fluxfile, fluxfile and profilefile
(default: -9999, np.nan is not possible)
plot bool, if True performs plotting (default: False)
Output
------
flux+stor.csv file containing fluxes and flags where storage fluxes are
added in an additional column and storage fluxes are appended
to the end of the file
Restrictions
------------
Works only with half hourly time steps, all files in sync
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Sep 2014
'''
###########################################################################
# time interval
int = 30.
dt = int * 60.
if plot:
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.backends.backend_pdf as pdf
###########################################################################
# reading input files
# fluxes to correct for storage changes
d1 = np.loadtxt(fluxfile, dtype='|S100', delimiter=delimiter[0])
# original flux file from EddyFlux containing air density rho_a
d2 = np.loadtxt(fluxfile2, dtype='|S100', delimiter=delimiter[1])
# file containing profile data (can be meteo file if no profile available)
d3 = np.loadtxt(profilefile, dtype='|S100', delimiter=delimiter[2])
assert (d1.shape[1] == 11) | (
d1.shape[1] == 19
), 'profile2storage: fluxfile must be from fluxflag or profiletostorage and have 11 or 19 cols'
assert d2.shape[
1] == 68, 'profile2storage: fluxfile2 must be from EddyFlux and have 68 cols'
assert d1.shape[0] == d2.shape[
0], 'profile2storage: fluxfile and fluxfile2 must be in sync'
assert d1.shape[0] == d3.shape[
0], 'profile2storage: fluxfile and profilefile must be in sync'
assert (
((H2O == None) & (rH == None)) ^ ((H2O != None) ^ (rH != None))
), 'profile2storage: give either H2O or rH, both would be double correction'
if format[0] == 'ascii':
datev = date2dec(ascii=d1[skiprows[0]:, 0])
elif format[0] == 'eng':
datev = date2dec(eng=d1[skiprows[0]:, 0])
else:
raise ValueError('profile2storage: unknown format')
if format[2] == 'ascii':
datem = date2dec(ascii=d2[skiprows[2]:, 0])
elif format[2] == 'eng':
datem = date2dec(eng=d2[skiprows[2]:, 0])
else:
raise ValueError('profile2storage: unknown format')
flux1 = np.where(d1[skiprows[0]:, 1:] == '', str(undef),
d1[skiprows[0]:, 1:]).astype(np.float)
flux2 = np.where(d2[skiprows[1]:, 1:] == '', str(undef),
d2[skiprows[1]:, 1:]).astype(np.float)
prof = np.where(d3[skiprows[2]:, 1:] == '', str(undef),
d3[skiprows[2]:, 1:]).astype(np.float)
flux1 = np.ma.array(flux1, mask=flux1 == undef, hard_mask=True)
flux2 = np.ma.array(flux2, mask=flux2 == undef)
prof = np.ma.array(prof, mask=prof == undef)
###########################################################################
# assign variables
if d1.shape[1] == 11:
H, Hflag = flux1[:, 0], flux1[:, 1]
Le, Leflag = flux1[:, 2], flux1[:, 3]
E, Eflag = flux1[:, 4], flux1[:, 5]
C, Cflag = flux1[:, 6], flux1[:, 7]
else:
H, Hflag = flux1[:, 0], flux1[:, 2]
Le, Leflag = flux1[:, 3], flux1[:, 5]
E, Eflag = flux1[:, 6], flux1[:, 8]
C, Cflag = flux1[:, 9], flux1[:, 11]
p = flux2[:, 58] # [hPa]
rho = flux2[:, 62] # [kg/m3]
###########################################################################
# prepare output array
d4 = np.copy(d1)
if d1.shape[1] == 11:
temp = np.empty((d1.shape[0], 4), dtype='|S100')
temp[:] = ' ' * (11 - len(str(undef))) + str(undef)
temp[0, :] = [
' H+sT', ' LE+sLE', ' E+sE', ' C+sC'
]
d4 = np.insert(d4, [2, 4, 6, 8], temp, axis=1)
temp[0, :] = [
' sT', ' sLE', ' sE', ' sC'
]
d4 = np.append(d4, temp, axis=1)
###########################################################################
# calls
if CO2:
CO2 = prof[:, CO2]
assert CO2.shape[1] == len(
heights), 'profile2storage: number of CO2 cols must equal heights'
# calculate storage flux and storage flux flag
sfCO2 = stor2flux(CO2, rho, heights, dt, 'CO2')
sfCO2flag = sfCO2.mask.astype(np.int)
# add to eddy flux
newC = C + np.ma.filled(sfCO2, 0)
# format and write into output array
newC_str = np.array(['%11.5f' % x for x in np.ma.filled(newC, undef)])
newC_str = np.where(newC_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
newC_str)
sfCO2_str = np.array(
['%11.5f' % x for x in np.ma.filled(sfCO2, undef)])
sfCO2_str = np.where(sfCO2_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
sfCO2_str)
d4[skiprows[0]:, 11] = newC_str
d4[skiprows[0]:, 18] = sfCO2_str
if plot:
storplot(CO2, datev, heights, C, sfCO2, newC, 'storageCO2.pdf',
pdf, plt, mpl, outdir)
if H2O:
H2O = prof[:, H2O]
assert H2O.shape[1] == len(
heights), 'profile2storage: number of H2O cols must equal heights'
# calculate storage flux and storage flux flag
sfH2O = stor2flux(H2O, rho, heights, dt, 'H2O')
sfH2O_Wm2 = sfH2O * mmol_h2o * latentheat_vaporization / 1.e6
sfH2Oflag = sfH2O.mask.astype(np.int)
# add to eddy flux
newE = E + np.ma.filled(sfH2O, 0)
newLe = Le + np.ma.filled(sfH2O_Wm2, 0)
# format and write into output array
newE_str = np.array(['%11.5f' % x for x in np.ma.filled(newE, undef)])
newLe_str = np.array(
['%11.5f' % x for x in np.ma.filled(newLe, undef)])
sfH2O_str = np.array(
['%11.5f' % x for x in np.ma.filled(sfH2O, undef)])
sfH2O_Wm2_str = np.array(
['%11.5f' % x for x in np.ma.filled(sfH2O_Wm2, undef)])
newE_str = np.where(newE_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
newE_str)
newLe_str = np.where(newLe_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
newLe_str)
sfH2O_str = np.where(sfH2O_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
sfH2O_str)
sfH2O_Wm2_str = np.where(sfH2O_Wm2_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
sfH2O_Wm2_str)
d4[skiprows[0]:, 8] = newE_str
d4[skiprows[0]:, 17] = sfH2O_str
d4[skiprows[0]:, 5] = newLe_str
d4[skiprows[0]:, 16] = sfH2O_Wm2_str
if plot:
storplot(H2O, datev, heights, E, sfH2O, newE, 'storageH2O.pdf',
pdf, plt, mpl, outdir)
if T:
T = prof[:, T]
assert T.shape[1] == len(
heights), 'profile2storage: number of T cols must equal heights'
# calculate storage flux and storage flux flag
sfT = stor2flux(T, rho, heights, dt, 'T')
sfTflag = sfT.mask.astype(np.int)
# add to eddy flux
newH = H + np.ma.filled(sfT, 0)
# format and write into output array
newH_str = np.array(['%11.5f' % x for x in np.ma.filled(newH, undef)])
newH_str = np.where(newH_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
newH_str)
sfT_str = np.array(['%11.5f' % x for x in np.ma.filled(sfT, undef)])
sfT_str = np.where(sfT_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef), sfT_str)
d4[skiprows[0]:, 2] = newH_str
d4[skiprows[0]:, 15] = sfT_str
if plot:
storplot(T, datev, heights, H, sfT, newH, 'storageT.pdf', pdf, plt,
mpl, outdir)
if rH:
rH = prof[:, rH]
assert rH.shape[1] == len(
heights), 'profile2storage: number of rH cols must equal heights'
# calculate specific humidity
vapourpressure = esat(T + T0) * (rH / 100.) / 100. #[hPa]
specifichumidity = (mmol_h2o / mmol_air * vapourpressure) / (
p - (1. - mmol_h2o / mmol_air) * vapourpressure)
# calculate storage flux and storage flux flag
sfrH_Wm2 = stor2flux(specifichumidity, rho, heights, dt, 'rH')
sfrH = sfrH_Wm2 * 1.e6 / (mmol_h2o * latentheat_vaporization)
sfrHflag = sfrH.mask.astype(np.int)
# add to eddy flux
newE = E + np.ma.filled(sfrH, 0)
newLe = Le + np.ma.filled(sfrH_Wm2, 0)
# format and write into output array
newE_str = np.array(['%11.5f' % x for x in np.ma.filled(newE, undef)])
newLe_str = np.array(
['%11.5f' % x for x in np.ma.filled(newLe, undef)])
sfrH_str = np.array(['%11.5f' % x for x in np.ma.filled(sfrH, undef)])
sfrH_Wm2_str = np.array(
['%11.5f' % x for x in np.ma.filled(sfrH_Wm2, undef)])
newE_str = np.where(newE_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
newE_str)
newLe_str = np.where(newLe_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
newLe_str)
sfrH_str = np.where(sfrH_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
sfrH_str)
sfrH_Wm2_str = np.where(sfrH_Wm2_str == '%11.5f' % undef,
' ' * (11 - len(str(undef))) + str(undef),
sfrH_Wm2_str)
d4[skiprows[0]:, 8] = newE_str
d4[skiprows[0]:, 17] = sfrH_str
d4[skiprows[0]:, 5] = newLe_str
d4[skiprows[0]:, 16] = sfrH_Wm2_str
if plot:
storplot(rH, datev, heights, E, sfH2O, newE, 'storagerH.pdf', pdf,
plt, mpl, outdir)
###########################################################################
# write output
np.savetxt('%s/flux+stor.csv' % outdir, d4, '%s', delimiter=',')
def timestepcheck(indir, pat, outfile, begin, end, numhead=1, timeint=30,
format='ascii', empty='-9999', delimiter=',', skiprows=0):
'''
Checks ascii data files with first column being a time stamp in ascii
of eng style for correctness. If for a given time interval time steps
are missing, they will will be filled with the correct time stamp and
empty values. A beginning and end for the output files can be set to which
data should be kept or filled. Pat defines the file name pattern to consider
in the check. Multiple files, which match the pat are concatenated.
Definition
----------
timestepcheck(indir, pat, outfile, begin, end, numhead=1, timeint=30,
format='ascii', empty='-9999', delimiter=',', skiprows=0):
Input
-----
indir str, path of the folder where the input files are
pat str, name or regular expression of the input files
outfile str, path and name of the output file
begin str, start time of the output file, must be in the same format
as time stamps in the input files
end str, end time of the output file, must be in the same format
as time stamps in the input files
Optional Input
--------------
numhead int, number of header lines in the input files (default: 1)
timeint int, time interval of the input file in minutes (default: 30)
format str, format of time stamps in input files. 'ascii' or 'eng' is
possible (default: 'ascii')
empty str, value for missing values (default: '-9999')
delimiter str, delimiter of the input files (default: ',')
skiprows int, rows to skip in input files, e.g. logger fuzzle before
actual data header starts (default: 0)
Output
------
outfile file with missing time steps filled with empty values cut from
begin to end
Restrictions
------------
TODO: tested thoroughly only for timeint=30
TODO: more bad value checks can be included, see sternchen and tuedelchen
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Aug 2014
'''
###########################################################################
# time interval list
interval = range(0,60,timeint)
#jdint = date2dec(yr=-4712,mo=1,dy=1,hr=12,mi=timeint)
#jdmin = date2dec(yr=-4712,mo=1,dy=1,hr=12,mi=0,sc=30)# (=precision)
jdint = date2dec(yr=1,mo=1,dy=1,hr=12,mi=timeint) % 1
jdmin = date2dec(yr=1,mo=1,dy=1,hr=12,mi=0,sc=30) % 1# (=precision)
if format == 'ascii':
jdbegin = date2dec(ascii = np.array([begin]))
jdend = date2dec(ascii = np.array([end]))
elif format == 'eng':
jdbegin = date2dec(eng = np.array([begin]))
jdend = date2dec(eng = np.array([end]))
###########################################################################
# reading input directory
pat = re.compile(pat)
new = True
filelist = os.listdir(indir)
for file in filelist:
if re.search(pat, file):
if new:
data = np.loadtxt('./%s/%s'%(indir, file), dtype='|S21',\
delimiter=delimiter, skiprows=skiprows)
if data.shape[0] == 0:
print('Warning: File %s is empty!' %(file))
else:
if np.shape(data.shape)[0] == 1:
data = data.reshape((1,-1))
new = False
else:
add_data = np.loadtxt('./%s/%s'%(indir, file), dtype='|S21',
delimiter=delimiter,
skiprows=numhead+skiprows)
if add_data.shape[0] == 0:
print('Warning: File %s is empty!' %(file))
elif np.shape(add_data.shape)[0] == 1:
add_data = add_data.reshape((1,-1))
data = np.append(data, add_data, 0)
else:
data = np.append(data, add_data, 0)
###########################################################################
# sternchen check :-D
# replace with regular expression check
data[data=='***********'] = empty #!!! uberprufen auf lange und re
data[data=='********'] = empty #!!! uberprufen auf lange und re
data[data=='*********'] = empty
###########################################################################
# tuedelchen check :-D
# replace with regular expression check
if data[numhead,0][0] == '"':
data[numhead:,0] = np.array([x[1:-1] for x in data[numhead:,0]])
###########################################################################
# "NAN" check :-D
# replace with regular expression check
data[data=='"NAN"'] = empty #!!! uberprufen auf lange und re
###########################################################################
# leerzeilen check
blankline = np.where(data[0:2,0]=='')[0]
data = np.delete(data, blankline, 0)
###########################################################################
# missing values check
data[data==''] = empty
data[data=='""'] = empty
columns = np.shape(data)[1]-1
###########################################################################
# calculate julian date
if format == 'ascii':
import time
jd = date2dec(ascii = data[numhead:,0])
elif format == 'eng':
jd = date2dec(eng = data[numhead:,0])
###########################################################################
# wrong time stamp check
diff = jd[1:] - jd[:-1]
minute = np.array([x.split()[1][3:5] for x in data[numhead:,0]]).astype(int)
nii = np.nonzero(~np.in1d(minute, interval))[0]
ts = np.nonzero(np.less(diff, jdint-jdmin))[0]
wrong = np.unique(np.append(nii, ts))
if data.shape[0]-numhead-2 in wrong:
wrong = np.append(wrong, [data.shape[0]-numhead-1], 0)
delete = []
for i in wrong:
print('\nHERE IS SOMETHING WRONG:\n')
print('BOF' if numhead+i-2<0 else data[numhead+i-2,:4])
print('BOF' if numhead+i-1<0 else data[numhead+i-1,:4])
print('-----------------------------------------------')
print(data[numhead+i,:4])
print('-----------------------------------------------')
print('EOF' if numhead+i+1>=np.shape(data)[0] else data[numhead+i+1,:4])
print('EOF' if numhead+i+2>=np.shape(data)[0] else data[numhead+i+2,:4])
do = raw_input("\n(d)elete entry, (s)et to empty, (t)ype in date, (i)gnore: ")
if do == 'd':
delete += [numhead+i]
elif do == 's':
data[numhead+i,1:] = empty
elif do == 't':
newdate = str(raw_input("\nreplace with: "))
data[numhead+i,0] = newdate
# newmin = str(raw_input("\n%s"%(data[numhead+i,0][:-2])))
# data[numhead+i,0] = data[numhead+i,0][:-2] + newmin
elif do == 'i':
pass
data = np.delete(data, delete, 0)
###########################################################################
# calculate julian date again
if format == 'ascii':
jd = date2dec(ascii = data[numhead:,0])
elif format == 'eng':
jd = date2dec(eng = data[numhead:,0])
###########################################################################
# check time step
diff = jd[1:] - jd[:-1]
ingap = np.where(np.greater(diff, jdint+jdmin))[0]
nugap = np.rint((diff[ingap]/jdint)-1)
###########################################################################
# insert missing time steps
for i in range(np.size(ingap))[::-1]:
where = np.ones(int(nugap[i]), dtype=int)*(ingap[i]+1+numhead)
if format == 'ascii':
span = np.arange(1,nugap[i]+1)*jdint + jd[ingap[i]]
what = dec2date(span.astype('|S16').astype(float), ascii=True)
elif format == 'eng':
span = np.arange(1,nugap[i]+1)*jdint + jd[ingap[i]]
what = dec2date(span.astype('|S16').astype(float), eng=True)
what = np.array([x[:-3] for x in what])
miss = np.empty((int(nugap[i]),columns), dtype='|S11')
miss[:] = empty
what = np.append(np.reshape(what, (-1,1)), miss, 1)
data = np.insert(data, where, what, 0)
###########################################################################
# fill/cut up/off beginning and end
start = np.where(data[:,0]==begin)[0]
if start == numhead:
pass
elif start > numhead:
data = np.delete(data, np.arange(numhead, start), 0)
else:
if format == 'ascii':
tofill = int((date2dec(ascii = data[numhead,0]) - jdbegin)/jdint)
span = np.arange(0,tofill)*jdint + jdbegin # tofill+1 weggenommen!!!
what = dec2date(span.astype('|S16').astype(float), ascii=True)
elif format == 'eng':
tofill = int((date2dec(eng = data[numhead,0]) - jdbegin)/jdint)-1
span = np.arange(0,tofill)*jdint + jdbegin # tofill+1 weggenommen!!!
what = dec2date(span.astype('|S16').astype(float), eng=True)
what = np.array([x[:-3] for x in what])
where = np.ones(tofill, dtype=int)*numhead # tofill+1 weggenommen!!!
#if doidate:
# miss = np.empty((tofill,columns-2), dtype='|S11') # tofill+1 weggenommen!!!
#else:
miss = np.empty((tofill,columns), dtype='|S11') # tofill+1 weggenommen!!!
miss[:] = empty
what = np.append(np.reshape(what, (-1,1)), miss, 1)
data = np.insert(data, where, what, 0)
stop = np.where(data[:,0]==end)[0]
maxind = np.shape(data)[0]-1
if stop == maxind:
pass
elif stop < maxind:
#data = data[:stop+1,:]
data = data[:stop[0]+1,:]
else:
if format == 'ascii':
tofill = int((jdend - date2dec(ascii = data[-1,0]))/jdint)
span = np.arange(1,tofill+1)*jdint + date2dec(ascii = data[-1,0])
what = dec2date(span.astype('|S16').astype(float), ascii=True)
elif format == 'eng':
tofill = int((jdend - date2dec(eng = data[-1,0]))/jdint)
span = np.arange(1,tofill+1)*jdint + date2dec(eng = data[-1,0])
what = dec2date(span.astype('|S16').astype(float), eng=True)
what = np.array([x[:-3] for x in what])
#if doidate:
# miss = np.empty((tofill,columns-2), dtype='|S11')
#else:
miss = np.empty((tofill,columns), dtype='|S11')
miss[:] = empty
what = np.append(np.reshape(what, (-1,1)), miss, 1)
data = np.append(data, what, 0)
###########################################################################
# save data to txt file
np.savetxt('%s'%outfile, data, fmt='%s', delimiter=delimiter)
def spikeflag(date, data, inflag, isday, outdir, window=13, iter=1,
fill_days=1, t_int=48, z=7, deriv=0, udef=-9999, spike_v=2,
plot=False):
'''
Spike detection for Eddy Covariance data (and basically all other data)
using a moving median absolute difference filter. Multiple iterations
possible. Originally coded by Tino Rau.
Definition
----------
spikeflag(date, data, inflag, isday, window=13, iter=1,
fill_days=1, t_int=48, z=5.5, deriv=0, udef=-9999, spike_v=2,
plot=False):
Input
-----
date np.array(N), julian date (used only for plotting)
data np.array(N,M), data array where spike detection is applied on
each column (M)
inflag np.array(N,M), dtype=int, quality flag of data, spike detection
is only applied where inflag=0, all other data is ignored
isday np.array(N), dtype=bool, True where it is day and False where
it is night
outdir path where plots are saved
Optional Input
--------------
window int, size of the moving window where mad is calculated in days
(default: 13)
iter int, how often the running window mad shall be applied
(default: 1)
fill_days int, number of days where mad is applied within moving window
(default: 1)
t_int int, number of data points within one day (default: 48)
z int/float, data is allowed to deviate maximum z standard
deviations from the median (default: 7)
deriv int, 0: Act on raw data; 1: use first derivatives;
2: use 2nd derivatives (default: 0)
udef int/float, missing value of data (default: -9999) NaN values are
excluded from computations anyhow.
spike_v int, spike value which shall be returned when a spike is
detected (default: 2)
plot bool, if True data and spikes are plotted (default: False)
Output
------
flag np.array(N), flag array where everything is 0 except where
spikes were detected, there it is spike_v.
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2014 Arndt Piayda
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written, AP, Aug 2014
'''
rows, cols = np.shape(data)
flag = np.zeros_like(inflag).astype(np.int)
# mad window length and flag window length
period = np.int(window*t_int)/2
fill_win = np.int(fill_days*t_int)/2
# calculate dusk and dawn times and separate in day and night
isdawn = np.zeros(rows,dtype=np.bool)
isdusk = np.zeros(rows,dtype=np.bool)
dis = (isday.astype(int) - np.roll(isday,-1).astype(int)).astype(bool)
isdawn[:-1] = np.where(dis[:-1] == -1, True, False)
isdusk[:-1] = np.where(dis[:-1] == 1, True, False)
isddday = isdawn
tmp = np.roll(isdusk,1)
isddday[1:] += tmp[1:]
isddnight = isdusk
tmp = np.roll(isdawn,1)
isddnight[1:] += tmp[1:]
# iterate over each column of data
for col in xrange(cols):
# iterate as much as iter
for i in xrange(iter):
# get day and night data#
day_data = np.where((isday | isddday) & (inflag[:,col]==0) &
((data[:,col]!=udef) | (~np.isnan(data[:,col]))),
data[:,col], np.nan)
night_data = np.where((~isday | isddnight) & (inflag[:,col]==0) &
((data[:,col]!=udef) | (~np.isnan(data[:,col]))),
data[:,col], np.nan)
# iterate over flag window
fill_points = xrange(fill_win, isday.size-1, 2*fill_win)
for j in fill_points:
j1 = np.max([ j - period - 1,0])
j2 = np.min([ j + period + 1,isday.size])
fill_start = np.max([ j - fill_win,1])
fill_end = np.min([ j + fill_win,isday.size-1])
day_flag = mad(np.ma.masked_array(data=day_data[j1:j2],
mask=(np.isnan(day_data[j1:j2]))),
z=z, deriv=deriv)
flag[fill_start:fill_end,col] += np.where(day_flag[fill_start-j1-1:fill_end-j1-1],
spike_v, 0)
night_flag = mad(np.ma.masked_array(data=night_data[j1:j2],
mask=(np.isnan(night_data[j1:j2]))),
z=z, deriv=deriv)
flag[fill_start:fill_end,col] += np.where(night_flag[fill_start-j1-1:fill_end-j1-1],
spike_v, 0)
if plot:
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.backends.backend_pdf as pdf
majticks = mpl.dates.MonthLocator(bymonthday=1)
format_str='%d %m %Y %H:%M'
date01 = date2dec(yr=1, mo=1, dy=2, hr=0, mi=0, sc=0)
fig1 = plt.figure(1)
sub1 = fig1.add_subplot(111)
valid = (inflag[:,col]==0) & ((data[:,col]!=udef) |
(~np.isnan(data[:,col])))
l1 =sub1.plot(date[valid]-date01, data[valid,col], '-b')
l2 =sub1.plot(date[flag[:,col]!=0]-date01, data[flag[:,col]!=0,col], 'or')
sub1.xaxis.set_major_locator(majticks)
sub1.xaxis.set_major_formatter(mpl.dates.DateFormatter(format_str))
fig1.autofmt_xdate()
plt.show()
pp1 = pdf.PdfPages(outdir+'/spike_%i.pdf'%col)
fig1.savefig(pp1, format='pdf')
pp1.close()
return flag