def profile_limits(tohpa=False, simple_names=False):
""" Get RTTOV v11 Profile Limits
Pressure (hPa)
Temperature min (K)
Temperature max (K)
Water vapour min (ppmv) -> topa > hPa
Water vapour max (ppmv)
Ozone min (ppmv)
Ozone max (ppmv)
CO2 min (ppmv)
CO2 max (ppmv)
Returns
-------
rttov table
"""
from raso.met import ppmv2pa
from raso.config import _profile_limits as data
# data = pd.read_csv(get_data('rttov_54L_limits.csv'))
if data is None:
raise RuntimeError('No RTTOV Profile limits present. Check Module data')
data = data.copy()
if tohpa:
for ivar in data.columns[data.columns.str.contains('ppmv')].tolist():
data.loc[:, ivar] = ppmv2pa(data[ivar], data[u'Pressure (hPa)']) # ppmv to pa
data.columns = data.columns.str.replace('ppmv', 'hPa') # rename
if simple_names:
data.columns = ['p', 'tmin', 'tmax', 'vpmin', 'vpmax', 'omin', 'omax', 'co2min', 'co2max']
return data
def flag(data, iflag, fillvalue, verbose=1):
""" Set certain quality flagged values to the fillvalue
Parameters
----------
data DataFrame Input Radiosonde Database style
flags str Quality iflag
fillvalue value
verbose int
Returns
-------
data
"""
var_to_flag = {'T': 't', 'R': 'r', 'Q': 'q', 'P': 'p', 'D': 'dpd', 'V': 'vp', 'Y': 'td', '3': 'dpd'}
if iflag not in var_to_flag.keys():
print "FLAGS: ", var_to_flag.keys()
print "VARS : ", var_to_flag.values()
raise ValueError("Flag not standard ! %s" % iflag)
print_verbose("Changing flagged values (%s) in %s to : %s" % (iflag, var_to_flag[iflag], str(fillvalue)))
if hasnames(data, 'qual'):
data = data.copy() # Make sure we make a copy
logic = flag_inside(data.qual, iflag)
ivar = var_to_flag[iflag]
data[ivar] = np.where(logic, fillvalue, data[ivar].values)
data['qual'] = np.where(logic, data.qual.replace(iflag, '').values, data.qual.values) # remove FLAG
print_verbose("[QR] %s : %d" % (iflag, np.sum(np.sum(logic))), verbose)
return data
def enforcer(data, method='murphy_koop'):
""" Acts on quality flags
Td > T -> T, DPD=0, VP(T)
Does not handle dpd30 issue
Parameters
----------
data
method
Returns
-------
copy of data
Notes
-----
Looks for Y, D, V, T, R and Q Flags
"""
from met import esat_functions
if not isinstance(data, (pd.DataFrame, pd.Panel)):
raise ValueError("Requires a DataFrame or Panel")
if hasnames(data, 'qual'):
data = data.copy() # copy
vpfunc = getattr(esat_functions, method)
# Saturation
if hasnames(data, ['t', 'td', 'vp', 'dpd']):
logic = (data['td'].values > data['t'].values) # BAD, GOOD
data['dpd'] = np.where(logic, 0, data['dpd'].values) # set dpd to small value
data['td'] = np.where(logic, data['t'].values, data['td'].values) # set Td to small value
data['vp'] = np.where(logic, vpfunc(data['t'].values), data['vp'].values) # set vp
# Flagged because of T-Td diff
data['qual'] = np.where(logic, data.qual.replace('Y', '').values, data.qual.values) # remove FLAG: Y
data['qual'] = np.where(logic, data.qual.replace('D', '').values, data.qual.values) # remove FLAG: D
data['qual'] = np.where(logic, data.qual.replace('V', '').values, data.qual.values) # remove FLAG: V
data['t'] = np.where(flag_inside(data.qual, 'T'), np.nan, data.t.values) # Apply? BAD, GOOD
data['dpd'] = np.where(flag_inside(data.qual, 'D'), np.nan, data.dpd.values) # Apply? BAD, GOOD
data['vp'] = np.where(flag_inside(data.qual, 'V'), np.nan, data.vp.values) # Apply? BAD, GOOD
if hasnames(data, 'r'):
data['r'] = np.where(flag_inside(data.qual, 'R'), np.nan, data.r.values) # Apply? BAD, GOOD
if hasnames(data, 'q'):
data['q'] = np.where(flag_inside(data.qual, 'Q'), np.nan, data.q.values) # Apply? BAD, GOOD
return data
def control(data, tvar='t', dpdvar='dpd', vpvar='vp', rvar='r', qvar='q', tdvar='td', addlimits=False, replace=False,
report=None, verbose=0, **kwargs):
""" Flag data that does not fit into defined ranges and set to missing (apply_correction)
Parameters
----------
data DataFrame Input Radiosonde Database style
addlimits bool
replace bool
verbose int
kwargs dict
Returns
-------
data
"""
############################################################################
funcid = "[QC] "
if not isinstance(data, (pd.DataFrame, pd.Panel)):
raise ValueError(funcid + "Requires a pandas DataFrame (Database style) or Panel")
remove_pressure = False
if not hasnames(data, 'p'):
if isinstance(data, pd.Panel):
data['p'] = 0. # add minor_axis as p
data.loc['p', :, :] = np.asarray(data.minor_axis)[np.newaxis, np.newaxis, :]
remove_pressure = True
else:
raise ValueError("Require p in data")
###################
q_absmin = 0
q_absmax = 1
r_absmin = 0
r_absmax = 1
dpd_absmin = 0
dpd_absmax = 60
###################
rt = profile_limits(tohpa=True, simple_names=True)
rt['p'] *= 100. # hPa to Pa
rt['vpmin'] *= 100. # hPa to Pa
rt['vpmax'] *= 100. # hPa to Pa
###################
data = data.copy()
if not hasnames(data, 'qual'):
data['qual'] = '' # Quality Column
journal(funcid+"Params: %s (R: %s)" % (str(data.shape), replace), report, verbose)
############################################################################
#
# Pressure
#
# Valid Range: ]0, - ]
logic = (data.p.values < 0 & np.isfinite(data.p.values)) # Range? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'P', data.qual.replace('P', '').values) # FLAG: P
if replace:
data['p'] = np.where(logic, np.nan, data.p.values) # Apply? BAD, GOOD
journal(funcid + '#%8d P flagged.' % (np.sum(np.sum(logic))), report, verbose)
#############################################################################
#
#
# T
# Valid Range: [183.15, 333.15] K
if hasnames(data, tvar):
tmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.tmin.values, left=rt.tmin.min(),
right=rt.tmin.max()) # Interpolate Minimum
tmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.tmax.values, left=rt.tmax.min(),
right=rt.tmax.max()) # Interpolate Maximum
logic = ((data[tvar].values < tmins) | (data[tvar].values > tmaxs)) & np.isfinite(data[tvar].values) # Range ? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'T', data.qual.replace('T', '').values) # FLAG: T
if replace:
data[tvar] = np.where(logic, np.nan, data[tvar].values) # Apply? BAD, GOOD
if addlimits:
data['t_min'] = np.where(logic, tmins, np.nan)
data['t_max'] = np.where(logic, tmaxs, np.nan)
journal(funcid + '#%8d T flagged.' % (np.sum(np.sum(logic))), report, verbose)
#############################################################################
#
#
# Q
# Valid Range: [0, 1] kg/kg
if hasnames(data, qvar):
logic = ((data[qvar].values < q_absmin) | (data[qvar].values > q_absmax)) & np.isfinite(data[qvar].values) # Range? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'Q', data.qual.replace('Q', '').values) # FLAG: Q
if replace:
data[qvar] = np.where(logic, np.nan, data[qvar].values) # Apply? BAD, GOOD
journal(funcid + '#%8d Q flagged.' % (np.sum(np.sum(logic))), report, verbose)
#############################################################################
#
#
# R
# Valid Range: [ 0; 1] ratio
if hasnames(data, rvar):
logic = ((data[rvar].values < r_absmin) | (data[rvar].values > r_absmax)) & np.isfinite(data[rvar].values) # Range? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'R', data.qual.replace('R', '').values) # FLAG: R
if replace:
data[rvar] = np.where(logic, np.nan, data[rvar].values) # Apply? BAD, GOOD
journal(funcid + '#%8d R flagged.' % (np.sum(np.sum(logic))), report, verbose)
#############################################################################
#
#
# DPD
# Valid Range: [0, 60] K
if hasnames(data, dpdvar):
logic = ((data[dpdvar].values < dpd_absmin) | (data[dpdvar].values > dpd_absmax)) & np.isfinite(data[dpdvar].values) # Range ? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'D', data.qual.replace('D', '').values) # FLAG: D
if replace:
data[dpdvar] = np.where(logic, np.nan, data[dpdvar].values) # Apply? BAD, GOOD
journal(funcid + '#%8d DPD flagged.' % (np.sum(np.sum(logic))), report, verbose)
#############################################################################
#
#
# Td
# Valid Range: [183 - 333] K & < T
if hasnames(data, tdvar):
# raises warning for value not valid for less comparison
logic = ((data[tdvar].values < (rt.tmin.min() - dpd_absmax)) | (data[tdvar].values > rt.tmax.max()) | (
data[tdvar].values > data[tvar].values)) & np.isfinite(data[tdvar].values - data[tvar].values) # Range? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'Y', data.qual.replace('Y', '').values) # FLAG: Y
if replace:
data[tdvar] = np.where(logic, np.nan, data[tdvar].values) # Apply? BAD, GOOD
journal(funcid + '#%8d Td>T flagged.' % (np.sum(np.sum(logic))), report, verbose)
#############################################################################
#
#
# VP
# Valid Range: [0, 20000] Pa
if hasnames(data, vpvar):
vpmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmin.values, left=rt.vpmin.min(),
right=rt.vpmin.max()) # Interpolate Minimum
vpmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmax.values, left=rt.vpmax.min(),
right=rt.vpmax.max()) # Interpolate Maximum
# Range? BAD, GOOD
logic = ((data[vpvar].values < vpmins) | (data[vpvar].values > vpmaxs)) & np.isfinite(data[vpvar].values)
data['qual'] = np.where(logic, data.qual.values + 'V', data.qual.replace('V', '').values) # FLAG: V
if replace:
data[vpvar] = np.where(logic, np.nan, data[vpvar].values) # Apply? BAD, GOOD
if addlimits:
data['vp_min'] = np.where(logic, vpmins, np.nan)
data['vp_max'] = np.where(logic, vpmaxs, np.nan)
# D and Y FLAGS ?
journal(funcid + '#%8d VP flagged.' % (np.sum(np.sum(logic))), report, verbose)
############################################################################
# Check for duplicate Flags
data['qual'] = unique_flags(data['qual'])
if remove_pressure:
del data['p']
return data