def profile_limits(tohpa=False, simple_names=False): """ Get RTTOV v11 Profile Limits Pressure (hPa) Temperature min (K) Temperature max (K) Water vapour min (ppmv) -> topa > hPa Water vapour max (ppmv) Ozone min (ppmv) Ozone max (ppmv) CO2 min (ppmv) CO2 max (ppmv) Returns ------- rttov table """ from raso.met import ppmv2pa from raso.config import _profile_limits as data # data = pd.read_csv(get_data('rttov_54L_limits.csv')) if data is None: raise RuntimeError('No RTTOV Profile limits present. Check Module data') data = data.copy() if tohpa: for ivar in data.columns[data.columns.str.contains('ppmv')].tolist(): data.loc[:, ivar] = ppmv2pa(data[ivar], data[u'Pressure (hPa)']) # ppmv to pa data.columns = data.columns.str.replace('ppmv', 'hPa') # rename if simple_names: data.columns = ['p', 'tmin', 'tmax', 'vpmin', 'vpmax', 'omin', 'omax', 'co2min', 'co2max'] return data
def flag(data, iflag, fillvalue, verbose=1): """ Set certain quality flagged values to the fillvalue Parameters ---------- data DataFrame Input Radiosonde Database style flags str Quality iflag fillvalue value verbose int Returns ------- data """ var_to_flag = {'T': 't', 'R': 'r', 'Q': 'q', 'P': 'p', 'D': 'dpd', 'V': 'vp', 'Y': 'td', '3': 'dpd'} if iflag not in var_to_flag.keys(): print "FLAGS: ", var_to_flag.keys() print "VARS : ", var_to_flag.values() raise ValueError("Flag not standard ! %s" % iflag) print_verbose("Changing flagged values (%s) in %s to : %s" % (iflag, var_to_flag[iflag], str(fillvalue))) if hasnames(data, 'qual'): data = data.copy() # Make sure we make a copy logic = flag_inside(data.qual, iflag) ivar = var_to_flag[iflag] data[ivar] = np.where(logic, fillvalue, data[ivar].values) data['qual'] = np.where(logic, data.qual.replace(iflag, '').values, data.qual.values) # remove FLAG print_verbose("[QR] %s : %d" % (iflag, np.sum(np.sum(logic))), verbose) return data
def enforcer(data, method='murphy_koop'): """ Acts on quality flags Td > T -> T, DPD=0, VP(T) Does not handle dpd30 issue Parameters ---------- data method Returns ------- copy of data Notes ----- Looks for Y, D, V, T, R and Q Flags """ from met import esat_functions if not isinstance(data, (pd.DataFrame, pd.Panel)): raise ValueError("Requires a DataFrame or Panel") if hasnames(data, 'qual'): data = data.copy() # copy vpfunc = getattr(esat_functions, method) # Saturation if hasnames(data, ['t', 'td', 'vp', 'dpd']): logic = (data['td'].values > data['t'].values) # BAD, GOOD data['dpd'] = np.where(logic, 0, data['dpd'].values) # set dpd to small value data['td'] = np.where(logic, data['t'].values, data['td'].values) # set Td to small value data['vp'] = np.where(logic, vpfunc(data['t'].values), data['vp'].values) # set vp # Flagged because of T-Td diff data['qual'] = np.where(logic, data.qual.replace('Y', '').values, data.qual.values) # remove FLAG: Y data['qual'] = np.where(logic, data.qual.replace('D', '').values, data.qual.values) # remove FLAG: D data['qual'] = np.where(logic, data.qual.replace('V', '').values, data.qual.values) # remove FLAG: V data['t'] = np.where(flag_inside(data.qual, 'T'), np.nan, data.t.values) # Apply? BAD, GOOD data['dpd'] = np.where(flag_inside(data.qual, 'D'), np.nan, data.dpd.values) # Apply? BAD, GOOD data['vp'] = np.where(flag_inside(data.qual, 'V'), np.nan, data.vp.values) # Apply? BAD, GOOD if hasnames(data, 'r'): data['r'] = np.where(flag_inside(data.qual, 'R'), np.nan, data.r.values) # Apply? BAD, GOOD if hasnames(data, 'q'): data['q'] = np.where(flag_inside(data.qual, 'Q'), np.nan, data.q.values) # Apply? BAD, GOOD return data
def control(data, tvar='t', dpdvar='dpd', vpvar='vp', rvar='r', qvar='q', tdvar='td', addlimits=False, replace=False, report=None, verbose=0, **kwargs): """ Flag data that does not fit into defined ranges and set to missing (apply_correction) Parameters ---------- data DataFrame Input Radiosonde Database style addlimits bool replace bool verbose int kwargs dict Returns ------- data """ ############################################################################ funcid = "[QC] " if not isinstance(data, (pd.DataFrame, pd.Panel)): raise ValueError(funcid + "Requires a pandas DataFrame (Database style) or Panel") remove_pressure = False if not hasnames(data, 'p'): if isinstance(data, pd.Panel): data['p'] = 0. # add minor_axis as p data.loc['p', :, :] = np.asarray(data.minor_axis)[np.newaxis, np.newaxis, :] remove_pressure = True else: raise ValueError("Require p in data") ################### q_absmin = 0 q_absmax = 1 r_absmin = 0 r_absmax = 1 dpd_absmin = 0 dpd_absmax = 60 ################### rt = profile_limits(tohpa=True, simple_names=True) rt['p'] *= 100. # hPa to Pa rt['vpmin'] *= 100. # hPa to Pa rt['vpmax'] *= 100. # hPa to Pa ################### data = data.copy() if not hasnames(data, 'qual'): data['qual'] = '' # Quality Column journal(funcid+"Params: %s (R: %s)" % (str(data.shape), replace), report, verbose) ############################################################################ # # Pressure # # Valid Range: ]0, - ] logic = (data.p.values < 0 & np.isfinite(data.p.values)) # Range? BAD, GOOD data['qual'] = np.where(logic, data.qual.values + 'P', data.qual.replace('P', '').values) # FLAG: P if replace: data['p'] = np.where(logic, np.nan, data.p.values) # Apply? BAD, GOOD journal(funcid + '#%8d P flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################# # # # T # Valid Range: [183.15, 333.15] K if hasnames(data, tvar): tmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.tmin.values, left=rt.tmin.min(), right=rt.tmin.max()) # Interpolate Minimum tmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.tmax.values, left=rt.tmax.min(), right=rt.tmax.max()) # Interpolate Maximum logic = ((data[tvar].values < tmins) | (data[tvar].values > tmaxs)) & np.isfinite(data[tvar].values) # Range ? BAD, GOOD data['qual'] = np.where(logic, data.qual.values + 'T', data.qual.replace('T', '').values) # FLAG: T if replace: data[tvar] = np.where(logic, np.nan, data[tvar].values) # Apply? BAD, GOOD if addlimits: data['t_min'] = np.where(logic, tmins, np.nan) data['t_max'] = np.where(logic, tmaxs, np.nan) journal(funcid + '#%8d T flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################# # # # Q # Valid Range: [0, 1] kg/kg if hasnames(data, qvar): logic = ((data[qvar].values < q_absmin) | (data[qvar].values > q_absmax)) & np.isfinite(data[qvar].values) # Range? BAD, GOOD data['qual'] = np.where(logic, data.qual.values + 'Q', data.qual.replace('Q', '').values) # FLAG: Q if replace: data[qvar] = np.where(logic, np.nan, data[qvar].values) # Apply? BAD, GOOD journal(funcid + '#%8d Q flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################# # # # R # Valid Range: [ 0; 1] ratio if hasnames(data, rvar): logic = ((data[rvar].values < r_absmin) | (data[rvar].values > r_absmax)) & np.isfinite(data[rvar].values) # Range? BAD, GOOD data['qual'] = np.where(logic, data.qual.values + 'R', data.qual.replace('R', '').values) # FLAG: R if replace: data[rvar] = np.where(logic, np.nan, data[rvar].values) # Apply? BAD, GOOD journal(funcid + '#%8d R flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################# # # # DPD # Valid Range: [0, 60] K if hasnames(data, dpdvar): logic = ((data[dpdvar].values < dpd_absmin) | (data[dpdvar].values > dpd_absmax)) & np.isfinite(data[dpdvar].values) # Range ? BAD, GOOD data['qual'] = np.where(logic, data.qual.values + 'D', data.qual.replace('D', '').values) # FLAG: D if replace: data[dpdvar] = np.where(logic, np.nan, data[dpdvar].values) # Apply? BAD, GOOD journal(funcid + '#%8d DPD flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################# # # # Td # Valid Range: [183 - 333] K & < T if hasnames(data, tdvar): # raises warning for value not valid for less comparison logic = ((data[tdvar].values < (rt.tmin.min() - dpd_absmax)) | (data[tdvar].values > rt.tmax.max()) | ( data[tdvar].values > data[tvar].values)) & np.isfinite(data[tdvar].values - data[tvar].values) # Range? BAD, GOOD data['qual'] = np.where(logic, data.qual.values + 'Y', data.qual.replace('Y', '').values) # FLAG: Y if replace: data[tdvar] = np.where(logic, np.nan, data[tdvar].values) # Apply? BAD, GOOD journal(funcid + '#%8d Td>T flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################# # # # VP # Valid Range: [0, 20000] Pa if hasnames(data, vpvar): vpmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmin.values, left=rt.vpmin.min(), right=rt.vpmin.max()) # Interpolate Minimum vpmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmax.values, left=rt.vpmax.min(), right=rt.vpmax.max()) # Interpolate Maximum # Range? BAD, GOOD logic = ((data[vpvar].values < vpmins) | (data[vpvar].values > vpmaxs)) & np.isfinite(data[vpvar].values) data['qual'] = np.where(logic, data.qual.values + 'V', data.qual.replace('V', '').values) # FLAG: V if replace: data[vpvar] = np.where(logic, np.nan, data[vpvar].values) # Apply? BAD, GOOD if addlimits: data['vp_min'] = np.where(logic, vpmins, np.nan) data['vp_max'] = np.where(logic, vpmaxs, np.nan) # D and Y FLAGS ? journal(funcid + '#%8d VP flagged.' % (np.sum(np.sum(logic))), report, verbose) ############################################################################ # Check for duplicate Flags data['qual'] = unique_flags(data['qual']) if remove_pressure: del data['p'] return data