def standard_water_vapor(data, tvar='t', dpdvar='dpd', vpvar='vp', rvar='r', qvar='q', update=False, replace=False,
method='murphy_koop', inplace=False, report=None, verbose=0, **kwargs):
""" Convert humidity variables to water vapor pressure
1. RH (Esat)
2. Q (P)
3. DPD (Esat)
--> VP
Limits are from the RTTOV Coefficient file table 54 levels
Notes
-----
http://nwpsaf.eu/oldsite/deliverables/rtm/rttov11_coefficients.html#54L_reg_limits
Parameters
----------
data DataFrame Input database
tvar str
dpdvar str
vpvar str
rvar str
qvar str
update bool
esat str
inplace bool
verbose int
kwargs **
Returns
-------
data
"""
from raso.met.conversion import sh2vap
from raso.met import esat_functions
from raso.qc import profile_limits
############################################################################
rt = profile_limits(tohpa=True, simple_names=True) # RTTOV Variable Limits
rt['p'] *= 100. # hPa to Pa
rt['vpmin'] *= 100. # hPa to Pa
rt['vpmax'] *= 100. # hPa to Pa
############################################################################
funcid = "[SV] "
if not isinstance(data, (pd.DataFrame, pd.Panel)):
raise ValueError(funcid + "Requires a Dataframe or Panel")
if not inplace:
data = data.copy()
if not hasnames(data, 'qual'):
data['qual'] = ''
if hasnames(data, [rvar, qvar, dpdvar], value=0):
raise ValueError(funcid + " Requires a humidity variable: r, q or dpd")
if hasnames(data, tvar, value=0):
raise ValueError(funcid + " Requires variable: t")
vpfunc = getattr(esat_functions, method)
if not hasnames(data, vpvar) or update:
remove_pressure = False
if isinstance(data, pd.Panel):
data['p'] = 0. # add minor_axis as p
data.loc['p', :, :] = np.asarray(data.minor_axis)[np.newaxis, np.newaxis, :]
remove_pressure = True
ndates = data.shape[1] * data.shape[2]
else:
ndates = data.shape[0]
############################################################################
# First use R, which includes Q
# R
#
if hasnames(data, rvar):
journal(funcid + "Using r (%s) and t (%s) for vp (%s)" % (rvar, tvar, vpvar), report, verbose)
data[vpvar] = data[rvar].values * vpfunc(data[tvar].values) # Convert r,t to vp
############################################################################
#
# Q
#
elif hasnames(qvar, data):
journal(funcid + "Using q (%s) and p to fill up gaps for vp (%s)" % (qvar, vpvar), report, verbose)
vp = sh2vap(data[qvar].values, data['p'].values) # Convert q,p to vp
if hasnames(data, vpvar):
logic = (np.isfinite(vp) & ~np.isfinite(data[vpvar].values))
data[vpvar] = np.where(logic, vp, data[vpvar].values)
# TODO missing else
# data.loc[:, vpvar] = np.where(logic, vp, data[vpvar].values)
# data.loc[:, 'qual'] = np.where(logic, data.qual.values + 'Q', data.qual.values) # FLAG: F fill
############################################################################
#
# Second use DPD (IGRA)
# DPD
elif hasnames(data, dpdvar):
journal(funcid + "Warning using dpd (%s) for vp (%s)" % (dpdvar, vpvar), report, verbose)
vp = vpfunc((data[tvar] - data[dpdvar]).values) # Convert Td to vp
if hasnames(data, vpvar):
logic = (np.isfinite(vp) & ~np.isfinite(data[vpvar].values)) # Update? GOOD, BAD
data[vpvar] = np.where(logic, vp, data[vpvar].values) # Update? NEW, OLD
else:
data[vpvar] = vp
else:
raise RuntimeError("No humidity variable found!")
############################################################################
#
# Quality control
#
vpmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmin.values, left=rt.vpmin.min(),
right=rt.vpmin.max()) # Interpolate Minimum
vpmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmax.values, left=rt.vpmax.min(),
right=rt.vpmax.max()) # Interpolate Maximum
# Range? BAD, GOOD
logic = ((data[vpvar].values < vpmins) | (data[vpvar].values > vpmaxs)) & np.isfinite(data[vpvar].values)
data['qual'] = np.where(logic, data.qual.values + 'V', data.qual.replace('V', '').values) # FLAG: V
# data.loc[:, 'qual'] = np.where(logic, data.qual.values + 'V', data.qual.replace('V', '').values) # FLAG: V
journal(funcid + "#%8d V flagged. (%d)" % (np.sum(np.sum(flag_inside(data.qual, 'V'))), ndates), report,
verbose)
if replace:
data[vpvar] = np.where(logic, np.nan, data[vpvar].values) # Apply? BAD, GOOD
# data.loc[:, vpvar] = np.where(logic, np.nan, data[vpvar].values) # Apply? BAD, GOOD
############################################################################
#
# Unique Flags
#
data['qual'] = unique_flags(data['qual'])
############################################################################
#
# Pressure
#
if remove_pressure:
del data['p']
if not inplace:
return data
def from_era(ident, alt=0, lon=0, lat=0, savename='extract_ERA', verbose=0):
""" Updated Version of profiles and surface
# Can read ERA data from retrieved and converted MARS Requests
ERADIR
Parameters
----------
ident str/int/float Radiosonde ID
alt int/float Radiosonde Station Altitude
lon int/float Radiosonde Station Longitude
lat int/float Radiosonde Station Latitude
savename str Extract savename
verbose int verboseness
Returns
-------
"""
from raso.support import print_verbose, panel_to_database
from raso.met.conversion import sh2ppmv, sh2vap, bolton_inverse, vap2sh
from raso.met.esat_functions import foeewm
from raso.config import rasodir # eradir
from raso.qc import profile_limits
# read new era file
if isinstance(ident, (int, float)):
ident = "%06d" % ident
filename = rasodir + "/%s/%s.h5" % (ident, savename)
cnames = ['t2m', 'q2m', 'msl', 'u10', 'v10', 'fetch', 'skt', 'FASTEM_1', 'FASTEM_2', 'FASTEM_3',
'FASTEM_4', 'FASTEM_5', 'surf_type', 'water_type', 'elev', 'lat', 'lon', 'sat_zen', 'sat_azi',
'sol_zen', 'sol_azi', 'cltp', 'clf']
surface = {} # pd.DataFrame(columns=cnames)
era = {}
if not os.path.isfile(filename):
raise IOError('File not found: %s' % filename)
with pd.HDFStore(filename, 'r') as store:
for ikey in store.keys():
iname = ikey.replace('/', '')
if ikey[1:] in ['t', 'q', 'o3']:
# profile data
tmp = store[ikey]
tmp.index.name = 'date'
tmp.columns.name = 'p'
# tmp = tmp.reset_index().drop_duplicates('date').set_index('date') # remove duplicates
era[iname] = tmp # DataFrame (time x p)
else:
# surface data
if iname == 'sp':
surface['msl'] = store[ikey][0] / 100. # [dates] / 100. # hPa
elif iname == 'ci':
# sea ice cover
surface['surf_type'] = store[ikey][0]
elif iname == 'u10':
surface['u10'] = store[ikey][0]
elif iname == 'v10':
surface['v10'] = store[ikey][0]
else:
surface[iname] = store[ikey][0]
era = pd.Panel(era)
era = panel_to_database(era, remove=False)
# ERA can have negative Q values !?
qerror = era.query('q<=0').q.count()
print_verbose("[%s] Found %d negative Q-values!" % (ident, qerror), verbose)
# Interpolate limits
rt = profile_limits(tohpa=True, simple_names=True)
rt['qmin'] = vap2sh(rt.vpmin.values, rt.p.values) # both missing hPa factor
qmins = np.interp(np.log(era.p.values), np.log(rt.p.values*100.), rt.qmin.values, left=rt.qmin.min(), right=rt.qmin.max()) # Interpolate Minimum Q
era['q'] = np.where(era.q < qmins, qmins, era.q) # replace with RTTOV limit
# Make a dataframe
surface = pd.DataFrame(surface)
surface['q2m'] = sh2ppmv(era.query('p==100000')['q'], 100000.) # ppmv
surface['fetch'] = 100000. # m
# FASTEM Surface model Parameters:
surface['FASTEM_1'] = 3.0
surface['FASTEM_2'] = 5.0
surface['FASTEM_3'] = 15.0
surface['FASTEM_4'] = 0.1
surface['FASTEM_5'] = 0.3
# surface properties
if 'surf_type' in surface.columns:
if surface['surf_type'].isnull().any(0):
if surface['surf_type'].count() > 0:
surface['surf_type'].fillna(method='bfill') # backwards fill
surface['surf_type'].fillna(method='ffill') # forward fill
else:
surface['surf_type'] = 0. # surface type (0=land, 1=sea, 2=sea-ice)
else:
surface['surf_type'] = 0. # surface type (0=land, 1=sea, 2=sea-ice)
surface['water_type'] = 0. # water type (0=fresh, 1=ocean)
surface['elev'] = alt / 1000. # ELEVATION km
surface['lat'] = lat # LAT
surface['lon'] = lon # LON
surface['sat_azi'] = 0. # Satellite Azimuth Angle
surface['sat_zen'] = 53.1 # OSCAR , SSMIS
surface['sol_zen'] = 0. # Solar Zenith Angle
surface['sol_azi'] = 0. # Solar Azimuth Angle
surface['cltp'] = 500. # Cloud top pressure
surface['clf'] = 0. # Cloud fraction
# right order
surface = surface[cnames].copy()
# CONVERT TO WATER VAPOR PRESSURE in Pa
era['vp'] = sh2vap(era['q'], era['p']) # formula
era['td'] = bolton_inverse(era.vp.values) # default method from Murphy and Koop (iterative)
logic = era['td'] > era['t'] # impossible
era.loc[:, 'td'] = np.where(logic, era['t'].values, era['td'].values)
era.loc[:, 'vp'] = np.where(logic, foeewm(era['t'].values), era['vp'].values) # ? fixes some problems ?
return surface, era
def standard_rel_humidity(data, rvar='r', tvar='t', dpdvar='dpd', qvar='q', update=False, replace=False,
method='murphy_koop', inplace=False, report=None, verbose=0):
""" convert humidity variables to relative humidity
1. q to vp
2. dpd to vp
3. merge vp
4. vp to rh
set quality flags
Parameters
----------
data DataFrame Input database
rvar str
tvar str
dpdvar str
qvar str
update bool Update existing
replace bool set flagged to nan
method str Saturation water vapor
inplace bool
verbose int
kwargs **
Returns
-------
data
"""
from raso.met.conversion import sh2vap
from raso.met import esat_functions
from raso.qc import profile_limits
############################################################################
rt = profile_limits(tohpa=True, simple_names=True) # RTTOV Variable Limits
rt['p'] *= 100. # hPa to Pa
rt['vpmin'] *= 100. # hPa to Pa
rt['vpmax'] *= 100. # hPa to Pa
############################################################################
funcid = "[SV] "
r_absmin = 0
r_absmax = 1
if not isinstance(data, (pd.DataFrame, pd.Panel)):
raise ValueError(funcid + "Requires a Dataframe or Panel")
if not inplace:
data = data.copy()
if not hasnames('qual', data):
data['qual'] = ''
if hasnames(data, [rvar, qvar, dpdvar], value=0):
raise ValueError(funcid + " Requires a humidity variable: %s, %s or %s" % (rvar, qvar, dpdvar))
if hasnames(data,tvar, value=0):
raise ValueError(funcid + " Requires variable: %s" % tvar)
vpfunc = getattr(esat_functions, method)
if not hasnames(data, rvar) or update:
remove_pressure = False
if isinstance(data, pd.Panel):
data['p'] = 0. # add minor_axis as p
data.loc['p', :, :] = np.asarray(data.minor_axis)[np.newaxis, np.newaxis, :]
remove_pressure = True
journal(funcid + "rel. humidity (%s) update: %s" % (rvar, update), report, verbose)
# Quality control
vpmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmin.values, left=rt.vpmin.min(),
right=rt.vpmin.max()) # Interpolate Minimum
vpmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmax.values, left=rt.vpmax.min(),
right=rt.vpmax.max()) # Interpolate Maximum
vpsat = vpfunc(data[tvar].values)
if hasnames(data, rvar):
vp = data[rvar].values * vpsat # Convert r,t to vp
logic = ((vp < vpmins) | (vp > vpmaxs)) & np.isfinite(vp)
data['qual'] = np.where(logic, data.qual.values + 'R', data.qual.replace('R', '').values) # FLAG: R (?)
journal(funcid + "rel. humidity (%s) available (replace: %s)" % (rvar, replace), report, verbose)
if replace:
vp = np.where(logic, np.nan, vp) # Apply? BAD, GOOD
else:
vp = np.full(data[tvar].shape, np.nan)
if hasnames(data, qvar):
qvp = sh2vap(data[qvar].values, data['p'].values) # only a formula no approximation
logic = ((qvp < vpmins) | (qvp > vpmaxs)) & np.isfinite(qvp)
data['qual'] = np.where(logic, data.qual.values + 'Q', data.qual.replace('Q', '').values) # FLAG: R (?)
journal(funcid + "spec. humidity (%s) available (replace: %s)" % (qvar, replace), report, verbose)
if replace:
qvp = np.where(logic, np.nan, qvp) # Apply? BAD, GOOD
# Fill Gaps
logic = (np.isfinite(qvp) & (~np.isfinite(vp))) # GOOD, BAD
vp = np.where(logic, qvp, vp) # UPDATE, OLD
if hasnames(data, dpdvar):
dvp = vpfunc((data[tvar] - data[dpdvar])) # Dewpoint -> vp
logic = ((dvp < vpmins) | (dvp > vpmaxs)) & np.isfinite(dvp)
data['qual'] = np.where(logic, data.qual.values + 'D', data.qual.replace('D', '').values) # FLAG: R (?)
journal(funcid + "DPD (%s) available (replace: %s)" % (dpdvar, replace), report, verbose)
if replace:
dvp = np.where(logic, np.nan, dvp) # Apply? BAD, GOOD
# fill gaps
logic = (np.isfinite(dvp) & (~np.isfinite(vp))) # GOOD, BAD
vp = np.where(logic, dvp, vp) # UPDATE, OLD
############################################################################
#
# Convert VP to RH
#
data[rvar] = vp / vpsat # rel. Humidity
logic = ((data[rvar].values < r_absmin) | (data[rvar].values > r_absmax)) & np.isfinite(
data[rvar].values) # Range? BAD, GOOD
data['qual'] = np.where(logic, data.qual.values + 'R', data.qual.replace('R', '').values) # FLAG: R
if replace:
data[rvar] = np.where(logic, np.nan, data.r.values) # Apply? BAD, GOOD
############################################################################
#
# Unique Flags
#
data['qual'] = unique_flags(data['qual'])
############################################################################
#
# Pressure
#
if remove_pressure:
del data['p']
if not inplace:
return data
