예제 #1
0
def standard_water_vapor(data, tvar='t', dpdvar='dpd', vpvar='vp', rvar='r', qvar='q', update=False, replace=False,
                         method='murphy_koop', inplace=False, report=None, verbose=0, **kwargs):
    """ Convert humidity variables to water vapor pressure
    1. RH (Esat)
    2. Q  (P)
    3. DPD (Esat)
    --> VP

    Limits are from the RTTOV Coefficient file table 54 levels

    Notes
    -----
    http://nwpsaf.eu/oldsite/deliverables/rtm/rttov11_coefficients.html#54L_reg_limits

    Parameters
    ----------
    data        DataFrame       Input database
    tvar        str
    dpdvar      str
    vpvar       str
    rvar        str
    qvar        str
    update      bool
    esat        str
    inplace     bool
    verbose     int
    kwargs      **

    Returns
    -------
    data
    """
    from raso.met.conversion import sh2vap
    from raso.met import esat_functions
    from raso.qc import profile_limits
    ############################################################################
    rt = profile_limits(tohpa=True, simple_names=True)  # RTTOV Variable Limits
    rt['p'] *= 100.         # hPa to Pa
    rt['vpmin'] *= 100.     # hPa to Pa
    rt['vpmax'] *= 100.     # hPa to Pa
    ############################################################################
    funcid = "[SV] "

    if not isinstance(data, (pd.DataFrame, pd.Panel)):
        raise ValueError(funcid + "Requires a Dataframe or Panel")

    if not inplace:
        data = data.copy()

    if not hasnames(data, 'qual'):
        data['qual'] = ''

    if hasnames(data, [rvar, qvar, dpdvar], value=0):
        raise ValueError(funcid + " Requires a humidity variable: r, q or dpd")

    if hasnames(data, tvar, value=0):
        raise ValueError(funcid + " Requires variable: t")

    vpfunc = getattr(esat_functions, method)

    if not hasnames(data, vpvar) or update:
        remove_pressure = False
        if isinstance(data, pd.Panel):
            data['p'] = 0.  # add minor_axis as p
            data.loc['p', :, :] = np.asarray(data.minor_axis)[np.newaxis, np.newaxis, :]
            remove_pressure = True
            ndates = data.shape[1] * data.shape[2]
        else:
            ndates = data.shape[0]
        ############################################################################
        # First use R, which includes Q
        # R
        #
        if hasnames(data, rvar):
            journal(funcid + "Using r (%s) and t (%s) for vp (%s)" % (rvar, tvar, vpvar), report, verbose)
            data[vpvar] = data[rvar].values * vpfunc(data[tvar].values)  # Convert r,t to vp
        ############################################################################
        #
        # Q
        #
        elif hasnames(qvar, data):
            journal(funcid + "Using q (%s) and p to fill up gaps for vp (%s)" % (qvar, vpvar), report, verbose)
            vp = sh2vap(data[qvar].values, data['p'].values)  # Convert q,p to vp
            if hasnames(data, vpvar):
                logic = (np.isfinite(vp) & ~np.isfinite(data[vpvar].values))
                data[vpvar] = np.where(logic, vp, data[vpvar].values)
            # TODO missing else
                # data.loc[:, vpvar] = np.where(logic, vp, data[vpvar].values)
                # data.loc[:, 'qual'] = np.where(logic, data.qual.values + 'Q', data.qual.values)  # FLAG: F fill
        ############################################################################
        #
        # Second use DPD (IGRA)
        # DPD
        elif hasnames(data, dpdvar):
            journal(funcid + "Warning using dpd (%s) for vp (%s)" % (dpdvar, vpvar), report, verbose)
            vp = vpfunc((data[tvar] - data[dpdvar]).values)  # Convert Td to vp

            if hasnames(data, vpvar):
                logic = (np.isfinite(vp) & ~np.isfinite(data[vpvar].values))  # Update? GOOD, BAD
                data[vpvar] = np.where(logic, vp, data[vpvar].values)  # Update? NEW, OLD

            else:
                data[vpvar] = vp
        else:
            raise RuntimeError("No humidity variable found!")
        ############################################################################
        #
        # Quality control
        #
        vpmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmin.values, left=rt.vpmin.min(),
                           right=rt.vpmin.max())  # Interpolate Minimum
        vpmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmax.values, left=rt.vpmax.min(),
                           right=rt.vpmax.max())  # Interpolate Maximum
        # Range? BAD, GOOD
        logic = ((data[vpvar].values < vpmins) | (data[vpvar].values > vpmaxs)) & np.isfinite(data[vpvar].values)
        data['qual'] = np.where(logic, data.qual.values + 'V', data.qual.replace('V', '').values)  # FLAG: V
        # data.loc[:, 'qual'] = np.where(logic, data.qual.values + 'V', data.qual.replace('V', '').values)  # FLAG: V
        journal(funcid + "#%8d V flagged. (%d)" % (np.sum(np.sum(flag_inside(data.qual, 'V'))), ndates), report,
                verbose)

        if replace:
            data[vpvar] = np.where(logic, np.nan, data[vpvar].values)  # Apply? BAD, GOOD
            # data.loc[:, vpvar] = np.where(logic, np.nan, data[vpvar].values)  # Apply? BAD, GOOD
        ############################################################################
        #
        # Unique Flags
        #
        data['qual'] = unique_flags(data['qual'])
        ############################################################################
        #
        # Pressure
        #
        if remove_pressure:
            del data['p']

    if not inplace:
        return data
예제 #2
0
def from_era(ident, alt=0, lon=0, lat=0, savename='extract_ERA', verbose=0):
    """ Updated Version of profiles and surface
    # Can read ERA data from retrieved and converted MARS Requests

    ERADIR

    Parameters
    ----------
    ident       str/int/float   Radiosonde ID
    alt         int/float       Radiosonde Station Altitude
    lon         int/float       Radiosonde Station Longitude
    lat         int/float       Radiosonde Station Latitude
    savename    str             Extract savename
    verbose     int             verboseness

    Returns
    -------

    """
    from raso.support import print_verbose, panel_to_database
    from raso.met.conversion import sh2ppmv, sh2vap, bolton_inverse, vap2sh
    from raso.met.esat_functions import foeewm
    from raso.config import rasodir # eradir
    from raso.qc import profile_limits
    # read new era file
    if isinstance(ident, (int, float)):
        ident = "%06d" % ident

    filename = rasodir + "/%s/%s.h5" % (ident, savename)

    cnames = ['t2m', 'q2m', 'msl', 'u10', 'v10', 'fetch', 'skt', 'FASTEM_1', 'FASTEM_2', 'FASTEM_3',
              'FASTEM_4', 'FASTEM_5', 'surf_type', 'water_type', 'elev', 'lat', 'lon', 'sat_zen', 'sat_azi',
              'sol_zen', 'sol_azi', 'cltp', 'clf']

    surface = {}  # pd.DataFrame(columns=cnames)
    era = {}
    if not os.path.isfile(filename):
        raise IOError('File not found: %s' % filename)

    with pd.HDFStore(filename, 'r') as store:
        for ikey in store.keys():
            iname = ikey.replace('/', '')
            if ikey[1:] in ['t', 'q', 'o3']:
                # profile data
                tmp = store[ikey]
                tmp.index.name = 'date'
                tmp.columns.name = 'p'
                # tmp = tmp.reset_index().drop_duplicates('date').set_index('date')  # remove duplicates
                era[iname] = tmp  # DataFrame (time x p)

            else:
                # surface data
                if iname == 'sp':
                    surface['msl'] = store[ikey][0] / 100.  # [dates] / 100.  # hPa

                elif iname == 'ci':
                    # sea ice cover
                    surface['surf_type'] = store[ikey][0]

                elif iname == 'u10':
                    surface['u10'] = store[ikey][0]

                elif iname == 'v10':
                    surface['v10'] = store[ikey][0]

                else:
                    surface[iname] = store[ikey][0]

    era = pd.Panel(era)
    era = panel_to_database(era, remove=False)
    # ERA can have negative Q values !?
    qerror = era.query('q<=0').q.count()
    print_verbose("[%s] Found %d negative Q-values!" % (ident, qerror), verbose)
    # Interpolate limits
    rt = profile_limits(tohpa=True, simple_names=True)
    rt['qmin'] = vap2sh(rt.vpmin.values, rt.p.values)  # both missing hPa factor
    qmins = np.interp(np.log(era.p.values), np.log(rt.p.values*100.), rt.qmin.values, left=rt.qmin.min(), right=rt.qmin.max())  # Interpolate Minimum Q
    era['q'] = np.where(era.q < qmins, qmins, era.q)  # replace with RTTOV limit
    # Make a dataframe
    surface = pd.DataFrame(surface)
    surface['q2m'] = sh2ppmv(era.query('p==100000')['q'], 100000.)  # ppmv
    surface['fetch'] = 100000.  # m
    # FASTEM Surface model Parameters:
    surface['FASTEM_1'] = 3.0
    surface['FASTEM_2'] = 5.0
    surface['FASTEM_3'] = 15.0
    surface['FASTEM_4'] = 0.1
    surface['FASTEM_5'] = 0.3
    # surface properties
    if 'surf_type' in surface.columns:
        if surface['surf_type'].isnull().any(0):
            if surface['surf_type'].count() > 0:
                surface['surf_type'].fillna(method='bfill')  # backwards fill
                surface['surf_type'].fillna(method='ffill')  # forward fill

            else:
                surface['surf_type'] = 0.  # surface type (0=land, 1=sea, 2=sea-ice)

    else:
        surface['surf_type'] = 0.  # surface type (0=land, 1=sea, 2=sea-ice)

    surface['water_type'] = 0.  # water type (0=fresh, 1=ocean)
    surface['elev'] = alt / 1000.  # ELEVATION km
    surface['lat'] = lat  # LAT
    surface['lon'] = lon  # LON
    surface['sat_azi'] = 0.  # Satellite Azimuth Angle
    surface['sat_zen'] = 53.1  # OSCAR , SSMIS
    surface['sol_zen'] = 0.  # Solar Zenith Angle
    surface['sol_azi'] = 0.  # Solar Azimuth Angle
    surface['cltp'] = 500.  # Cloud top pressure
    surface['clf'] = 0.  # Cloud fraction

    # right order
    surface = surface[cnames].copy()

    # CONVERT TO WATER VAPOR PRESSURE in Pa
    era['vp'] = sh2vap(era['q'], era['p'])  # formula
    era['td'] = bolton_inverse(era.vp.values)  # default method from Murphy and Koop (iterative)
    logic = era['td'] > era['t']  # impossible
    era.loc[:, 'td'] = np.where(logic, era['t'].values, era['td'].values)
    era.loc[:, 'vp'] = np.where(logic, foeewm(era['t'].values), era['vp'].values)  # ? fixes some problems ?
    return surface, era
예제 #3
0
def standard_rel_humidity(data, rvar='r', tvar='t', dpdvar='dpd', qvar='q', update=False, replace=False,
                          method='murphy_koop', inplace=False, report=None, verbose=0):
    """ convert humidity variables to relative humidity
    1. q to vp
    2. dpd to vp
    3. merge vp
    4. vp to rh

    set quality flags

    Parameters
    ----------
    data        DataFrame   Input database
    rvar        str
    tvar        str
    dpdvar      str
    qvar        str
    update      bool        Update existing
    replace     bool        set flagged to nan
    method      str         Saturation water vapor
    inplace     bool
    verbose     int
    kwargs      **

    Returns
    -------
    data
    """
    from raso.met.conversion import sh2vap
    from raso.met import esat_functions
    from raso.qc import profile_limits
    ############################################################################
    rt = profile_limits(tohpa=True, simple_names=True)  # RTTOV Variable Limits
    rt['p'] *= 100.         # hPa to Pa
    rt['vpmin'] *= 100.     # hPa to Pa
    rt['vpmax'] *= 100.     # hPa to Pa
    ############################################################################
    funcid = "[SV] "
    r_absmin = 0
    r_absmax = 1
    if not isinstance(data, (pd.DataFrame, pd.Panel)):
        raise ValueError(funcid + "Requires a Dataframe or Panel")

    if not inplace:
        data = data.copy()

    if not hasnames('qual', data):
        data['qual'] = ''

    if hasnames(data, [rvar, qvar, dpdvar], value=0):
        raise ValueError(funcid + " Requires a humidity variable: %s, %s or %s" % (rvar, qvar, dpdvar))

    if hasnames(data,tvar, value=0):
        raise ValueError(funcid + " Requires variable: %s" % tvar)

    vpfunc = getattr(esat_functions, method)

    if not hasnames(data, rvar) or update:
        remove_pressure = False
        if isinstance(data, pd.Panel):
            data['p'] = 0.  # add minor_axis as p
            data.loc['p', :, :] = np.asarray(data.minor_axis)[np.newaxis, np.newaxis, :]
            remove_pressure = True

        journal(funcid + "rel. humidity (%s) update: %s" % (rvar, update), report, verbose)
        # Quality control
        vpmins = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmin.values, left=rt.vpmin.min(),
                           right=rt.vpmin.max())  # Interpolate Minimum
        vpmaxs = np.interp(np.log(data.p.values), np.log(rt.p.values), rt.vpmax.values, left=rt.vpmax.min(),
                           right=rt.vpmax.max())  # Interpolate Maximum
        vpsat = vpfunc(data[tvar].values)
        if hasnames(data, rvar):
            vp = data[rvar].values * vpsat  # Convert r,t to vp
            logic = ((vp < vpmins) | (vp > vpmaxs)) & np.isfinite(vp)
            data['qual'] = np.where(logic, data.qual.values + 'R', data.qual.replace('R', '').values)  # FLAG: R (?)
            journal(funcid + "rel. humidity (%s) available (replace: %s)" % (rvar, replace), report, verbose)
            if replace:
                vp = np.where(logic, np.nan, vp)  # Apply? BAD, GOOD

        else:
            vp = np.full(data[tvar].shape, np.nan)

        if hasnames(data, qvar):
            qvp = sh2vap(data[qvar].values, data['p'].values)  # only a formula no approximation
            logic = ((qvp < vpmins) | (qvp > vpmaxs)) & np.isfinite(qvp)
            data['qual'] = np.where(logic, data.qual.values + 'Q', data.qual.replace('Q', '').values)  # FLAG: R (?)
            journal(funcid + "spec. humidity (%s) available (replace: %s)" % (qvar, replace), report, verbose)
            if replace:
                qvp = np.where(logic, np.nan, qvp)  # Apply? BAD, GOOD

            # Fill Gaps
            logic = (np.isfinite(qvp) & (~np.isfinite(vp)))  # GOOD, BAD
            vp = np.where(logic, qvp, vp)  # UPDATE, OLD

        if hasnames(data, dpdvar):
            dvp = vpfunc((data[tvar] - data[dpdvar]))  # Dewpoint -> vp
            logic = ((dvp < vpmins) | (dvp > vpmaxs)) & np.isfinite(dvp)
            data['qual'] = np.where(logic, data.qual.values + 'D', data.qual.replace('D', '').values)  # FLAG: R (?)
            journal(funcid + "DPD (%s) available (replace: %s)" % (dpdvar, replace), report, verbose)
            if replace:
                dvp = np.where(logic, np.nan, dvp)  # Apply? BAD, GOOD

            # fill gaps
            logic = (np.isfinite(dvp) & (~np.isfinite(vp)))  # GOOD, BAD
            vp = np.where(logic, dvp, vp)  # UPDATE, OLD
        ############################################################################
        #
        # Convert VP to RH
        #
        data[rvar] = vp / vpsat  # rel. Humidity
        logic = ((data[rvar].values < r_absmin) | (data[rvar].values > r_absmax)) & np.isfinite(
            data[rvar].values)  # Range? BAD, GOOD
        data['qual'] = np.where(logic, data.qual.values + 'R', data.qual.replace('R', '').values)  # FLAG: R

        if replace:
            data[rvar] = np.where(logic, np.nan, data.r.values)  # Apply? BAD, GOOD
        ############################################################################
        #
        # Unique Flags
        #
        data['qual'] = unique_flags(data['qual'])
        ############################################################################
        #
        # Pressure
        #
        if remove_pressure:
            del data['p']

    if not inplace:
        return data