Ejemplo n.º 1
0
def gather_information(directory, pattern, force=False, report=None, verbose=0):
    """
    Read Information for NetCDF global attributes about time and position of GPS RO event

    Parameters
    ----------
    directory   search directory for GPS RO NetCDF files
    pattern     search pattern

    Returns
    -------
    pandas.DataFrame

    Notes
    -----
    Can take quite long to collect (2h)
    stores info in outdir/GPS_info.h5
    """
    from ncio import read_history
    from raso.config import outdir
    # check midpoints of GPS profiles and create a list with

    filename = outdir + '/GPS_info.h5'
    if os.path.isfile(filename) and not force:
        tmp = pd.read_hdf(filename, 'data')
        journal('GPSRO_info : %s %s' % (filename, tmp.shape), report, verbose)
        return tmp

    # filenames , lon, lat, time
    files = find_files(directory, pattern, recursive=True)
    data = {}
    pbar = ProgressBar(maxval=len(files))
    pbar.start()
    for i, ifile in enumerate(files):
        hist = read_history(ifile)
        data[i] = {'file': ifile, 'lon': hist['lon'],
                   'lat': hist['lat'],
                   'date': datetime(hist['year'], hist['month'], hist['day'], hist['hour'], hist['minute'])}
        pbar.update(i + 1)
    pbar.finish()
    data = pd.DataFrame.from_dict(data, orient='index')
    # save
    data.to_hdf(filename, 'data', format='table', append=False)
    journal('GPSRO_info : %s %s' % (filename, data.shape), report, verbose)
    return data
Ejemplo n.º 2
0
def from_gruan(sonde, directory=None, save=True,
               var=['cor_temp', 'temp', 'press', 'cor_rh', 'rh', 'WVMR', 'alt', 'u_temp', 'u_rh', 'u_press'],
               force=False, attach=None, attach_var=None, verbose=0, **kwargs):

    """
    Read GRUAN Data and convert to a radiosonde class object
    Read and convert time to std_times, rename temp and press to t and p.
    Drop duplicate entries.

    Possible Station Names:
    'BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU',
    'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD', 'TAT'

    Parameters
    ----------
    sonde       str         Appreviation of Sonde Station
    directory   str         Directory of GRUAN data
    var         list        of variables to include
    save        bool        Save radiosonde to store ?
    force       bool        Force a raw data reread ?
    attach      radiosonde  Attach to radiosonde class object
    attach_var  list        Variables to attach from HDFStore
    verbose     int         Level of verbosity
    kwargs      dict        Additional Keywords

    Returns
    -------
    radiosonde class object

    Raises
    ------
    ValueError  sonde not in possible Stations

    """
    from raso.standard_dates_times import _fix_datetime
    from raso.config import outdir
    from raso import radiosonde
    from . import from_store
    from ncio import read_netcdf, read_history

    avail_sondes = ['BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU', 'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD',
                    'TAT']

    if not isinstance(sonde,str):
        raise ValueError("Requires a str: sonde")

    if attach is not None:
        if not isinstance(attach,radiosonde):
            raise ValueError("Requires a radiosonde class object: attach")

    if sonde not in avail_sondes:
        raise ValueError("Only one of %s allowed!" % str(avail_sondes))

    filename = outdir + "/GRUAN_" + sonde + ".h5"

    if os.path.isfile(filename) and not force:
        print "Recovering: %s"%filename
        isonde = from_store(0, filename=filename)

        if attach is not None:
            if attach_var is None:
                attach_var = isonde.vars

            for ivar in isonde.vars:
                if ivar in attach_var:
                    attach.add_data(ivar, getattr(isonde,ivar), history="GRUAN, %s %s" %(sonde, ivar))

            return
        else:
            return isonde

    if directory is None:
        raise RuntimeError("[GRUAN] requires a directory!")
    #
    print_verbose("[GRUAN] Find files ...", verbose)
    files = find_files(directory, '*' + sonde + '*.nc', recursive=True)
    print_verbose("[GRUAN][%s] Files found: %d" % (sonde, len(files)), verbose)
    data = []
    if verbose > 0:
        pbar = ProgressBar(maxval=len(files))
        pbar.start()
        i = 0

    for ifile in files:
        tmp = pd.DataFrame(read_netcdf(ifile, verbose=verbose - 2))  # convert it all to a big Frame
        # select only necessary information
        tmp = tmp[ var ]
        tmp.index.name = 'orig_date' # retain original sounding times
        tmp.rename(columns=lambda x: x.replace('temp', 't').replace('press', 'p').replace('rh','r'), inplace=True) # rename
        tmp.p *= 100.  # from hPa to Pa
        # WVMR to q
        tmp['q'] = tmp.eval("WVMR/(WVMR+1)") # specific humidity
        tmp['date'] = tmp.index.to_period(freq='h').to_datetime() # truncate to hours
        tmp = tmp.reset_index().set_index('date') # change index
        rx = map(_fix_datetime, tmp.index)  # change time to 0, 6, 12, 18 UTC
        tmp.index = rx
        tmp.drop_duplicates(inplace=True)
        data.append(tmp)

        if verbose > 0:
            pbar.update(i + 1)
            i += 1

    if verbose > 0:
        pbar.finish()

    hist = read_history(ifile) #
    try:
        ident = "%06d" % int(hist['g.General.SiteWmoId'])
    except:
        ident = sonde

    lat = float(hist['g.MeasuringSystem.Latitude'].split(' ')[0])
    lon = float(hist['g.MeasuringSystem.Longitude'].split(' ')[0])
    alt = float(hist['g.MeasuringSystem.Altitude'].split(' ')[0])
    #
    print_verbose("[GRUAN] Concatenate data ...",verbose)
    data = pd.concat(data, axis=0)
    # drop duplicates
    # print_verbose("[GRUAN] Drop duplicates ...", verbose)
    # data.drop_duplicates(inplace=True)
    #
    if attach is None:
        print_verbose("[GRUAN] Create radiosonde object ...", verbose)
        isonde = radiosonde(ident)
        isonde.filename = filename
        isonde.add_attr('lon', lon)
        isonde.add_attr('lat', lat)
        isonde.add_attr('alt', alt)
        isonde.add_attr('is_gruan',True)
        isonde.add_data('gruan', data, history="GRUAN, %s " %(sonde))
        if save:
            isonde.save(filename=filename, verbose=verbose)
        return isonde
    else:
        attach.add_data('gruan', data, history="GRUAN, %s " %(sonde))
        if save:
            attach.save(verbose=verbose)
        return