def from_gpsro(directory, pattern, ident, radius, save=True, esat='murphy_koop', verbose=0): """ Read GPS RO events into a timeseries for a certain location Parameters ---------- directory str search directory pattern str search pattern ident int/str radiosonde ID radius float allowed distance to ilon, ilat of GPS RO events [km] save bool save radiosonde class object verbose int verbosity Raises ------ ValueError wrong input types RuntimeError no GPS RO event within specified distance """ from . import from_store from raso.config import rasodir from raso import radiosonde if not isinstance(directory, str): raise ValueError("Require a string: directory") if not isinstance(pattern, str): raise ValueError("Require a string: pattern") if not isinstance(ident, (int, str, float)): raise ValueError("Require a int, str, float: ident") if not isinstance(radius, (float, int)): raise ValueError("Require a int, float: radius") if isinstance(ident, (int, float)): ident = "%06d" % ident name = "gpsro_%06d" % int(radius) iname = "gpsro_infos_%06d" % int(radius) filename = rasodir + "/%s/%s.h5" % (ident, name) # RECOVER if os.path.isfile(filename): print_verbose("Recovering previous: %s" % filename, verbose) sonde = from_store(ident, variables=[name, iname]) return sonde if not os.path.isdir(rasodir + "/%s" % ident): os.makedirs(rasodir + "/%s" % ident) try: sonde = from_store(ident, variables=[]) # Reads the empty Radiosonde class >> IOError ilon = sonde.lon # >> Attribute Error ilat = sonde.lat except IOError: sonde = radiosonde(ident) # Make a new one sondes = radiosondelist() ilon = sondes.ix[ident, 'lon'].values ilat = sondes.ix[ident, 'lat'].values except AttributeError: sondes = radiosondelist() ilon = sondes.ix[ident, 'lon'].values ilat = sondes.ix[ident, 'lat'].values if ilon is None or ilat is None: raise RuntimeError("Can't find any useful lon, lat Information on Radiosonde: %s" % ident) infos, data = read_nearest_events(directory, pattern, ilon, ilat, radius, esat=esat, verbose=verbose) journal("GPSRO Data: r=%d km [%s/%s]" % (radius, directory, pattern), sonde.history, verbose) sonde.add_attr('lon', ilon) sonde.add_attr('lat', ilat) sonde.add_data(iname, infos) sonde.add_data(name, data) if save: sonde.save(verbose=verbose) else: return sonde
def from_mars(ident, var=None, filename=None, save=False, update=True, mars_save=False, attach=None, force=False, verbose=0, **kwargs): """Read MARS (ECMWF) dump from ODB 1. Read MARS dump or ODB dump 2. Convert to pandas dataframes Typical data in mars dump: data RAW radiosonde data (time,p, variables) an Analysis departures ERA-Interim fg First-guess departures ERA-Interim bias Bias estimate station Station (lon,lat,alt,sonde_type) Parameters ---------- ident int/str Radiosonde ID var list Variables in the MARS filename str filename of mars dump save bool save radiosonde class ? mars_save bool save mars dump to HDF5 ? attach radiosonde attach to this radiosonde verbose int verbosness kwargs dict ** Returns ------- radiosonde Examples -------- >>> isonde = from_mars(11035, verbose=1) """ from raso import radiosonde selection = False if var is not None: if isinstance(var, str): var = [var] for ivar in var: if ivar not in ['data', 'bias', 'fg', 'an']: raise ValueError("Unknown variable! %s" % ivar) selection = True try: ident = int(ident) # ident needs to be a number here tmp = mars_to_dataframe(ident, filename=filename, save=mars_save, verbose=verbose - 1) print_verbose('[MARS] %d : %s' % (ident, str(tmp.shape)), verbose) station = tmp[['lon', 'lat', 'alt', 'sonde_type']].drop_duplicates() if 'std_time' in tmp.columns: # legacy support tmp.drop('std_time', axis=1, inplace=True) tmp.drop(['sonde_type', 'obs_type', 'code_type', 'id', 'lon', 'lat', 'alt'], axis=1, inplace=True) # ['varno' 'p' 'value' 'biascorr' 'fg_dep' 'an_dep'] tmp.replace({'varno': {2: 't', 3: 'u', 4: 'v', 7: 'q', 29: 'r'}}, inplace=True) # rename varno to vars # group by index and p data = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['value'].unstack(level=-1).reset_index().rename( columns={'level_0': 'date'}).set_index('date', drop=True) # if (selection and 'bias' in var) or not selection: bias = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['biascorr'].unstack( level=-1).reset_index().rename(columns={'level_0': 'date'}).set_index('date', drop=True) bias.ix[:, ~(bias.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + bias.ix[:, ~(bias.columns == 'p')] # if (selection and 'fg' in var) or not selection: fg = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['fg_dep'].unstack(level=-1).reset_index().rename( columns={'level_0': 'date'}).set_index('date', drop=True) fg.ix[:, ~(fg.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + fg.ix[:, ~(fg.columns == 'p')] # if (selection and 'an' in var) or not selection: an = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['an_dep'].unstack(level=-1).reset_index().rename( columns={'level_0': 'date'}).set_index('date', drop=True) an.ix[:, ~(an.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + an.ix[:, ~(an.columns == 'p')] if attach is None: out = radiosonde(id=ident) else: out = attach # if selection: print_verbose("MARS: %s" % (str(var)), verbose) for ivar in var: out.add_data(ivar, locals()[ivar]) else: out.add_data('data', data) out.add_data('bias', bias) out.add_data('fg', fg) out.add_data('an', an) # out.add_data('station', station) try: out.add_attr('lon', station.ix[-1, 'lon']) out.add_attr('lat', station.ix[-1, 'lat']) out.add_attr('alt', station.ix[-1, 'alt']) except IndexError: out.add_attr('lon', -1) out.add_attr('lat', -1) out.add_attr('alt', -1) if attach is None: if save: out.save(update=update, force=force, verbose=verbose) else: return out except: raise
def from_gruan(sonde, directory=None, save=True, var=['cor_temp', 'temp', 'press', 'cor_rh', 'rh', 'WVMR', 'alt', 'u_temp', 'u_rh', 'u_press'], force=False, attach=None, attach_var=None, verbose=0, **kwargs): """ Read GRUAN Data and convert to a radiosonde class object Read and convert time to std_times, rename temp and press to t and p. Drop duplicate entries. Possible Station Names: 'BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU', 'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD', 'TAT' Parameters ---------- sonde str Appreviation of Sonde Station directory str Directory of GRUAN data var list of variables to include save bool Save radiosonde to store ? force bool Force a raw data reread ? attach radiosonde Attach to radiosonde class object attach_var list Variables to attach from HDFStore verbose int Level of verbosity kwargs dict Additional Keywords Returns ------- radiosonde class object Raises ------ ValueError sonde not in possible Stations """ from raso.standard_dates_times import _fix_datetime from raso.config import outdir from raso import radiosonde from . import from_store from ncio import read_netcdf, read_history avail_sondes = ['BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU', 'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD', 'TAT'] if not isinstance(sonde,str): raise ValueError("Requires a str: sonde") if attach is not None: if not isinstance(attach,radiosonde): raise ValueError("Requires a radiosonde class object: attach") if sonde not in avail_sondes: raise ValueError("Only one of %s allowed!" % str(avail_sondes)) filename = outdir + "/GRUAN_" + sonde + ".h5" if os.path.isfile(filename) and not force: print "Recovering: %s"%filename isonde = from_store(0, filename=filename) if attach is not None: if attach_var is None: attach_var = isonde.vars for ivar in isonde.vars: if ivar in attach_var: attach.add_data(ivar, getattr(isonde,ivar), history="GRUAN, %s %s" %(sonde, ivar)) return else: return isonde if directory is None: raise RuntimeError("[GRUAN] requires a directory!") # print_verbose("[GRUAN] Find files ...", verbose) files = find_files(directory, '*' + sonde + '*.nc', recursive=True) print_verbose("[GRUAN][%s] Files found: %d" % (sonde, len(files)), verbose) data = [] if verbose > 0: pbar = ProgressBar(maxval=len(files)) pbar.start() i = 0 for ifile in files: tmp = pd.DataFrame(read_netcdf(ifile, verbose=verbose - 2)) # convert it all to a big Frame # select only necessary information tmp = tmp[ var ] tmp.index.name = 'orig_date' # retain original sounding times tmp.rename(columns=lambda x: x.replace('temp', 't').replace('press', 'p').replace('rh','r'), inplace=True) # rename tmp.p *= 100. # from hPa to Pa # WVMR to q tmp['q'] = tmp.eval("WVMR/(WVMR+1)") # specific humidity tmp['date'] = tmp.index.to_period(freq='h').to_datetime() # truncate to hours tmp = tmp.reset_index().set_index('date') # change index rx = map(_fix_datetime, tmp.index) # change time to 0, 6, 12, 18 UTC tmp.index = rx tmp.drop_duplicates(inplace=True) data.append(tmp) if verbose > 0: pbar.update(i + 1) i += 1 if verbose > 0: pbar.finish() hist = read_history(ifile) # try: ident = "%06d" % int(hist['g.General.SiteWmoId']) except: ident = sonde lat = float(hist['g.MeasuringSystem.Latitude'].split(' ')[0]) lon = float(hist['g.MeasuringSystem.Longitude'].split(' ')[0]) alt = float(hist['g.MeasuringSystem.Altitude'].split(' ')[0]) # print_verbose("[GRUAN] Concatenate data ...",verbose) data = pd.concat(data, axis=0) # drop duplicates # print_verbose("[GRUAN] Drop duplicates ...", verbose) # data.drop_duplicates(inplace=True) # if attach is None: print_verbose("[GRUAN] Create radiosonde object ...", verbose) isonde = radiosonde(ident) isonde.filename = filename isonde.add_attr('lon', lon) isonde.add_attr('lat', lat) isonde.add_attr('alt', alt) isonde.add_attr('is_gruan',True) isonde.add_data('gruan', data, history="GRUAN, %s " %(sonde)) if save: isonde.save(filename=filename, verbose=verbose) return isonde else: attach.add_data('gruan', data, history="GRUAN, %s " %(sonde)) if save: attach.save(verbose=verbose) return