def from_gpsro(directory, pattern, ident, radius, save=True, esat='murphy_koop', verbose=0):
"""
Read GPS RO events into a timeseries for a certain location
Parameters
----------
directory str search directory
pattern str search pattern
ident int/str radiosonde ID
radius float allowed distance to ilon, ilat of GPS RO events [km]
save bool save radiosonde class object
verbose int verbosity
Raises
------
ValueError wrong input types
RuntimeError no GPS RO event within specified distance
"""
from . import from_store
from raso.config import rasodir
from raso import radiosonde
if not isinstance(directory, str):
raise ValueError("Require a string: directory")
if not isinstance(pattern, str):
raise ValueError("Require a string: pattern")
if not isinstance(ident, (int, str, float)):
raise ValueError("Require a int, str, float: ident")
if not isinstance(radius, (float, int)):
raise ValueError("Require a int, float: radius")
if isinstance(ident, (int, float)):
ident = "%06d" % ident
name = "gpsro_%06d" % int(radius)
iname = "gpsro_infos_%06d" % int(radius)
filename = rasodir + "/%s/%s.h5" % (ident, name)
# RECOVER
if os.path.isfile(filename):
print_verbose("Recovering previous: %s" % filename, verbose)
sonde = from_store(ident, variables=[name, iname])
return sonde
if not os.path.isdir(rasodir + "/%s" % ident):
os.makedirs(rasodir + "/%s" % ident)
try:
sonde = from_store(ident, variables=[]) # Reads the empty Radiosonde class >> IOError
ilon = sonde.lon # >> Attribute Error
ilat = sonde.lat
except IOError:
sonde = radiosonde(ident) # Make a new one
sondes = radiosondelist()
ilon = sondes.ix[ident, 'lon'].values
ilat = sondes.ix[ident, 'lat'].values
except AttributeError:
sondes = radiosondelist()
ilon = sondes.ix[ident, 'lon'].values
ilat = sondes.ix[ident, 'lat'].values
if ilon is None or ilat is None:
raise RuntimeError("Can't find any useful lon, lat Information on Radiosonde: %s" % ident)
infos, data = read_nearest_events(directory, pattern, ilon, ilat, radius, esat=esat, verbose=verbose)
journal("GPSRO Data: r=%d km [%s/%s]" % (radius, directory, pattern), sonde.history, verbose)
sonde.add_attr('lon', ilon)
sonde.add_attr('lat', ilat)
sonde.add_data(iname, infos)
sonde.add_data(name, data)
if save:
sonde.save(verbose=verbose)
else:
return sonde
def from_mars(ident, var=None, filename=None, save=False, update=True, mars_save=False, attach=None,
force=False, verbose=0, **kwargs):
"""Read MARS (ECMWF) dump from ODB
1. Read MARS dump or ODB dump
2. Convert to pandas dataframes
Typical data in mars dump:
data RAW radiosonde data (time,p, variables)
an Analysis departures ERA-Interim
fg First-guess departures ERA-Interim
bias Bias estimate
station Station (lon,lat,alt,sonde_type)
Parameters
----------
ident int/str Radiosonde ID
var list Variables in the MARS
filename str filename of mars dump
save bool save radiosonde class ?
mars_save bool save mars dump to HDF5 ?
attach radiosonde attach to this radiosonde
verbose int verbosness
kwargs dict **
Returns
-------
radiosonde
Examples
--------
>>> isonde = from_mars(11035, verbose=1)
"""
from raso import radiosonde
selection = False
if var is not None:
if isinstance(var, str):
var = [var]
for ivar in var:
if ivar not in ['data', 'bias', 'fg', 'an']:
raise ValueError("Unknown variable! %s" % ivar)
selection = True
try:
ident = int(ident) # ident needs to be a number here
tmp = mars_to_dataframe(ident, filename=filename, save=mars_save, verbose=verbose - 1)
print_verbose('[MARS] %d : %s' % (ident, str(tmp.shape)), verbose)
station = tmp[['lon', 'lat', 'alt', 'sonde_type']].drop_duplicates()
if 'std_time' in tmp.columns:
# legacy support
tmp.drop('std_time', axis=1, inplace=True)
tmp.drop(['sonde_type', 'obs_type', 'code_type', 'id', 'lon', 'lat', 'alt'], axis=1, inplace=True)
# ['varno' 'p' 'value' 'biascorr' 'fg_dep' 'an_dep']
tmp.replace({'varno': {2: 't', 3: 'u', 4: 'v', 7: 'q', 29: 'r'}}, inplace=True) # rename varno to vars
# group by index and p
data = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['value'].unstack(level=-1).reset_index().rename(
columns={'level_0': 'date'}).set_index('date', drop=True)
#
if (selection and 'bias' in var) or not selection:
bias = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['biascorr'].unstack(
level=-1).reset_index().rename(columns={'level_0': 'date'}).set_index('date', drop=True)
bias.ix[:, ~(bias.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + bias.ix[:, ~(bias.columns == 'p')]
#
if (selection and 'fg' in var) or not selection:
fg = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['fg_dep'].unstack(level=-1).reset_index().rename(
columns={'level_0': 'date'}).set_index('date', drop=True)
fg.ix[:, ~(fg.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + fg.ix[:, ~(fg.columns == 'p')]
#
if (selection and 'an' in var) or not selection:
an = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['an_dep'].unstack(level=-1).reset_index().rename(
columns={'level_0': 'date'}).set_index('date', drop=True)
an.ix[:, ~(an.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + an.ix[:, ~(an.columns == 'p')]
if attach is None:
out = radiosonde(id=ident)
else:
out = attach
#
if selection:
print_verbose("MARS: %s" % (str(var)), verbose)
for ivar in var:
out.add_data(ivar, locals()[ivar])
else:
out.add_data('data', data)
out.add_data('bias', bias)
out.add_data('fg', fg)
out.add_data('an', an)
#
out.add_data('station', station)
try:
out.add_attr('lon', station.ix[-1, 'lon'])
out.add_attr('lat', station.ix[-1, 'lat'])
out.add_attr('alt', station.ix[-1, 'alt'])
except IndexError:
out.add_attr('lon', -1)
out.add_attr('lat', -1)
out.add_attr('alt', -1)
if attach is None:
if save:
out.save(update=update, force=force, verbose=verbose)
else:
return out
except:
raise
def from_gruan(sonde, directory=None, save=True,
var=['cor_temp', 'temp', 'press', 'cor_rh', 'rh', 'WVMR', 'alt', 'u_temp', 'u_rh', 'u_press'],
force=False, attach=None, attach_var=None, verbose=0, **kwargs):
"""
Read GRUAN Data and convert to a radiosonde class object
Read and convert time to std_times, rename temp and press to t and p.
Drop duplicate entries.
Possible Station Names:
'BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU',
'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD', 'TAT'
Parameters
----------
sonde str Appreviation of Sonde Station
directory str Directory of GRUAN data
var list of variables to include
save bool Save radiosonde to store ?
force bool Force a raw data reread ?
attach radiosonde Attach to radiosonde class object
attach_var list Variables to attach from HDFStore
verbose int Level of verbosity
kwargs dict Additional Keywords
Returns
-------
radiosonde class object
Raises
------
ValueError sonde not in possible Stations
"""
from raso.standard_dates_times import _fix_datetime
from raso.config import outdir
from raso import radiosonde
from . import from_store
from ncio import read_netcdf, read_history
avail_sondes = ['BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU', 'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD',
'TAT']
if not isinstance(sonde,str):
raise ValueError("Requires a str: sonde")
if attach is not None:
if not isinstance(attach,radiosonde):
raise ValueError("Requires a radiosonde class object: attach")
if sonde not in avail_sondes:
raise ValueError("Only one of %s allowed!" % str(avail_sondes))
filename = outdir + "/GRUAN_" + sonde + ".h5"
if os.path.isfile(filename) and not force:
print "Recovering: %s"%filename
isonde = from_store(0, filename=filename)
if attach is not None:
if attach_var is None:
attach_var = isonde.vars
for ivar in isonde.vars:
if ivar in attach_var:
attach.add_data(ivar, getattr(isonde,ivar), history="GRUAN, %s %s" %(sonde, ivar))
return
else:
return isonde
if directory is None:
raise RuntimeError("[GRUAN] requires a directory!")
#
print_verbose("[GRUAN] Find files ...", verbose)
files = find_files(directory, '*' + sonde + '*.nc', recursive=True)
print_verbose("[GRUAN][%s] Files found: %d" % (sonde, len(files)), verbose)
data = []
if verbose > 0:
pbar = ProgressBar(maxval=len(files))
pbar.start()
i = 0
for ifile in files:
tmp = pd.DataFrame(read_netcdf(ifile, verbose=verbose - 2)) # convert it all to a big Frame
# select only necessary information
tmp = tmp[ var ]
tmp.index.name = 'orig_date' # retain original sounding times
tmp.rename(columns=lambda x: x.replace('temp', 't').replace('press', 'p').replace('rh','r'), inplace=True) # rename
tmp.p *= 100. # from hPa to Pa
# WVMR to q
tmp['q'] = tmp.eval("WVMR/(WVMR+1)") # specific humidity
tmp['date'] = tmp.index.to_period(freq='h').to_datetime() # truncate to hours
tmp = tmp.reset_index().set_index('date') # change index
rx = map(_fix_datetime, tmp.index) # change time to 0, 6, 12, 18 UTC
tmp.index = rx
tmp.drop_duplicates(inplace=True)
data.append(tmp)
if verbose > 0:
pbar.update(i + 1)
i += 1
if verbose > 0:
pbar.finish()
hist = read_history(ifile) #
try:
ident = "%06d" % int(hist['g.General.SiteWmoId'])
except:
ident = sonde
lat = float(hist['g.MeasuringSystem.Latitude'].split(' ')[0])
lon = float(hist['g.MeasuringSystem.Longitude'].split(' ')[0])
alt = float(hist['g.MeasuringSystem.Altitude'].split(' ')[0])
#
print_verbose("[GRUAN] Concatenate data ...",verbose)
data = pd.concat(data, axis=0)
# drop duplicates
# print_verbose("[GRUAN] Drop duplicates ...", verbose)
# data.drop_duplicates(inplace=True)
#
if attach is None:
print_verbose("[GRUAN] Create radiosonde object ...", verbose)
isonde = radiosonde(ident)
isonde.filename = filename
isonde.add_attr('lon', lon)
isonde.add_attr('lat', lat)
isonde.add_attr('alt', alt)
isonde.add_attr('is_gruan',True)
isonde.add_data('gruan', data, history="GRUAN, %s " %(sonde))
if save:
isonde.save(filename=filename, verbose=verbose)
return isonde
else:
attach.add_data('gruan', data, history="GRUAN, %s " %(sonde))
if save:
attach.save(verbose=verbose)
return
