def from_igra(ident, var=['p', 't', 'r', 'dpd', 'wd', 'ws'], filename=None, attach=None, save=False, verbose=0, **kwargs):
"""Read IGRA data from text
Parameters
----------
ident Radiosonde ID (WMO or IGRA ID)
wmo convert WMO ID to IGRA
variables select variables
filename filename of txt.gz
attach radiosonde object to attach with
verbose Verbosity
Returns
-------
radiosonde object / save to HDF5
"""
wmoid = None
igraid = wmo2igra(ident) # try to convert WMO to IGRA ?
if igraid is not None:
wmoid = ident # WMO ID
ident = igraid # found IGRA ID
print_verbose("[IGRA] WMO %s to IGRA %s"%(wmoid, ident), verbose)
data = from_igra_to_dataframe(ident, filename=filename, verbose=verbose)
lon = data.ix[-1, 'lon'] / 10000. # IGRA specific
lat = data.ix[-1, 'lat'] / 10000.
data = data[var]
#
if isinstance(attach, radiosonde):
attach.add_data('igra', data)
attach.add_attr('igraid', ident)
if 'lon' not in attach.attrs:
attach.add_attr('lon', lon)
if 'lat' not in attach.attrs:
attach.add_attr('lat', lat)
journal("[IGRA] data read with %s" % ident, attach.history, verbose)
else:
# try to convert to WMO ID
if wmoid is None:
wmoid = igra2wmo(ident)
if wmoid is None:
out = radiosonde(id=ident)
else:
out = radiosonde(id=wmoid)
out.add_attr('igraid', ident)
out.add_data('igra', data)
out.add_attr('lon', lon)
out.add_attr('lat', lat) # convert to float
out.history.append(now() + " IGRA Data read")
if save:
out.save()
else:
return out
def assessment_report(network, ids=None, directory=None, verbose=0, **kwargs):
"""
Create figures for an assessment report
Parameters
----------
network
ids
var
verbose
kwargs
Returns
-------
"""
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from raso.plot import event_plot
from raso.support import print_verbose
if directory is None:
directory = network.directory
plt.ioff()
plot_stationmap(network, ids=ids, save=True, **kwargs)
# network infos
nsondes, nvars = network.infos.shape # 2D
npages = int(np.ceil(nsondes/100.)) # 100 per page
pdf = PdfPages(directory + "infos.pdf")
figsize = kwargs.pop('figsize', (12, 9)) # TODO: make it dependent on nvars
info_columns = network.infos.columns.tolist()
for ipage in range(npages):
print_verbose("INFOS: %s " % ipage, verbose)
ax = network.infos.plot(subplots=True, kind='bar', legend=False, sharex=True, title=network.name, grid=True,
figsize=figsize)
for i, iax in enumerate(ax):
iax.set_title('')
iax.set_ylabel(info_columns[i]) # LABEL
fig = ax.get_figure()
pdf.savefig(fig)
plt.close(fig)
pdf.close()
plt.close('all')
# network Availability
network.load_infodata()
if len(network.infodata) > 0:
for ivar in network.infodata:
filename = directory + '%s.pdf' % ivar
print_verbose("AVAIL: %s "% ivar, verbose)
event_plot(getattr(network, ivar),save=True, filename=filename, vmin=0, vmax=70)
def load_data(self, name, directory=None, variable=None, suffix=None, verbose=0):
""" Load data from Store
Parameters
----------
name
directory
suffix
verbose
Returns
-------
"""
from raso.config import rasodir
if not hasattr(name, '__iter__'):
name = [name]
if directory is None:
if self.directory is None:
directory = rasodir
else:
directory = self.directory
if suffix is None:
suffix = ""
else:
if suffix[-1] != '_':
suffix += '_'
default = directory + '/%s' % self.id
for j in name:
if os.path.isfile(default+'/%s.h5' % j):
with pd.HDFStore(default+'/%s.h5' % j) as store:
for ivar in store.keys():
if variable is not None:
if ivar.replace('/','') != variable:
continue
iname = ivar.replace('/', suffix)
self.add_data(iname, store[ivar], replace=True)
self.is_saved = True # Update Vars
self.var_saved[iname] = (iname == j) # Update Vars
print_verbose("[LD] %s : %s : %s" % (j, iname, str(store[ivar].shape)), verbose)
else:
print "%s not found " % j
self.is_saved = np.any(self.var_saved.values())
def read_nearest_events(directory, pattern, ilon, ilat, radius, esat='murphy_koop', verbose=0):
""" Internal Function to find nearest events
Parameters
----------
directory
pattern
ilon
ilat
radius
esat
verbose
Returns
-------
"""
from raso.met import esat_functions, vap2sh
from ncio import read_variable
print_verbose("Collecting information ...", verbose)
infos = gather_information(directory, pattern, verbose=verbose)
# calculate distance to ilon ilat
print_verbose("Calculating distances ...", verbose)
infos['distance'] = distance(infos['lon'], infos['lat'], ilon, ilat)
n = infos.shape[0]
infos = infos.query('distance<%f' % radius)
if infos.shape[0] == 0:
raise RuntimeError("No GPS RO events (%d) found within the specified radius (%f)!" % (n, radius))
vpfunc = getattr(esat_functions, esat)
data = []
print_verbose("Procesing Profiles (%d) ..." % n, verbose)
for iprof, irow in infos.iterrows():
tmp = pd.DataFrame(read_variable(irow['file']))
tmp.columns = ['lat', 'lon', 'p', 'ref', 'ref_obs', 't', 'vp']
tmp.index.name = 'alt'
tmp['date'] = irow['date']
tmp = tmp.reset_index().set_index('date')
tmp['t'] += 273.15
tmp['p'] *= 100. # mb (hPa) to Pa
tmp['alt'] *= 1000. # km to m
tmp['vp'] *= 100. # mb (hPa) to Pa
tmp['r'] = tmp['vp'] / vpfunc(tmp['t'])
tmp['q'] = vap2sh(tmp['vp'], tmp['p'])
tmp['dist'] = distance(tmp['lon'], tmp['lat'], ilon, ilat)
data.append(tmp)
data = pd.concat(data, axis=0) # list of DataFrames
data.dropna(subset=['p', 'ref', 'ref_obs', 't', 'vp'], how='all', inplace=True) # drop missing
if data.duplicated().any():
print "Duplicates found! ", data.duplicated().sum() # check for duplicates ?
print_verbose("GPSRO Events: %d" % data.index.unique().size, verbose)
return infos, data
def _2d_detection(data, var, window, dist, thres, min_levels, verbose=0):
""" Detect Breakpoints in 2D (time x pressure levels)
1. Run SNHT per level
2. Count significant peaks from all levels
2.1. Local Maxima (>END)
2.2. Above Threshold
3. add breaks variable with 1 and 2 (break in that level)
Parameters
----------
data Panel Inputdata (vars, time, p)
var str Variable to consider
window float Window size of Detection
dist float distance between breakpoints
thres int SNHT threshold
min_levels int Minimum required levels to detect breakpoint
verbose
Returns
-------
"""
stest = np.squeeze(np.apply_along_axis(snht, 0, data[var].values, window, window / 4)) # per level
data['%s_snht' % var] = stest
data['%s_breaks' % var] = 0
imax = np.asarray(local_maxima(np.sum(stest, 1), dist=dist)) # for all levels
if len(imax) == 0:
print_verbose(
"No Breakpoints detected: %s min_levels (%d) found: %f (%f)" % (var, min_levels, np.max(stest), thres),
verbose)
return False
print_verbose("Local Maxima (%s): %d" % (var, len(imax)), verbose)
# how many above threshold
tstat = np.sum(stest[imax, :] > thres, 1) # could weight levels ? upper levels are less relevant?
if not np.any(tstat >= min_levels):
print_verbose("No Breakpoints detected: %s min_levels (%d) found: %d" % (var, min_levels, np.max(tstat)),
verbose)
return False
else:
if verbose > 1:
print "Breaks (%s): " % var
for i, ib in enumerate(tstat):
print "%s (%d) %s" % (str(data.major_axis[imax[i]]), ib, color_boolean(ib >= min_levels))
print_verbose("Final Breaks (%s): %d" % (var, np.sum(tstat >= min_levels)), verbose)
itx = imax[tstat >= min_levels] # indices of breaks
ivar = data.items.get_loc("%s_breaks" % var) # index of variable
iarray = 1 + np.int_(stest[itx, :] > thres)
data.iloc[ivar, itx, :] = iarray[np.newaxis, ::] # (1,...) is required to set
return True
def mars_to_dataframe(ident, filename=None, save=False, verbose=0):
""" Read MARS DUMP HDF5 file
Parameters
----------
ident
filename
save
verbose
Returns
-------
DataFrame
"""
from raso.config import marsdir, marspattern, rasodir
if filename is None:
ident = "%06d" % int(ident)
filename = rasodir + '/%s/mars_dump.h5' % ident # Default DUMP FILE
if os.path.isfile(filename):
print_verbose("[MARS] %s" % filename, verbose)
try:
return pd.read_hdf(filename, 'mars')
except KeyError:
store = pd.HDFStore(filename)
print "Using first of:", store.keys()," for MARS"
store['mars'] = store[store.keys()[0]]
del store[store.keys()[0]]
store.close()
return pd.read_hdf(filename, 'mars')
else:
filename = marsdir + '/' + marspattern % int(ident)
print_verbose("[MARS] %s" % filename, verbose)
return mars_gzip(filename, save=save, verbose=verbose)
def _1d_detection(data, var, window, dist, thres, verbose=0):
stest = snht(data[var].values, window, window / 4)
data['%s_snht' % var] = stest
# could be too close together
imax = np.asarray(local_maxima(stest, dist=dist)) # local maxima within a certain distance!
if len(imax) == 0:
print_verbose("No Breakpoints detected: %s found: %f (%f)" % (var, np.max(stest), thres), verbose)
return False
print_verbose("Local Maxima (%s): %d" % (var, len(imax)), verbose)
tstat = stest[imax] > thres # above threshold
if not np.any(tstat > 0):
print_verbose("No Breakpoints detected: %s" % (var), verbose)
return False
else:
print_verbose("Final Breaks (%s): %d" % (var, np.sum(tstat > 0)), verbose)
data['%s_breaks' % var] = 0
ivar = data.columns.get_loc('%s_breaks' % var)
data.iloc[imax[tstat > 0], ivar] = 2
return True
def snht(ident, levels=None, outputdir=None, verbose=0):
"""
Parameters
----------
ident
levels
outputdir
verbose
Returns
-------
"""
from raso import load_radiosonde
from raso.config import outdir
import matplotlib.pyplot as plt
yplevs = [1000, 3000, 7000, 12500, 17500, 22500, 30000, 40000, 50000, 60000, 85000, 92500] # Pressure Levels, Ticks
isonde = load_radiosonde(ident, variables=[])
if outputdir is None:
outputdir = outdir + '/figures'
#
# DATA
#
if 'detect_opts' not in isonde.attrs:
print "No Detection Options found ?", ident
return
if levels is None:
levels = [10000, 30000, 50000, 60000, 70000, 85000, 92500]
for icor, iopts in isonde.detect_opts.items():
iname = iopts['savename']
isonde.load_data(iname)
ithres = iopts['thres']
ivar = iopts['variable']
idata = getattr(isonde, iname)
daynight = iopts.get('daynight', False)
#
# CONTOUR
#
if daynight:
timeseries_snht(idata[:, idata.major_axis.hour == 12, :], '%s_dep' % ivar,
levels=list(ithres * np.array([1, 2, 5, 10, 20])), yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_12Z_snht.pdf" % (outputdir, ident, iname, ivar))
print_verbose("%s/%s_%s_%s_12Z_snht.pdf" % (outputdir, ident, iname, ivar), verbose)
timeseries_snht(idata[:, idata.major_axis.hour == 0, :], '%s_dep' % ivar,
levels=list(ithres * np.array([1, 2, 5, 10, 20])), yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_00Z_snht.pdf" % (outputdir, ident, iname, ivar))
print_verbose("%s/%s_%s_%s_00Z_snht.pdf" % (outputdir, ident, iname, ivar), verbose)
else:
timeseries_snht(idata, '%s_dep' % ivar, levels=list(ithres * np.array([1, 2, 5, 10, 20])),
yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_snht.pdf" % (outputdir, ident, iname, ivar))
print_verbose("%s/%s_%s_%s_snht.pdf" % (outputdir, ident, iname, ivar), verbose)
#
# LEVEL
#
for ilev in levels:
fig, ax = plt.subplots(1, 1, figsize=(12, 2))
if daynight:
timeseries_line_snht(idata[:, idata.major_axis.hour == 12, :], '%s_dep' % ivar,
'%s_dep_snht' % ivar, ilev, '%s dep (12Z) [K]' % ivar,
breaks='%s_dep_breaks' % ivar, ax=ax, ylim=(-10, 10))
ax.set_xlabel('Time (RM:60)')
plt.savefig('%s/%s_%s_%s_12Z_snht_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)))
print_verbose('%s/%s_%s_%s_12Z_snht_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)), verbose)
timeseries_line_snht(idata[:, idata.major_axis.hour == 0, :], '%s_dep' % ivar,
'%s_dep_snht' % ivar, ilev, '%s dep (00Z) [K]' % ivar,
breaks='%s_dep_breaks' % ivar, ax=ax, ylim=(-10, 10))
ax.set_xlabel('Time (RM:60)')
plt.savefig('%s/%s_%s_%s_00Z_snht_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)))
print_verbose('%s/%s_%s_%s_00Z_snht_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)), verbose)
else:
timeseries_line_snht(idata, '%s_dep' % ivar, '%s_dep_snht' % ivar, ilev, '%s dep [K]' % ivar,
breaks='%s_dep_breaks' % ivar, ax=ax, ylim=(-10, 10))
ax.set_xlabel('Time (RM:60)')
plt.savefig('%s/%s_%s_%s_snht_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)))
print_verbose('%s/%s_%s_%s_snht_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)), verbose)
plt.close('all')
def from_era(ident, alt=0, lon=0, lat=0, savename='extract_ERA', verbose=0):
""" Updated Version of profiles and surface
# Can read ERA data from retrieved and converted MARS Requests
ERADIR
Parameters
----------
ident str/int/float Radiosonde ID
alt int/float Radiosonde Station Altitude
lon int/float Radiosonde Station Longitude
lat int/float Radiosonde Station Latitude
savename str Extract savename
verbose int verboseness
Returns
-------
"""
from raso.support import print_verbose, panel_to_database
from raso.met.conversion import sh2ppmv, sh2vap, bolton_inverse, vap2sh
from raso.met.esat_functions import foeewm
from raso.config import rasodir # eradir
from raso.qc import profile_limits
# read new era file
if isinstance(ident, (int, float)):
ident = "%06d" % ident
filename = rasodir + "/%s/%s.h5" % (ident, savename)
cnames = ['t2m', 'q2m', 'msl', 'u10', 'v10', 'fetch', 'skt', 'FASTEM_1', 'FASTEM_2', 'FASTEM_3',
'FASTEM_4', 'FASTEM_5', 'surf_type', 'water_type', 'elev', 'lat', 'lon', 'sat_zen', 'sat_azi',
'sol_zen', 'sol_azi', 'cltp', 'clf']
surface = {} # pd.DataFrame(columns=cnames)
era = {}
if not os.path.isfile(filename):
raise IOError('File not found: %s' % filename)
with pd.HDFStore(filename, 'r') as store:
for ikey in store.keys():
iname = ikey.replace('/', '')
if ikey[1:] in ['t', 'q', 'o3']:
# profile data
tmp = store[ikey]
tmp.index.name = 'date'
tmp.columns.name = 'p'
# tmp = tmp.reset_index().drop_duplicates('date').set_index('date') # remove duplicates
era[iname] = tmp # DataFrame (time x p)
else:
# surface data
if iname == 'sp':
surface['msl'] = store[ikey][0] / 100. # [dates] / 100. # hPa
elif iname == 'ci':
# sea ice cover
surface['surf_type'] = store[ikey][0]
elif iname == 'u10':
surface['u10'] = store[ikey][0]
elif iname == 'v10':
surface['v10'] = store[ikey][0]
else:
surface[iname] = store[ikey][0]
era = pd.Panel(era)
era = panel_to_database(era, remove=False)
# ERA can have negative Q values !?
qerror = era.query('q<=0').q.count()
print_verbose("[%s] Found %d negative Q-values!" % (ident, qerror), verbose)
# Interpolate limits
rt = profile_limits(tohpa=True, simple_names=True)
rt['qmin'] = vap2sh(rt.vpmin.values, rt.p.values) # both missing hPa factor
qmins = np.interp(np.log(era.p.values), np.log(rt.p.values*100.), rt.qmin.values, left=rt.qmin.min(), right=rt.qmin.max()) # Interpolate Minimum Q
era['q'] = np.where(era.q < qmins, qmins, era.q) # replace with RTTOV limit
# Make a dataframe
surface = pd.DataFrame(surface)
surface['q2m'] = sh2ppmv(era.query('p==100000')['q'], 100000.) # ppmv
surface['fetch'] = 100000. # m
# FASTEM Surface model Parameters:
surface['FASTEM_1'] = 3.0
surface['FASTEM_2'] = 5.0
surface['FASTEM_3'] = 15.0
surface['FASTEM_4'] = 0.1
surface['FASTEM_5'] = 0.3
# surface properties
if 'surf_type' in surface.columns:
if surface['surf_type'].isnull().any(0):
if surface['surf_type'].count() > 0:
surface['surf_type'].fillna(method='bfill') # backwards fill
surface['surf_type'].fillna(method='ffill') # forward fill
else:
surface['surf_type'] = 0. # surface type (0=land, 1=sea, 2=sea-ice)
else:
surface['surf_type'] = 0. # surface type (0=land, 1=sea, 2=sea-ice)
surface['water_type'] = 0. # water type (0=fresh, 1=ocean)
surface['elev'] = alt / 1000. # ELEVATION km
surface['lat'] = lat # LAT
surface['lon'] = lon # LON
surface['sat_azi'] = 0. # Satellite Azimuth Angle
surface['sat_zen'] = 53.1 # OSCAR , SSMIS
surface['sol_zen'] = 0. # Solar Zenith Angle
surface['sol_azi'] = 0. # Solar Azimuth Angle
surface['cltp'] = 500. # Cloud top pressure
surface['clf'] = 0. # Cloud fraction
# right order
surface = surface[cnames].copy()
# CONVERT TO WATER VAPOR PRESSURE in Pa
era['vp'] = sh2vap(era['q'], era['p']) # formula
era['td'] = bolton_inverse(era.vp.values) # default method from Murphy and Koop (iterative)
logic = era['td'] > era['t'] # impossible
era.loc[:, 'td'] = np.where(logic, era['t'].values, era['td'].values)
era.loc[:, 'vp'] = np.where(logic, foeewm(era['t'].values), era['vp'].values) # ? fixes some problems ?
return surface, era
def detection(data, var='t', thres=50, window=1460, dist=730, levels=None, min_levels=3, database=False, freq='12h',
valid_times=[0, 12], verbose=0, **kwargs):
"""Break Detection in timeseries using a Standard Normal Homogeneity Test (SNHT)
Parameters
----------
data Series/DataFrame/Panel Input Radiosonde data (standard)
var str/list Variable to use for detection
thres number Threshold for SNHT
window number Window size for SNHT
dist number Distance between breakpoints
levels list only these levels
min_levels int Minimum required number of levels
database bool
verbose int
kwargs **
Returns
-------
input + var_breaks + var_snht
"""
funcid = '[BD] '
if not isinstance(data, (pd.Series, pd.DataFrame, pd.Panel)):
raise ValueError(funcid + " Require a Series, DataFrame or Panel")
if not isinstance(var, (list, str)):
raise ValueError(funcid + " Variable needs to be a list or string")
if isinstance(var, str):
var = [var] # as list
data = data.copy()
if len(valid_times) == 1:
if freq != '24h':
raise RuntimeError("Frequency must be 24 hours then")
elif len(valid_times) == 2:
if freq != '12h':
raise RuntimeError("Frequency must be 12 hours then")
else:
raise RuntimeWarning("Make sure the freqency fits to the valid times")
if isinstance(data, pd.Series):
data = data.to_frame()
if len(var) > 1:
raise ValueError("Series object has only one variable")
data.columns = var
data = data.ix[np.in1d(data.index.hour, valid_times), :] # VIEW
data = data.resample(freq).asfreq()
# run 1d
found_breaks = _1d_detection(data, var, window, dist, thres, verbose=verbose)
print_verbose(var + ": " + str(filter_series(data['%s_breaks' % var] > 0).unique()), verbose)
return found_breaks, data # dict, dataframe
if isinstance(data, pd.DataFrame):
for ivar in var:
if ivar not in data.columns:
raise ValueError(funcid + "Variable not found: %s in %s" % (ivar, str(data.columns)))
data = database_to_panel(data, levels=levels, verbose=verbose)
if isinstance(data, pd.DataFrame):
# only one level
data = data.ix[np.in1d(data.index.hour, valid_times), :] # VIEW
data = data.resample(freq).asfreq()
found_breaks = {}
for ivar in var:
found_breaks[ivar] = _1d_detection(data, ivar, window, dist, thres, verbose=verbose)
print_verbose(ivar + ": " + str(filter_series(data['%s_breaks' % ivar] > 0).unique()), verbose)
return found_breaks, data # dict, dataframe
# Panel
for ivar in var:
if ivar not in data.items:
raise ValueError(funcid + "Variable not found: %s in %s" % (ivar, str(data.items)))
# Select times:
data = data.loc[:, np.in1d(data.major_axis.hour, valid_times), :]
# Resample to frequency
if data.major_axis.inferred_freq != freq:
print_verbose(funcid + "Resampling to %s frequency" % freq, verbose)
newdates = pd.date_range(data.major_axis[0], data.major_axis[-1], freq=freq)
data = data.reindex(major_axis=newdates)
data.major_axis.name = 'date'
if levels is not None:
data = data.ix[:, :, data.minor_axis.isin(levels)].copy()
print_verbose(funcid + "Selecting only specified levels ...", verbose)
print_verbose(funcid + "p-Levels: " + ",".join(["%d" % (ip / 100) for ip in levels]) + ' hPa', verbose)
# 2D detection
found_breaks = {}
for ivar in var:
found_breaks[ivar] = _2d_detection(data, ivar, window, dist, thres, min_levels,
verbose=verbose)
if verbose > 0:
print "Minimum amount of levels required: ", min_levels
print "Breakpoints"
for ivar in var:
print ivar + ": ", str(data.major_axis[(data['%s_breaks' % ivar] > 0).all(1)])
# Output as Database
if database:
print_verbose(funcid + "as database.", verbose)
data = data.to_frame(filter_observations=False).reset_index().sort_values(by=['date', 'p']).set_index('date', drop=True)
return found_breaks, data # dict, panel / database
def mars_gzip(filename, convert=True, save=False, verbose=0):
"""Read compressed ECMWF Mars dump as done by L. Haimberger,
15.12.2014
"""
from raso.config import rasodir
compressed = True # Can be compressed but might not ?
if not os.path.isfile(filename):
if '.gz' in filename:
if not os.path.isfile(filename.replace('.gz', '')):
raise IOError("File not found: %s" % filename)
filename = filename.replace('.gz', '')
compressed = False
if '.gz' not in filename:
compressed = False
try:
if compressed:
infile = gzip.open(filename, 'rb')
else:
infile = open(filename, 'r')
tmp = infile.read() # alternative readlines (slower)
data = tmp.splitlines() # Memory (faster)
finally:
infile.close()
nl = len(data)
dates = []
nf = len(data[0].strip().replace('NULL', 'NaN').split(' '))
raw = np.zeros((nl, nf), dtype=float)
itx = []
if verbose > 0:
pbar = ProgressBar(maxval=nl)
pbar.start()
# 'i4,i4,i4,i4,i4,f4,f4,f4,f4,i4,f8,f8,f8,f8'))
for i, line in enumerate(data):
# datetime , int , int, float, float .... int float int (flags)
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13
# date time obstype codetype sonde_type ident lat lon stalt press varno obsvalue biascorr fg_depar
# 14 15 16 17
# an_depar status anflag event1
try:
raw[i] = np.float_(line.strip().replace('NULL', 'NaN').split(' '))
dates.append(datetime.datetime.strptime("%8d%04d" % (raw[i, 0], raw[i, 1]), '%Y%m%d%H%M%S'))
itx.append(i)
if verbose > 0:
pbar.update(i + 1)
except:
continue
if verbose > 0:
pbar.finish()
if not convert:
out = pd.DataFrame(data=raw[itx], index=dates)
print_verbose("[MARS] Not converted!", verbose)
print_verbose("[MARS] %20s ( %12d ) <> ( %12d )" % (filename.split('/')[-1], nl, out.shape[0]), verbose)
return out
out = pd.DataFrame(data=raw[np.ix_(itx, [5, 2, 3, 4, 10, 6, 7, 8, 9, 11, 12, 13, 14])], index=dates,
columns=['id', 'obs_type', 'code_type', 'sonde_type', 'varno', 'lat', 'lon', 'alt', 'p', 'value',
'biascorr', 'fg_dep', 'an_dep'])
out.dropna(subset=['value'], inplace=True)
print_verbose("%20s ( %12d ) <> ( %12d )" % (filename.split('/')[-1], nl, out.shape[0]), verbose)
if len(out.id.unique()) > 1:
print_verbose("Multiple IDs: " + str(out.id.unique()), verbose)
return out
if save:
ident = "%06d" % int(out['id'][0])
filename_out = rasodir + '/%s/mars_dump.h5' % ident
out.to_hdf(filename_out, 'mars', format='table', complib="blosc", complevel=9)
print_verbose("[%06d] Saved: %s" % (int(ident), filename_out), verbose)
return out
def save(self, var=None, directory=None, update=False, hdfformat='table', hdfcomplevel=9, hdfcomplib='blosc', verbose=0):
""" Write Radiosonde Class Object to Directory
Parameters
----------
var str/list
directory str
update bool
force bool
mode str
hdfformat str
hdfcomplevel int
hdfcomplib str
verbose int
"""
from raso.config import rasodir
import pickle
if directory is None:
if self.directory is None:
directory = rasodir
else:
directory = self.directory
default = directory + '/%s/' % self.id
if not os.path.isdir(default):
os.makedirs(default)
if var is None:
varis = self.vars # All Variables
else:
if isinstance(var, str):
varis = [var]
else:
varis = var
update = True # Make sure we write it
journal("Saving to dir: %s" % default, self.history, verbose)
for ivar in varis:
written = False
if self.var_saved[ivar] and not update:
if os.path.isfile(default + "%s.h5" % ivar) or os.path.isfile(default + "%s.pickle" % ivar):
print_verbose("%20s [%5s]" % (ivar, color_boolean(written)), verbose)
continue
tmp = getattr(self, ivar)
#
# Try as HDF
#
if isinstance(tmp, (pd.Series, pd.DataFrame, pd.Panel)):
try:
tmp.to_hdf(default + "%s.h5" % ivar, ivar, mode='a', format=hdfformat, complevel=hdfcomplevel, complib=hdfcomplib)
fio = 'H5'
written = True
n = tmp.shape
except Exception as e:
if verbose:
print ivar, " >> ", repr(e)
#
# And as Pickle
#
if not written:
try:
pickle.dump(tmp, open(default + "%s.pickle" % ivar, 'w'))
fio = 'PKL'
written = True
n = len(tmp)
except Exception as e:
if verbose:
print ivar, " >> ", repr(e)
print_verbose("%20s %20s [%5s] %s" % (ivar, n, color_boolean(written), fio), verbose)
attrs = {}
for iatt in self.attrs:
attrs[iatt] = getattr(self, iatt)
pickle.dump(attrs, open(default + 'attributes.pickle', 'w'))
attrs = {}
for s in self.history:
if 'Saved' in s:
self.history.pop(self.history.index(s))
journal("Saved", self.history, 0)
for iatt in ['id', 'is_saved', 'is_empty']:
attrs[iatt] = getattr(self, iatt)
pickle.dump(attrs, open(default + 'radiosonde.pickle', 'w'))
f = open(default+'history.txt', 'w')
f.write("\n".join(self.history))
f.close()
if self.notes != "":
f = open(default + 'notes.txt', 'w')
f.write(self.notes)
f.close()
self.directory = directory
def data(ident, outputdir=None, verbose=0):
""" Automated Data count plots
Parameters
----------
ident
outputdir
verbose
Returns
-------
"""
from raso import load_radiosonde
from raso.config import outdir
import matplotlib.pyplot as plt
yplevs = [1000, 3000, 7000, 12500, 17500, 22500, 30000, 40000, 50000, 60000, 85000, 92500] # Pressure Levels, Ticks
isonde = load_radiosonde(ident, variables=['igra', 'data', 'std_data'])
if outputdir is None:
outputdir = outdir + '/figures'
#
# DATA
#
if 'data' in isonde.vars:
varis = ['t', 'r']
if isonde.data.columns.isin(varis).sum() == len(varis):
dates = pd.Series(1, index=pd.DatetimeIndex(
isonde.data[varis].dropna().index.unique())) # events with t,r variables
else:
varis = isonde.data.columns.tolist()
dates = pd.Series(1, index=pd.DatetimeIndex(isonde.data.index.unique())) # events with all variables
events = dates.groupby(dates.index.to_period(freq='M')).count() # events per month
events.plot(grid=True, kind='area')
plt.title("Radiosonde %s (DATA), Sounding Events (%s)" % (isonde.id, ",".join(varis)))
plt.xlabel("Month")
plt.ylabel("Soundings per Month [#]")
plt.savefig('%s/%s_data_counts.pdf' % (outputdir, ident))
print_verbose('%s/%s_data_counts.pdf' % (outputdir, ident), verbose)
plt.close()
#
# IGRA
#
if 'igra' in isonde.vars:
varis = ['t', 'dpd']
if isonde.igra.columns.isin(varis).sum() == len(varis):
dates = pd.Series(1, index=pd.DatetimeIndex(
isonde.igra[varis].dropna().index.unique())) # events with t,r variables
else:
varis = isonde.igra.columns.tolist()
dates = pd.Series(1, index=pd.DatetimeIndex(isonde.igra.index.unique())) # events with all variables
events = dates.groupby(dates.index.to_period(freq='M')).count() # events per month
events.plot(grid=True, kind='area')
plt.title("Radiosonde %s (IGRA), Sounding Events (%s)" % (isonde.id, ",".join(varis)))
plt.xlabel("Month")
plt.ylabel("Soundings per Month [#]")
plt.savefig('%s/%s_igra_counts.pdf' % (outputdir, ident))
print_verbose('%s/%s_igra_counts.pdf' % (outputdir, ident), verbose)
plt.close()
#
# STD DATA
#
if 'std_data' in isonde.vars:
varis = ['t', 'r']
if isonde.std_data.columns.isin(varis).sum() == len(varis):
dates = pd.Series(1, index=pd.DatetimeIndex(
isonde.std_data[varis].dropna().index.unique())) # events with t,r variables
else:
varis = isonde.std_data.columns.tolist()
dates = pd.Series(1, index=pd.DatetimeIndex(isonde.std_data.index.unique())) # events with all variables
events = dates.groupby(dates.index.to_period(freq='M')).count() # events per month
events.plot(grid=True, kind='area')
plt.title("Radiosonde %s (STDDATA), Sounding Events (%s)" % (isonde.id, ",".join(varis)))
plt.xlabel("Month")
plt.ylabel("Soundings per Month [#]")
plt.savefig('%s/%s_std_data_counts.pdf' % (outputdir, ident))
plt.close()
#
for ivar in isonde.std_data.columns.tolist():
if ivar in ['p', 'u', 'v', 'qual']:
continue
timeseries_var(isonde.std_data, ivar, freq='M', title='RASO %s' % isonde.id, yticklabels=yplevs)
plt.savefig("%s/%s_%s_contour.pdf" % (outputdir, ident, ivar))
plt.close()
plt.close('all')
def corrections(ident, levels=None, outputdir=None, verbose=0):
"""
Parameters
----------
ident
levels
outputdir
verbose
Returns
-------
"""
from raso import load_radiosonde
from raso.config import outdir
import matplotlib.pyplot as plt
yplevs = [1000, 3000, 7000, 12500, 17500, 22500, 30000, 40000, 50000, 60000, 85000, 92500] # Pressure Levels, Ticks
isonde = load_radiosonde(ident, variables=[])
if outputdir is None:
outputdir = outdir + '/figures'
#
# DATA
#
if 'detect_opts' not in isonde.attrs:
print "No Detection Options found ?", ident
return
if levels is None:
levels = [10000, 30000, 50000, 60000, 70000, 85000, 92500]
for icor, iopts in isonde.detect_opts.items():
iname = iopts['savename']
isonde.load_data(iname)
ithres = iopts['thres']
ivar = iopts['variable']
idata = getattr(isonde, iname)
daynight = iopts.get('daynight', False)
#
# CONTOUR
#
for j, jvar in enumerate(['%s_mcor' % ivar, '%s_qcor' % ivar, '%s_qecor' % ivar]):
if jvar not in idata.items:
continue
if daynight:
timeseries_var(idata[:, idata.major_axis.hour == 12, :], jvar, freq='M', yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_12Z.pdf" % (outputdir, ident, iname, jvar))
print_verbose("%s/%s_%s_%s_12Z.pdf" % (outputdir, ident, iname, jvar), verbose)
timeseries_anomaly(idata[:, idata.major_axis.hour == 12, :], jvar, '%s_era' % ivar, freq='M',
yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_dep_12Z.pdf" % (outputdir, ident, iname, jvar))
print_verbose("%s/%s_%s_%s_dep_12Z.pdf" % (outputdir, ident, iname, jvar), verbose)
timeseries_var(idata[:, idata.major_axis.hour == 0, :], jvar, freq='M', yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_00Z.pdf" % (outputdir, ident, iname, jvar))
print_verbose("%s/%s_%s_%s_00Z.pdf" % (outputdir, ident, iname, jvar), verbose)
timeseries_anomaly(idata[:, idata.major_axis.hour == 0, :], jvar, '%s_era' % ivar, freq='M',
yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_dep_00Z.pdf" % (outputdir, ident, iname, jvar))
print_verbose("%s/%s_%s_%s_dep_00Z.pdf" % (outputdir, ident, iname, jvar), verbose)
else:
timeseries_var(idata, jvar, freq='M', yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s.pdf" % (outputdir, ident, iname, jvar))
print_verbose("%s/%s_%s_%s.pdf" % (outputdir, ident, iname, jvar), verbose)
timeseries_anomaly(idata, jvar, '%s_era' % ivar, freq='M', yticklabels=yplevs)
plt.savefig("%s/%s_%s_%s_dep.pdf" % (outputdir, ident, iname, jvar))
print_verbose("%s/%s_%s_%s_dep.pdf" % (outputdir, ident, iname, jvar), verbose)
#
# LEVEL
#
for ilev in levels:
if daynight:
fig, ax = plt.subplots(3, 1, figsize=(12, 6), sharex=True, sharey=True)
varnames = ['M', 'Q', 'QE']
ym = int(idata["%s_dep" % ivar, idata.major_axis.hour == 12, ilev].mean())
for j, jvar in enumerate(['%s_mcor' % ivar, '%s_qcor' % ivar, '%s_qecor' % ivar]):
if jvar not in idata.items:
continue
timeseries_line_correction(idata[:, idata.major_axis.hour == 12, :], jvar,
'%s_dep_snht' % ivar, ilev, '%s dep [K]' % ivar,
breaks='%s_dep_breaks' % ivar, ax=ax[j],
departures='%s_era' % ivar, ylim=(ym - 5, ym + 5),
force_snht=True, post_snht=True, plabel=None)
idata['%s_dep' % ivar, idata.major_axis.hour == 12, ilev].resample('12h').mean().rolling(window=60,
min_periods=1).mean().plot(
ax=ax[j], color='gray', zorder=1)
ax[j].set_xlim('1979', '2016')
ax[j].set_ylabel(varnames[j] + ' dep (ERA) [K]')
ax[j].grid(True)
ax[0].set_title('Corrections at %d hPa 12Z' % (ilev / 100))
ax[-1].set_xlabel('Time (RM:60)')
plt.savefig('%s/%s_%s_%s_corr_12Z_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)))
print_verbose('%s/%s_%s_%s_corr_12Z_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)), verbose)
fig, ax = plt.subplots(3, 1, figsize=(12, 6), sharex=True, sharey=True)
ym = int(idata["%s_dep" % ivar, idata.major_axis.hour == 0, ilev].mean())
for j, jvar in enumerate(['%s_mcor' % ivar, '%s_qcor' % ivar, '%s_qecor' % ivar]):
if jvar not in idata.items:
continue
timeseries_line_correction(idata[:, idata.major_axis.hour == 0, :], jvar,
'%s_dep_snht' % ivar, ilev, '%s dep [K]' % ivar,
breaks='%s_dep_breaks' % ivar, ax=ax[j],
departures='%s_era' % ivar, ylim=(ym - 5, ym + 5),
force_snht=True, post_snht=True, plabel=None)
idata['%s_dep' % ivar, idata.major_axis.hour == 0, ilev].resample('12h').mean().rolling(window=60,
min_periods=1).mean().plot(
ax=ax[j], color='gray', zorder=1)
ax[j].set_xlim('1979', '2016')
ax[j].set_ylabel(varnames[j] + ' dep (ERA) [K]')
ax[j].grid(True)
ax[0].set_title('Corrections at %d hPa 00Z' % (ilev / 100))
ax[-1].set_xlabel('Time (RM:60)')
plt.savefig('%s/%s_%s_%s_corr_00Z_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)))
print_verbose('%s/%s_%s_%s_corr_00Z_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)), verbose)
else:
fig, ax = plt.subplots(3, 1, figsize=(12, 6), sharex=True, sharey=True)
varnames = ['M', 'Q', 'QE']
ym = int(idata["%s_dep" % ivar, :, ilev].mean())
for j, jvar in enumerate(['%s_mcor' % ivar, '%s_qcor' % ivar, '%s_qecor' % ivar]):
if jvar not in idata.items:
continue
timeseries_line_correction(idata, jvar, '%s_dep_snht' % ivar, ilev, '%s dep [K]' % ivar,
breaks='%s_dep_breaks' % ivar, ax=ax[j],
departures='%s_era' % ivar, ylim=(ym - 5, ym + 5),
force_snht=True, post_snht=True, plabel=None)
idata['%s_dep' % ivar, :, ilev].resample('12h').mean().rolling(window=60,
min_periods=1).mean().plot(ax=ax[j],
color='gray',
zorder=1)
ax[j].set_xlim('1979', '2016')
ax[j].set_ylabel(varnames[j] + ' dep (ERA) [K]')
ax[j].grid(True)
ax[0].set_title('Corrections at %d hPa' % (ilev / 100))
ax[-1].set_xlabel('Time (RM:60)')
plt.savefig('%s/%s_%s_%s_corr_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)))
print_verbose('%s/%s_%s_%s_corr_%04d.pdf' % (outputdir, ident, iname, ivar, (ilev / 100)), verbose)
plt.close('all')
def from_store_deprecated(ident, filename=None, variables=None, directory=None, attributes=True, verbose=0, **kwargs):
"""Read previously saved Radiosonde from default directory
pandas HDFStore HDF5
Parameters
----------
ident Radiosonde Station Number
filename filename of HDFStore
variables select only given variables
directory directory to search for raso_######.h5 with station number
Returns
-------
radiosonde class object
"""
from raso.sonde.radiosonde import radiosonde
from raso.config import rasodir, wkdir
#
if isinstance(ident, (float, int)):
ident = "%06d" % ident
#
if filename is not None:
if 'h5' not in filename:
raise ValueError("This routine can only read HDF5 files from pandas!")
default = filename
elif directory is None:
default = rasodir + '/raso_%s.h5' % ident
else:
default = directory + '/raso_%s.h5' % ident
print_verbose("[STORE] %s" % default, verbose)
#
if not os.path.isfile(default):
raise IOError("File %s not found" % default.replace(wkdir, '.'))
#
out = radiosonde(id=ident)
metadata = False
if variables is not None:
if isinstance(variables, str):
variables = [variables] # convert to list
#
with pd.HDFStore(default, complevel=9, complib='blosc') as store:
for key in store.keys():
# drop /
if key[1:] == 'metadata':
metadata = True
continue
elif variables is not None:
if key[1:] in variables:
out.add_data(key[1:], store[key]) # only certain variables
else:
out.add_data(key[1:], store[key]) # default
print_verbose("[STORE] %s " % key, verbose)
#
if metadata:
print_verbose("[STORE] METADATA ", verbose)
imeta = store.get_storer('metadata')
#
# Attributes
if 'metadata' in imeta.attrs:
for ikey, ival in imeta.attrs['metadata'].items():
if ikey in ['metadata']:
continue # ? do we need that?
if attributes:
out.add_attr(ikey, ival)
elif ikey in ['lon', 'lat', 'alt']:
out.add_attr(ikey, ival)
else:
pass
#
# Infos (id history, saved)
if 'iparams' in imeta.attrs:
for ikey, ival in imeta.attrs['iparams'].items():
if ikey in ['infos']:
continue
setattr(out, ikey, ival)
out.filename = default
return out
def from_store(ident, variables=None, directory=None, verbose=0):
""" Read Radiosonde from Store
Skips any directory or file with extract in their name
As well as mars_dump.h5
Parameters
----------
ident str
variables list
directory str
verbose int
Returns
-------
radiosonde
Examples
--------
>>>import raso
>>>raso.read.from_store('011035')
"""
from raso.sonde.radiosonde import radiosonde
from raso.config import rasodir
if directory is None:
directory = rasodir
default = directory + '/%s/' % ident
if not os.path.isdir(default):
raise IOError("[from_store] Requires a valid directory: %s" % default)
if variables is not None:
if isinstance(variables, str):
variables = [variables]
out = radiosonde(id=ident)
for ifile in os.listdir(default):
if ifile == 'attributes.pickle':
attrs = pickle.load(open(default + 'attributes.pickle'))
for ikey, ival in attrs.items():
out.add_attr(ikey, ival)
elif ifile == 'radiosonde.pickle':
attrs = pickle.load(open(default + 'radiosonde.pickle'))
for ikey, ival in attrs.items():
setattr(out, ikey, ival)
elif ifile == 'history.txt':
f = open(default + 'history.txt')
setattr(out, 'history', f.read().splitlines())
f.close()
elif ifile == 'notes.txt':
f = open(default + 'notes.txt')
setattr(out, 'notes', f.read())
f.close()
elif ifile == 'mars_dump.h5':
print_verbose("%20s Skipped" % ifile, verbose)
continue
elif os.path.isdir(default+ifile):
continue # any directory is ignored
elif 'extract' in ifile:
print_verbose("%20s Skipped" % ifile, verbose) # extract is a codename for temp files
continue
else:
varname = ifile.split('.')[0]
if variables is not None:
if varname not in variables:
print_verbose("%20s [%5s]" % (varname, color_boolean(False)), verbose)
continue
# TODO add file timestamp as info
# print varname, "created: %s" % time.ctime(os.path.getctime(default + ifile)), time.ctime(os.path.getmtime(default + ifile))
if 'h5' in ifile:
out.add_data(varname, pd.read_hdf(default + ifile, varname), verbose=-1) # not reported
else:
out.add_data(varname, pickle.load(open(default + ifile)), verbose=-1) # not reported
out.var_saved[varname] = True
print_verbose("%20s [%5s]" % (varname, color_boolean(True),), verbose)
out.directory = directory
out.is_saved = True
return out
def merge(isonde, jsonde, update=False, save=False, verbose=0):
"""
Read and merge two radiosonde class to one radiosonde
Parameters
----------
isonde radiosonde
jsonde radiosonde
update bool
save bool
verbose int
Returns
-------
radiosonde / save
"""
from . import radiosonde
if not isinstance(isonde, radiosonde):
raise ValueError("[MERGE] Requires a radiosonde class: isonde")
if not isinstance(jsonde, radiosonde):
raise ValueError("[MERGE] Requires a radiosonde class: jsonde")
for ivar in jsonde.vars:
if ivar in isonde.vars:
if update:
setattr(isonde, ivar, getattr(jsonde, ivar))
print_verbose("[MERGE] %s replaced [%s]" % (ivar, color_boolean(update)), verbose)
else:
isonde.add(ivar, getattr(jsonde, ivar))
print_verbose("[MERGE] New variable: %s" % ivar, verbose)
# Attributes ?
for iatt in jsonde.attrs:
if iatt in isonde.attrs:
if getattr(isonde, iatt) == getattr(jsonde, iatt):
continue
elif iatt in ['history', 'is_saved']:
continue
elif update:
isonde.add_attr(iatt, getattr(jsonde, iatt))
else:
isonde.add_attr("new_" + iatt, getattr(jsonde, iatt))
print_verbose("[MERGE] %s replaced [%s]" % (iatt, color_boolean(update)), verbose)
else:
isonde.add_attr(iatt, getattr(jsonde, iatt))
print_verbose("[MERGE] New Attribute: %s " % ("new_" + iatt), verbose)
# History ?
isonde.history = history_duplicates(isonde.history + jsonde.history)
if save:
isonde.save(verbose=verbose)
else:
isonde.is_saved = False
return isonde
def from_gruan(sonde, directory=None, save=True,
var=['cor_temp', 'temp', 'press', 'cor_rh', 'rh', 'WVMR', 'alt', 'u_temp', 'u_rh', 'u_press'],
force=False, attach=None, attach_var=None, verbose=0, **kwargs):
"""
Read GRUAN Data and convert to a radiosonde class object
Read and convert time to std_times, rename temp and press to t and p.
Drop duplicate entries.
Possible Station Names:
'BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU',
'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD', 'TAT'
Parameters
----------
sonde str Appreviation of Sonde Station
directory str Directory of GRUAN data
var list of variables to include
save bool Save radiosonde to store ?
force bool Force a raw data reread ?
attach radiosonde Attach to radiosonde class object
attach_var list Variables to attach from HDFStore
verbose int Level of verbosity
kwargs dict Additional Keywords
Returns
-------
radiosonde class object
Raises
------
ValueError sonde not in possible Stations
"""
from raso.standard_dates_times import _fix_datetime
from raso.config import outdir
from raso import radiosonde
from . import from_store
from ncio import read_netcdf, read_history
avail_sondes = ['BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU', 'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD',
'TAT']
if not isinstance(sonde,str):
raise ValueError("Requires a str: sonde")
if attach is not None:
if not isinstance(attach,radiosonde):
raise ValueError("Requires a radiosonde class object: attach")
if sonde not in avail_sondes:
raise ValueError("Only one of %s allowed!" % str(avail_sondes))
filename = outdir + "/GRUAN_" + sonde + ".h5"
if os.path.isfile(filename) and not force:
print "Recovering: %s"%filename
isonde = from_store(0, filename=filename)
if attach is not None:
if attach_var is None:
attach_var = isonde.vars
for ivar in isonde.vars:
if ivar in attach_var:
attach.add_data(ivar, getattr(isonde,ivar), history="GRUAN, %s %s" %(sonde, ivar))
return
else:
return isonde
if directory is None:
raise RuntimeError("[GRUAN] requires a directory!")
#
print_verbose("[GRUAN] Find files ...", verbose)
files = find_files(directory, '*' + sonde + '*.nc', recursive=True)
print_verbose("[GRUAN][%s] Files found: %d" % (sonde, len(files)), verbose)
data = []
if verbose > 0:
pbar = ProgressBar(maxval=len(files))
pbar.start()
i = 0
for ifile in files:
tmp = pd.DataFrame(read_netcdf(ifile, verbose=verbose - 2)) # convert it all to a big Frame
# select only necessary information
tmp = tmp[ var ]
tmp.index.name = 'orig_date' # retain original sounding times
tmp.rename(columns=lambda x: x.replace('temp', 't').replace('press', 'p').replace('rh','r'), inplace=True) # rename
tmp.p *= 100. # from hPa to Pa
# WVMR to q
tmp['q'] = tmp.eval("WVMR/(WVMR+1)") # specific humidity
tmp['date'] = tmp.index.to_period(freq='h').to_datetime() # truncate to hours
tmp = tmp.reset_index().set_index('date') # change index
rx = map(_fix_datetime, tmp.index) # change time to 0, 6, 12, 18 UTC
tmp.index = rx
tmp.drop_duplicates(inplace=True)
data.append(tmp)
if verbose > 0:
pbar.update(i + 1)
i += 1
if verbose > 0:
pbar.finish()
hist = read_history(ifile) #
try:
ident = "%06d" % int(hist['g.General.SiteWmoId'])
except:
ident = sonde
lat = float(hist['g.MeasuringSystem.Latitude'].split(' ')[0])
lon = float(hist['g.MeasuringSystem.Longitude'].split(' ')[0])
alt = float(hist['g.MeasuringSystem.Altitude'].split(' ')[0])
#
print_verbose("[GRUAN] Concatenate data ...",verbose)
data = pd.concat(data, axis=0)
# drop duplicates
# print_verbose("[GRUAN] Drop duplicates ...", verbose)
# data.drop_duplicates(inplace=True)
#
if attach is None:
print_verbose("[GRUAN] Create radiosonde object ...", verbose)
isonde = radiosonde(ident)
isonde.filename = filename
isonde.add_attr('lon', lon)
isonde.add_attr('lat', lat)
isonde.add_attr('alt', alt)
isonde.add_attr('is_gruan',True)
isonde.add_data('gruan', data, history="GRUAN, %s " %(sonde))
if save:
isonde.save(filename=filename, verbose=verbose)
return isonde
else:
attach.add_data('gruan', data, history="GRUAN, %s " %(sonde))
if save:
attach.save(verbose=verbose)
return
def detect_and_correct(isonde, data='std_data', var='dpd', quality_controlled=True, savename='sdcor', save=True,
tfirst=True, daynight=False, verbose=0, **kwargs):
"""Detect and Correct Radiosonde biases from Departure Statistics
Use ERA-Interim departures to detect breakpoints and
correct these with a mean and a quantile adjustment going back in time.
uses raso.timeseries.breakpoint.detection / correction
Parameters
----------
isonde radiosonde Radiosonde class object
data str Radiosonde variable
var str Variable
quality_controlled bool Use QC to remove flagged values?
savename str store name
save bool Save?
tfirst bool Correct Temperature first ?
daynight bool Correct Day and Night Soundings separate?
verbose int verbosness
Additional Parameters
---------------------
thres int [50] SNHT Threshold
quantilen list/array [0-100] Quantile Ranges
levels list [None] Pressure levels
sample_size int [730] minimum Sample size
borders int [180] biased sample before and after a break
bias30k bool [T] remove 30K Bias for dpd ?
verbose int [0] verboseness
kwargs dict breakpoint.detection, breakpoint.correction ...
Returns
-------
"""
from ..detect_and_correct import detect_and_correct as detect_and_correct_data
from ..detect_and_correct import detect_and_correct_daynight
from .. import standard_dewpoint_depression, standard_water_vapor
funcid = "[DC] Sonde "
if not isinstance(isonde, radiosonde):
raise ValueError(funcid + "requires a radiosonde object!")
if isonde.is_empty:
raise ValueError(funcid + "Radiosonde is empty!")
funcid = "[DC] %s " % isonde.id
if data not in isonde.vars:
raise ValueError(funcid + "Required variable (%s) not present" % data)
prof = getattr(isonde, data).copy() # GET DATA
print_verbose(funcid+"Savename: %s" % savename, verbose)
if hasnames(prof, 'qual'):
#
# drop all the values that have a qual flag
#
if quality_controlled:
journal(funcid + "QC of %s " % data, isonde.history, verbose)
prof = enforcer(prof) # Work on FLAGS, but not DPD 30, makes a copy
#
# set DPD30 to missing
#
if hasnames(prof, 'dpd'):
prof['dpd'] = np.where(prof.qual.str.contains('3'), np.nan, prof.dpd.values) # Apply? BAD, GOOD
if hasnames(prof, 'td'):
prof['td'] = np.where(prof.qual.str.contains('3'), np.nan, prof.td.values) # Apply? BAD, GOOD
del prof['qual'] # prof.drop('qual', 1, inplace=True) # remove all flag information
print_verbose(funcid + " dropping qual ...", verbose)
if hasnames(prof, 'orig'):
del prof['orig'] # prof.drop('orig', 1, inplace=True) # indicates interpolated or not
if 'dpd' in var:
if not hasnames(prof, 'dpd_era'):
prof['dpd_era'] = prof['t_era'] - prof['td_era']
print_verbose(funcid + " Calculating dpd_era ...", verbose)
tbreaks = None
if var == 't' and tfirst:
tfirst = False
print_verbose(funcid + "tfirst=True only with temperature dependent variables", 1)
if tfirst:
journal(funcid + "Running T D&C first! ", isonde.history, verbose)
#
# Only Mean Adjustment for Temperature
#
if not daynight:
prof, tbreaks = detect_and_correct_data(prof, var='t', correct_q=False, bounded=None,
report=isonde.history, verbose=verbose - 1, **kwargs)
else:
prof, tbreaks = detect_and_correct_daynight(prof, var='t', correct_q=False, bounded=None,
report=isonde.history, verbose=verbose - 1, **kwargs)
tbreaks['breaks'] = tbreaks['00Z'] + tbreaks['12Z'] # 00Z and 12Z breaks
#
# new Columns: t_mcor, t_dep, t_dep_breaks, t_dep_snht
#
if len(tbreaks['breaks']) > 0:
journal(funcid + "T-breaks: %s" % str(tbreaks['breaks']), isonde.history, verbose)
# prof.major_axis.name = 'date'
# prof.minor_axis.name = 'p'
# prof = panel_to_database(prof) # to DataFrame -> Function requires it > Deprecated now
# Recalculate Temperature Dependent Variables:
prof = standard_water_vapor(prof, tvar='t_mcor', vpvar='vp_tcor', replace=True, report=isonde.history)
prof = standard_dewpoint_depression(prof, tvar='t_mcor', dpdvar='dpd_tcor', vpvar='vp_tcor',
tdvar='td_tcor', replace=True, report=isonde.history)
del prof['qual'] # prof.drop('qual', 1, inplace=True) # remove quality Flag again
# prof.rename(items={var: '%s_orig' % var}, inplace=True) # Rochade
# prof.rename(items={'%s_tcor' % var: var}, inplace=True) #
#
if hasnames(prof, '%s_tcor' % var):
journal(funcid + "Running t-correct %s D&C ..." % var, isonde.history, verbose)
prof['%s_tcor_era' % var] = prof['%s_era' % var]
if not daynight:
prof, dbreaks = detect_and_correct_data(prof, var='%s_tcor' % var, report=isonde.history,
verbose=verbose - 1, **kwargs)
else:
prof, dbreaks = detect_and_correct_daynight(prof, var='%s_tcor' % var, report=isonde.history,
verbose=verbose - 1, **kwargs)
dbreaks['breaks'] = dbreaks['00Z'] + dbreaks['12Z'] # Combine
del prof['%s_tcor_era' % var]
# prof.rename(items={var: '%s_tcor' % var,
# '%s_mcor' % var: '%s_tcor_mcor' % var,
# '%s_qcor' % var: '%s_tcor_qcor' % var,
# '%s_qecor' % var: '%s_tcor_qecor' % var,
# '%s_dep' % var: '%s_tcor_dep' % var,
# '%s_dep_breaks' % var: '%s_tcor_dep_breaks' % var,
# '%s_dep_snht' % var: '%s_tcor_dep_snht' % var,
# '%s_orig' % var: var}, inplace=True)
else:
journal(funcid + "No T breakpoints. ", isonde.history, verbose)
# prof = panel_to_database(prof) # Convert to DataFrame (after Detection) > Deprecated now
journal(funcid + "Running %s D&C" % var, isonde.history, verbose)
if not daynight:
prof, stat = detect_and_correct_data(prof, var=var, report=isonde.history, verbose=verbose, **kwargs)
else:
prof, stat = detect_and_correct_daynight(prof, var=var, report=isonde.history, verbose=verbose, **kwargs)
stat['breaks'] = stat['00Z'] + stat['12Z'] # Combine lists
#
isonde.add_data(savename, prof, replace=True, verbose=verbose) # DID anything change ?
#
# Options
#
thres = kwargs.get('thres', 50)
borders = kwargs.get('borders', 180)
sample_size = kwargs.get('sample_size', 730)
quantilen = kwargs.get('quantilen', np.arange(0, 101, 10)) # missing from history > add
journal(funcid + "%s (T:%d, N:%d, B:%d, Q:%d)" % (var, int(thres), sample_size, borders, len(quantilen)),
isonde.history, 0)
if tbreaks is not None:
if len(tbreaks['breaks']) > 0:
stat['t-breaks'] = tbreaks['breaks']
if len(dbreaks['breaks']) > 0:
stat['t-dpd-breaks'] = dbreaks['breaks']
stat['thres'] = thres
stat['borders'] = borders
stat['sample_size'] = sample_size
stat['quantilen'] = quantilen
stat['source'] = data
stat['savename'] = savename
stat['variable'] = var
stat['daynight'] = daynight
if 'detect_opts' not in isonde.attrs:
isonde.add_attr('detect_opts', {"%s_%s" % (savename, var): stat}) # sdcor_dpd
else:
isonde.detect_opts["%s_%s" % (savename, var)] = stat # update
if save:
isonde.save(var=savename, update=True, verbose=verbose)
return stat
def breakpoints(ident, levels=None, outputdir=None, verbose=0):
"""
Parameters
----------
ident
levels
outputdir
verbose
Returns
-------
"""
from raso import load_radiosonde
from raso.config import outdir
import matplotlib.pyplot as plt
isonde = load_radiosonde(ident, variables=[])
if outputdir is None:
outputdir = outdir + '/figures'
#
# DATA
#
if 'detect_opts' not in isonde.attrs:
print "No Detection Options found ?", ident
return
if levels is None:
levels = [10000, 30000, 50000, 60000, 70000, 85000, 92500]
for icor, iopts in isonde.detect_opts.items():
iname = iopts['savename']
isonde.load_data(iname)
ithres = iopts['thres']
ivar = iopts['variable']
idata = getattr(isonde, iname)
daynight = iopts.get('daynight', False)
#
# LEVEL
#
for ilev in levels:
if daynight:
ym = int(idata[ivar, idata.major_axis.hour == 12, ilev].mean())
breakpoint_timeseries(idata[:, idata.major_axis.hour == 12, :], ivar, p=ilev, post_snht=True,
window=365, min_periods=1, ylim=(ym - 5, ym + 5))
plt.savefig("%s/%s_%s_%s_12Z_breakstat_%04d.pdf" % (outputdir, ident, iname, ivar, ilev / 100))
print_verbose("%s/%s_%s_%s_12Z_breakstat_%04d.pdf" % (outputdir, ident, iname, ivar, ilev / 100),
verbose)
ym = int(idata[ivar, idata.major_axis.hour == 0, ilev].mean())
breakpoint_timeseries(idata[:, idata.major_axis.hour == 0, :], ivar, p=ilev, post_snht=True,
window=365, min_periods=1, ylim=(ym - 5, ym + 5))
plt.savefig("%s/%s_%s_%s_00Z_breakstat_%04d.pdf" % (outputdir, ident, iname, ivar, ilev / 100))
print_verbose("%s/%s_%s_%s_00Z_breakstat_%04d.pdf" % (outputdir, ident, iname, ivar, ilev / 100),
verbose)
else:
ym = int(idata[ivar, :, ilev].mean())
breakpoint_timeseries(idata, ivar, p=ilev, post_snht=True, window=730, min_periods=1,
ylim=(ym - 5, ym + 5))
plt.savefig("%s/%s_%s_%s_breakstat_%04d.pdf" % (outputdir, ident, iname, ivar, ilev / 100))
print_verbose("%s/%s_%s_%s_breakstat_%04d.pdf" % (outputdir, ident, iname, ivar, ilev / 100), verbose)
plt.close('all')
def from_gpsro(directory, pattern, ident, radius, save=True, esat='murphy_koop', verbose=0):
"""
Read GPS RO events into a timeseries for a certain location
Parameters
----------
directory str search directory
pattern str search pattern
ident int/str radiosonde ID
radius float allowed distance to ilon, ilat of GPS RO events [km]
save bool save radiosonde class object
verbose int verbosity
Raises
------
ValueError wrong input types
RuntimeError no GPS RO event within specified distance
"""
from . import from_store
from raso.config import rasodir
from raso import radiosonde
if not isinstance(directory, str):
raise ValueError("Require a string: directory")
if not isinstance(pattern, str):
raise ValueError("Require a string: pattern")
if not isinstance(ident, (int, str, float)):
raise ValueError("Require a int, str, float: ident")
if not isinstance(radius, (float, int)):
raise ValueError("Require a int, float: radius")
if isinstance(ident, (int, float)):
ident = "%06d" % ident
name = "gpsro_%06d" % int(radius)
iname = "gpsro_infos_%06d" % int(radius)
filename = rasodir + "/%s/%s.h5" % (ident, name)
# RECOVER
if os.path.isfile(filename):
print_verbose("Recovering previous: %s" % filename, verbose)
sonde = from_store(ident, variables=[name, iname])
return sonde
if not os.path.isdir(rasodir + "/%s" % ident):
os.makedirs(rasodir + "/%s" % ident)
try:
sonde = from_store(ident, variables=[]) # Reads the empty Radiosonde class >> IOError
ilon = sonde.lon # >> Attribute Error
ilat = sonde.lat
except IOError:
sonde = radiosonde(ident) # Make a new one
sondes = radiosondelist()
ilon = sondes.ix[ident, 'lon'].values
ilat = sondes.ix[ident, 'lat'].values
except AttributeError:
sondes = radiosondelist()
ilon = sondes.ix[ident, 'lon'].values
ilat = sondes.ix[ident, 'lat'].values
if ilon is None or ilat is None:
raise RuntimeError("Can't find any useful lon, lat Information on Radiosonde: %s" % ident)
infos, data = read_nearest_events(directory, pattern, ilon, ilat, radius, esat=esat, verbose=verbose)
journal("GPSRO Data: r=%d km [%s/%s]" % (radius, directory, pattern), sonde.history, verbose)
sonde.add_attr('lon', ilon)
sonde.add_attr('lat', ilat)
sonde.add_data(iname, infos)
sonde.add_data(name, data)
if save:
sonde.save(verbose=verbose)
else:
return sonde
def trends(ident, levels=None, outputdir=None, verbose=0):
"""
Parameters
----------
ident
levels
outputdir
verbose
Returns
-------
"""
from raso import load_radiosonde
from raso.config import outdir
import matplotlib.pyplot as plt
yplevs = [1000, 3000, 7000, 12500, 17500, 22500, 30000, 40000, 50000, 60000, 85000, 92500] # Pressure Levels, Ticks
isonde = load_radiosonde(ident, variables=[])
if outputdir is None:
outputdir = outdir + '/figures'
#
# DATA
#
if levels is None:
levels = [10000, 30000, 50000, 60000, 70000, 85000, 92500]
if 'detect_opts' not in isonde.attrs:
print "No Detection Options found ?", ident
return
for icor, iopts in isonde.detect_opts.items():
iname = iopts['savename']
isonde.load_data(iname)
ivar = iopts['variable']
idata = getattr(isonde, iname)
daynight = iopts.get('daynight', False)
#
# Profile
#
varis = [ivar, '%s_mcor' % ivar, '%s_qcor' % ivar, '%s_qecor' % ivar, '%s_era_adj' % ivar, '%s_era' % ivar]
varnames = ['U', 'M', 'Q', 'QE', 'EA', 'E']
# Period 1979 - 2015 (32y)
if daynight:
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 12, :], slice('1979', '2015'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_12Z_79-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_12Z_79-15.pdf' % (outputdir, ident, iname, ivar), verbose)
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 0, :], slice('1979', '2015'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_00Z_79-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_00Z_79-15.pdf' % (outputdir, ident, iname, ivar), verbose)
else:
ax, lgd = _trend_plotter(idata[varis], slice('1979', '2015'), varis, yplevs, varnames)
plt.savefig('%s/%s_%s_%s_trends_79-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_79-15.pdf' % (outputdir, ident, iname, ivar), verbose)
# Period 1990 - 2015 (15y)
if daynight:
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 12, :], slice('1990', '2015'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_12Z_90-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_12Z_90-15.pdf' % (outputdir, ident, iname, ivar), verbose)
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 0, :], slice('1990', '2015'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_00Z_90-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_00Z_90-15.pdf' % (outputdir, ident, iname, ivar), verbose)
else:
ax, lgd = _trend_plotter(idata[varis], slice('1990', '2015'), varis, yplevs, varnames)
plt.savefig('%s/%s_%s_%s_trends_79-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_90-15.pdf' % (outputdir, ident, iname, ivar), verbose)
# Period 1980 - 2000 (20y)
if daynight:
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 12, :], slice('1980', '2000'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_12Z_80-00.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_12Z_80-00.pdf' % (outputdir, ident, iname, ivar), verbose)
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 0, :], slice('1980', '2000'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_00Z_80-00.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_00Z_80-00.pdf' % (outputdir, ident, iname, ivar), verbose)
else:
ax, lgd = _trend_plotter(idata[varis], slice('1980', '2000'), varis, yplevs, varnames)
plt.savefig('%s/%s_%s_%s_trends_80-00.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_80-00.pdf' % (outputdir, ident, iname, ivar), verbose)
# Period 1975 - 2015 (30y)
if daynight:
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 12, :], slice('1985', '2015'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_12Z_85-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_12Z_85-15.pdf' % (outputdir, ident, iname, ivar), verbose)
ax, lgd = _trend_plotter(idata[varis, idata.major_axis.hour == 0, :], slice('1985', '2015'), varis, yplevs,
varnames)
plt.savefig('%s/%s_%s_%s_trends_00Z_85-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_00Z_85-15.pdf' % (outputdir, ident, iname, ivar), verbose)
else:
ax, lgd = _trend_plotter(idata[varis], slice('1985', '2015'), varis, yplevs, varnames)
plt.savefig('%s/%s_%s_%s_trends_85-15.pdf' % (outputdir, ident, iname, ivar), bbox_extra_artists=(lgd,),
bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_85-15.pdf' % (outputdir, ident, iname, ivar), verbose)
plt.close('all')
#
# LEVEL
#
for ilev in levels:
if daynight:
fig, ax = plt.subplots(len(varis), 1, figsize=(12, len(varis)), sharex=True, sharey=True)
ym = int(idata[ivar, idata.major_axis.hour == 12, ilev].mean())
for j, jvar in enumerate(varis):
if jvar in idata.items:
ax[j] = timeseries_line_trend(idata[:, idata.major_axis.hour == 12, :], jvar,
'%s_dep_snht' % ivar, ilev, '%s [K]' % varnames[j], ax=ax[j],
plabel=ym + 4, ylim=(ym - 5, ym + 5), window=365,
label=varnames[j])
ax[j].grid(True)
# lgd = ax.legend(bbox_to_anchor=(1.1, 1), loc='upper left', fontsize=10)
ax[j].set_xlim('1979', '2016')
ax[-1].set_xlabel('Time (RM:365)')
plt.savefig('%s/%s_%s_%s_trends_12Z_%04d.pdf' % (outputdir, ident, iname, ivar, ilev / 100)) #, bbox_extra_artists=(lgd,), bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_12Z_%04d.pdf' % (outputdir, ident, iname, ivar, ilev / 100), verbose)
plt.close()
#
fig, ax = plt.subplots(1, 1, figsize=(12, 4))
ym = int(idata[ivar, idata.major_axis.hour == 0, ilev].mean())
for j, jvar in enumerate(varis):
if jvar in idata.items:
ax = timeseries_line_trend(idata[:, idata.major_axis.hour == 0, :], jvar,
'%s_dep_snht' % ivar, ilev, '%s [K]' % ivar, ax=ax,
plabel=ym + 4, ylim=(ym - 5, ym + 5), window=365,
label=varnames[j])
ax.grid(True)
lgd = ax.legend(bbox_to_anchor=(1.1, 1), loc='upper left', fontsize=10)
ax.set_xlim('1979', '2016')
ax.set_xlabel('Time (RM:365)')
plt.savefig('%s/%s_%s_%s_trends_00Z_%04d.pdf' % (outputdir, ident, iname, ivar, ilev / 100),
bbox_extra_artists=(lgd,), bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_12Z_%04d.pdf' % (outputdir, ident, iname, ivar, ilev / 100), verbose)
plt.close()
else:
fig, ax = plt.subplots(1, 1, figsize=(12, 4))
ym = int(idata[ivar, :, ilev].mean())
for j, jvar in enumerate(varis):
if jvar in idata.items:
ax = timeseries_line_trend(idata, jvar, '%s_dep_snht' % ivar, ilev, '%s [K]' % ivar,
ax=ax, plabel=ym + 4, ylim=(ym - 5, ym + 5), window=730,
label=varnames[j])
ax.grid(True)
lgd = ax.legend(bbox_to_anchor=(1.1, 1), loc='upper left', fontsize=10)
ax.set_xlim('1979', '2016')
ax.set_xlabel('Time (RM:730)')
plt.savefig('%s/%s_%s_%s_trends_%04d.pdf' % (outputdir, ident, iname, ivar, ilev / 100),
bbox_extra_artists=(lgd,), bbox_inches='tight')
print_verbose('%s/%s_%s_%s_trends_%04d.pdf' % (outputdir, ident, iname, ivar, ilev / 100), verbose)
plt.close()
plt.close('all')
def from_mars(ident, var=None, filename=None, save=False, update=True, mars_save=False, attach=None,
force=False, verbose=0, **kwargs):
"""Read MARS (ECMWF) dump from ODB
1. Read MARS dump or ODB dump
2. Convert to pandas dataframes
Typical data in mars dump:
data RAW radiosonde data (time,p, variables)
an Analysis departures ERA-Interim
fg First-guess departures ERA-Interim
bias Bias estimate
station Station (lon,lat,alt,sonde_type)
Parameters
----------
ident int/str Radiosonde ID
var list Variables in the MARS
filename str filename of mars dump
save bool save radiosonde class ?
mars_save bool save mars dump to HDF5 ?
attach radiosonde attach to this radiosonde
verbose int verbosness
kwargs dict **
Returns
-------
radiosonde
Examples
--------
>>> isonde = from_mars(11035, verbose=1)
"""
from raso import radiosonde
selection = False
if var is not None:
if isinstance(var, str):
var = [var]
for ivar in var:
if ivar not in ['data', 'bias', 'fg', 'an']:
raise ValueError("Unknown variable! %s" % ivar)
selection = True
try:
ident = int(ident) # ident needs to be a number here
tmp = mars_to_dataframe(ident, filename=filename, save=mars_save, verbose=verbose - 1)
print_verbose('[MARS] %d : %s' % (ident, str(tmp.shape)), verbose)
station = tmp[['lon', 'lat', 'alt', 'sonde_type']].drop_duplicates()
if 'std_time' in tmp.columns:
# legacy support
tmp.drop('std_time', axis=1, inplace=True)
tmp.drop(['sonde_type', 'obs_type', 'code_type', 'id', 'lon', 'lat', 'alt'], axis=1, inplace=True)
# ['varno' 'p' 'value' 'biascorr' 'fg_dep' 'an_dep']
tmp.replace({'varno': {2: 't', 3: 'u', 4: 'v', 7: 'q', 29: 'r'}}, inplace=True) # rename varno to vars
# group by index and p
data = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['value'].unstack(level=-1).reset_index().rename(
columns={'level_0': 'date'}).set_index('date', drop=True)
#
if (selection and 'bias' in var) or not selection:
bias = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['biascorr'].unstack(
level=-1).reset_index().rename(columns={'level_0': 'date'}).set_index('date', drop=True)
bias.ix[:, ~(bias.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + bias.ix[:, ~(bias.columns == 'p')]
#
if (selection and 'fg' in var) or not selection:
fg = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['fg_dep'].unstack(level=-1).reset_index().rename(
columns={'level_0': 'date'}).set_index('date', drop=True)
fg.ix[:, ~(fg.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + fg.ix[:, ~(fg.columns == 'p')]
#
if (selection and 'an' in var) or not selection:
an = tmp.groupby([tmp.index, tmp.p, tmp.varno]).first()['an_dep'].unstack(level=-1).reset_index().rename(
columns={'level_0': 'date'}).set_index('date', drop=True)
an.ix[:, ~(an.columns == 'p')] = data.ix[:, ~(data.columns == 'p')] + an.ix[:, ~(an.columns == 'p')]
if attach is None:
out = radiosonde(id=ident)
else:
out = attach
#
if selection:
print_verbose("MARS: %s" % (str(var)), verbose)
for ivar in var:
out.add_data(ivar, locals()[ivar])
else:
out.add_data('data', data)
out.add_data('bias', bias)
out.add_data('fg', fg)
out.add_data('an', an)
#
out.add_data('station', station)
try:
out.add_attr('lon', station.ix[-1, 'lon'])
out.add_attr('lat', station.ix[-1, 'lat'])
out.add_attr('alt', station.ix[-1, 'alt'])
except IndexError:
out.add_attr('lon', -1)
out.add_attr('lat', -1)
out.add_attr('alt', -1)
if attach is None:
if save:
out.save(update=update, force=force, verbose=verbose)
else:
return out
except:
raise
