def del_data(self, name, fromdisk=False, directory=None):
"""remove variable from radiosonde
"""
from raso.config import rasodir
if not hasattr(name, '__iter__'):
name = [name]
if directory is None:
if self.directory is None:
directory = rasodir
else:
directory = self.directory
default = directory + '/%s' % self.id
for j in name:
if hasattr(self, j):
delattr(self, j)
self.vars.remove(j)
self.is_saved = False
self.var_saved.pop(j)
journal('Variable deleted: %s' % j, self.history, 0)
if fromdisk:
if os.path.isfile(default+'/%s.h5' % j):
os.remove(default+'/%s.h5' % j)
self.is_saved = True
elif os.path.isfile(default+'/%s.pickle' % j):
os.remove(default + '/%s.pickle' % j)
self.is_saved = True
else:
print "%s not saved in %s" % (j, default)
def from_igra(ident, var=['p', 't', 'r', 'dpd', 'wd', 'ws'], filename=None, attach=None, save=False, verbose=0, **kwargs):
"""Read IGRA data from text
Parameters
----------
ident Radiosonde ID (WMO or IGRA ID)
wmo convert WMO ID to IGRA
variables select variables
filename filename of txt.gz
attach radiosonde object to attach with
verbose Verbosity
Returns
-------
radiosonde object / save to HDF5
"""
wmoid = None
igraid = wmo2igra(ident) # try to convert WMO to IGRA ?
if igraid is not None:
wmoid = ident # WMO ID
ident = igraid # found IGRA ID
print_verbose("[IGRA] WMO %s to IGRA %s"%(wmoid, ident), verbose)
data = from_igra_to_dataframe(ident, filename=filename, verbose=verbose)
lon = data.ix[-1, 'lon'] / 10000. # IGRA specific
lat = data.ix[-1, 'lat'] / 10000.
data = data[var]
#
if isinstance(attach, radiosonde):
attach.add_data('igra', data)
attach.add_attr('igraid', ident)
if 'lon' not in attach.attrs:
attach.add_attr('lon', lon)
if 'lat' not in attach.attrs:
attach.add_attr('lat', lat)
journal("[IGRA] data read with %s" % ident, attach.history, verbose)
else:
# try to convert to WMO ID
if wmoid is None:
wmoid = igra2wmo(ident)
if wmoid is None:
out = radiosonde(id=ident)
else:
out = radiosonde(id=wmoid)
out.add_attr('igraid', ident)
out.add_data('igra', data)
out.add_attr('lon', lon)
out.add_attr('lat', lat) # convert to float
out.history.append(now() + " IGRA Data read")
if save:
out.save()
else:
return out
def gather_information(directory, pattern, force=False, report=None, verbose=0):
"""
Read Information for NetCDF global attributes about time and position of GPS RO event
Parameters
----------
directory search directory for GPS RO NetCDF files
pattern search pattern
Returns
-------
pandas.DataFrame
Notes
-----
Can take quite long to collect (2h)
stores info in outdir/GPS_info.h5
"""
from ncio import read_history
from raso.config import outdir
# check midpoints of GPS profiles and create a list with
filename = outdir + '/GPS_info.h5'
if os.path.isfile(filename) and not force:
tmp = pd.read_hdf(filename, 'data')
journal('GPSRO_info : %s %s' % (filename, tmp.shape), report, verbose)
return tmp
# filenames , lon, lat, time
files = find_files(directory, pattern, recursive=True)
data = {}
pbar = ProgressBar(maxval=len(files))
pbar.start()
for i, ifile in enumerate(files):
hist = read_history(ifile)
data[i] = {'file': ifile, 'lon': hist['lon'],
'lat': hist['lat'],
'date': datetime(hist['year'], hist['month'], hist['day'], hist['hour'], hist['minute'])}
pbar.update(i + 1)
pbar.finish()
data = pd.DataFrame.from_dict(data, orient='index')
# save
data.to_hdf(filename, 'data', format='table', append=False)
journal('GPSRO_info : %s %s' % (filename, data.shape), report, verbose)
return data
def __init__(self, id=None, verbose=0):
"""Initialize Radiosonde
"""
self.attrs = []
#
if isinstance(id, (float, int)):
id = "%06d" % id
self.id = str(id) # always convert this to a string
#
# Information
self.history = []
self.notes = ""
#
# Variables
self.vars = []
self.is_saved = False
self.var_saved = {}
self.is_empty = True
self.directory = None
#
journal("%s Radiosonde created! " % id, self.history, verbose)
def add_data(self, name, data, replace=False, verbose=0):
"""add variable to radiosonde class
"""
if name in self.vars:
# replace it ?
if replace:
self.vars.remove(name)
else:
print "[%s] Variable %s already attached!" % (self.id, name)
return
#
setattr(self, name, data)
self.vars.append(name)
self.is_empty = False
self.var_saved[name] = False
#
if replace:
journal("Variable replaced: %s" % name, self.history, verbose)
else:
journal("Variable added: %s" % name, self.history, verbose)
self.is_saved = False
def standard(isonde, var='data', save=True, update=False, force=False, verbose=0, **kwargs):
"""Standardize radiosonde data
1. Quality Control
2. Datetime fix (0/12 Soundings)
3. Interpolation / std levels (ERA Levels)
4. Variable conversion (td, vp, dpd)
5. Final Quality Control
6. Merge with ERA-Interim Data if availbale
Parameters
----------
isonde radiossonde Radiosonde class object
var str Radiosonde variable to use
save bool Save after complettion?
update bool Restandard?
force bool force write file
verbose int verboseness
Additional Parameters
---------------------
interpolate bool Interpolate to levels (default: era_plevels)
esat str saturation water vapor pressure formulation
levels list pressure levels for interpolation
remove bool Repalce Flagged values with nan ?
Returns
-------
radiosonde class object / or nothing (save=True)
Raises
------
"""
funcid = "[STD] Sonde "
if not isinstance(isonde, radiosonde):
raise ValueError(funcid+"requires a radiosonde object")
if not isinstance(var, str):
raise ValueError(funcid + "Requires a string")
if var not in isonde.vars:
raise ValueError(funcid + "No Variable found: %s in %s" % (var,isonde.id) )
svar = "std_%s" % var
if not hasattr(isonde, svar) or update:
journal(funcid+"Creating (%s) %s" %(update, svar), isonde.history, verbose)
tmp = getattr(isonde, var) # Get Data
tmp = control(tmp, replace=False) # MAke Report
isonde.add_attr('qc_%s' % var, report(tmp, show=False)) # Save Report
newdata = standard_data(tmp, report=isonde.history, verbose=verbose - 1, **kwargs)
isonde.add_attr('qc_%s' % svar, report(newdata, show=False)) # Save Report
isonde.add_data(svar, newdata, replace=True)
else:
# Check if Quality Reports are missing
if 'qc_%s' % var not in isonde.attrs or update:
journal(funcid + "QC %s : %s " % (isonde.id, var), isonde.history, verbose)
tmp = control(getattr(isonde, var), replace=False)
isonde.add_attr('qc_%s' % var, report(tmp, show=False)) # Add
if 'qc_%s' % svar not in isonde.attrs or update:
journal(funcid + "QC %s : %s " % (isonde.id, svar), isonde.history, verbose)
isonde.add_attr('qc_%s' % svar, report(getattr(isonde, svar), show=False)) # Add
journal(funcid + "%s : %s unchanged" % (isonde.id, var), isonde.history, verbose)
if hasattr(isonde, svar) and hasattr(isonde, 'eradata'):
if not getattr(isonde, svar).columns.str.contains('era').any() or update:
newdata = merge(getattr(isonde, svar), getattr(isonde, 'eradata'), report=isonde.history, verbose=verbose - 1)
isonde.add_data('%s' % svar, newdata, replace=True)
journal(funcid + "Merged ERA: %s" % svar, isonde.history, verbose)
else:
journal(funcid + "%s : %s not merged" % (str(isonde.id), var), isonde.history, verbose)
# batch = kwargs.get('batch', False)
# if batch:
# isonde.save(update=update, force=force, verbose=verbose - 1)
# return inquire_extended(isonde)
elif save:
isonde.save(update=update, force=force, verbose=verbose - 1)
else:
return isonde
def from_gpsro(directory, pattern, ident, radius, save=True, esat='murphy_koop', verbose=0):
"""
Read GPS RO events into a timeseries for a certain location
Parameters
----------
directory str search directory
pattern str search pattern
ident int/str radiosonde ID
radius float allowed distance to ilon, ilat of GPS RO events [km]
save bool save radiosonde class object
verbose int verbosity
Raises
------
ValueError wrong input types
RuntimeError no GPS RO event within specified distance
"""
from . import from_store
from raso.config import rasodir
from raso import radiosonde
if not isinstance(directory, str):
raise ValueError("Require a string: directory")
if not isinstance(pattern, str):
raise ValueError("Require a string: pattern")
if not isinstance(ident, (int, str, float)):
raise ValueError("Require a int, str, float: ident")
if not isinstance(radius, (float, int)):
raise ValueError("Require a int, float: radius")
if isinstance(ident, (int, float)):
ident = "%06d" % ident
name = "gpsro_%06d" % int(radius)
iname = "gpsro_infos_%06d" % int(radius)
filename = rasodir + "/%s/%s.h5" % (ident, name)
# RECOVER
if os.path.isfile(filename):
print_verbose("Recovering previous: %s" % filename, verbose)
sonde = from_store(ident, variables=[name, iname])
return sonde
if not os.path.isdir(rasodir + "/%s" % ident):
os.makedirs(rasodir + "/%s" % ident)
try:
sonde = from_store(ident, variables=[]) # Reads the empty Radiosonde class >> IOError
ilon = sonde.lon # >> Attribute Error
ilat = sonde.lat
except IOError:
sonde = radiosonde(ident) # Make a new one
sondes = radiosondelist()
ilon = sondes.ix[ident, 'lon'].values
ilat = sondes.ix[ident, 'lat'].values
except AttributeError:
sondes = radiosondelist()
ilon = sondes.ix[ident, 'lon'].values
ilat = sondes.ix[ident, 'lat'].values
if ilon is None or ilat is None:
raise RuntimeError("Can't find any useful lon, lat Information on Radiosonde: %s" % ident)
infos, data = read_nearest_events(directory, pattern, ilon, ilat, radius, esat=esat, verbose=verbose)
journal("GPSRO Data: r=%d km [%s/%s]" % (radius, directory, pattern), sonde.history, verbose)
sonde.add_attr('lon', ilon)
sonde.add_attr('lat', ilat)
sonde.add_data(iname, infos)
sonde.add_data(name, data)
if save:
sonde.save(verbose=verbose)
else:
return sonde
def datasets(
isonde,
mars_std=True,
igra_std=False,
update_mars=False,
update_igra=False,
update_era=False,
save=True,
mars_vars=None,
verbose=0,
):
"""
Create standard radiosonde datasets
1. Read Radiosonde data (MARS / IGRA)
2. Read ERA-Interim data at RS location
3. Read IGRAv2 data if available
4. Standardize the data
4.1. Quality Control
4.2. Interpolate to std levels
4.3. Convert to td, dpd, vp
4.4. Final Quality Control
5. Save as store
Parameters
----------
isonde radiosonde Radiosonde class
mars_std bool Standardize MARS Data ?
igra_std bool Standardize IGRA Data ?
update_mars bool Update MARS data ?
update_igra bool Update IGRA data?
update_era bool Update ERA data ?
save bool save radiosonde ?
mars_vars list MARS variables: data, an, fg, bias
debug bool raises Errors
verbose int verboseness
kwargs dict
Returns
-------
attach to isonde
"""
from raso.read import from_era, from_igra, from_mars
funcid = "[DS] "
if not isinstance(isonde, radiosonde):
raise ValueError(funcid + "requires a radiosonde object")
std_vars = []
status = True
if mars_vars is None:
mars_vars = ["data", "an"]
try:
if "data" not in isonde.vars or update_mars:
from_mars(isonde.id, var=mars_vars, attach=isonde, verbose=verbose - 1)
journal(funcid + "from mars: %s" % str(mars_vars), isonde.history, verbose)
if mars_std:
std_vars.append("data")
except Exception, e:
journal(funcid + " MARS failed: %s" % repr(e), isonde.history, verbose)
status = False
try:
if "data" not in isonde.vars or update_mars:
from_mars(isonde.id, var=mars_vars, attach=isonde, verbose=verbose - 1)
journal(funcid + "from mars: %s" % str(mars_vars), isonde.history, verbose)
if mars_std:
std_vars.append("data")
except Exception, e:
journal(funcid + " MARS failed: %s" % repr(e), isonde.history, verbose)
status = False
# IGRA
try:
if "igra" not in isonde.vars or update_igra:
from_igra(isonde.id, attach=isonde, verbose=verbose - 1)
journal(funcid + "from igra: %s" % isonde.igraid, isonde.history, verbose)
if igra_std:
std_vars.append("igra")
except Exception, e:
journal(funcid + " IGRA failed: %s" % repr(e), isonde.history, verbose)
status = False
# ERA
try:
if "eradata" not in isonde.vars or "surface" not in isonde.vars or update_era:
surf, prof = from_era(
isonde.id,
alt=getattr(isonde, "alt", 0),
lon=getattr(isonde, "lon", None),
lat=getattr(isonde, "lat", None),
def detect_and_correct(isonde, data='std_data', var='dpd', quality_controlled=True, savename='sdcor', save=True,
tfirst=True, daynight=False, verbose=0, **kwargs):
"""Detect and Correct Radiosonde biases from Departure Statistics
Use ERA-Interim departures to detect breakpoints and
correct these with a mean and a quantile adjustment going back in time.
uses raso.timeseries.breakpoint.detection / correction
Parameters
----------
isonde radiosonde Radiosonde class object
data str Radiosonde variable
var str Variable
quality_controlled bool Use QC to remove flagged values?
savename str store name
save bool Save?
tfirst bool Correct Temperature first ?
daynight bool Correct Day and Night Soundings separate?
verbose int verbosness
Additional Parameters
---------------------
thres int [50] SNHT Threshold
quantilen list/array [0-100] Quantile Ranges
levels list [None] Pressure levels
sample_size int [730] minimum Sample size
borders int [180] biased sample before and after a break
bias30k bool [T] remove 30K Bias for dpd ?
verbose int [0] verboseness
kwargs dict breakpoint.detection, breakpoint.correction ...
Returns
-------
"""
from ..detect_and_correct import detect_and_correct as detect_and_correct_data
from ..detect_and_correct import detect_and_correct_daynight
from .. import standard_dewpoint_depression, standard_water_vapor
funcid = "[DC] Sonde "
if not isinstance(isonde, radiosonde):
raise ValueError(funcid + "requires a radiosonde object!")
if isonde.is_empty:
raise ValueError(funcid + "Radiosonde is empty!")
funcid = "[DC] %s " % isonde.id
if data not in isonde.vars:
raise ValueError(funcid + "Required variable (%s) not present" % data)
prof = getattr(isonde, data).copy() # GET DATA
print_verbose(funcid+"Savename: %s" % savename, verbose)
if hasnames(prof, 'qual'):
#
# drop all the values that have a qual flag
#
if quality_controlled:
journal(funcid + "QC of %s " % data, isonde.history, verbose)
prof = enforcer(prof) # Work on FLAGS, but not DPD 30, makes a copy
#
# set DPD30 to missing
#
if hasnames(prof, 'dpd'):
prof['dpd'] = np.where(prof.qual.str.contains('3'), np.nan, prof.dpd.values) # Apply? BAD, GOOD
if hasnames(prof, 'td'):
prof['td'] = np.where(prof.qual.str.contains('3'), np.nan, prof.td.values) # Apply? BAD, GOOD
del prof['qual'] # prof.drop('qual', 1, inplace=True) # remove all flag information
print_verbose(funcid + " dropping qual ...", verbose)
if hasnames(prof, 'orig'):
del prof['orig'] # prof.drop('orig', 1, inplace=True) # indicates interpolated or not
if 'dpd' in var:
if not hasnames(prof, 'dpd_era'):
prof['dpd_era'] = prof['t_era'] - prof['td_era']
print_verbose(funcid + " Calculating dpd_era ...", verbose)
tbreaks = None
if var == 't' and tfirst:
tfirst = False
print_verbose(funcid + "tfirst=True only with temperature dependent variables", 1)
if tfirst:
journal(funcid + "Running T D&C first! ", isonde.history, verbose)
#
# Only Mean Adjustment for Temperature
#
if not daynight:
prof, tbreaks = detect_and_correct_data(prof, var='t', correct_q=False, bounded=None,
report=isonde.history, verbose=verbose - 1, **kwargs)
else:
prof, tbreaks = detect_and_correct_daynight(prof, var='t', correct_q=False, bounded=None,
report=isonde.history, verbose=verbose - 1, **kwargs)
tbreaks['breaks'] = tbreaks['00Z'] + tbreaks['12Z'] # 00Z and 12Z breaks
#
# new Columns: t_mcor, t_dep, t_dep_breaks, t_dep_snht
#
if len(tbreaks['breaks']) > 0:
journal(funcid + "T-breaks: %s" % str(tbreaks['breaks']), isonde.history, verbose)
# prof.major_axis.name = 'date'
# prof.minor_axis.name = 'p'
# prof = panel_to_database(prof) # to DataFrame -> Function requires it > Deprecated now
# Recalculate Temperature Dependent Variables:
prof = standard_water_vapor(prof, tvar='t_mcor', vpvar='vp_tcor', replace=True, report=isonde.history)
prof = standard_dewpoint_depression(prof, tvar='t_mcor', dpdvar='dpd_tcor', vpvar='vp_tcor',
tdvar='td_tcor', replace=True, report=isonde.history)
del prof['qual'] # prof.drop('qual', 1, inplace=True) # remove quality Flag again
# prof.rename(items={var: '%s_orig' % var}, inplace=True) # Rochade
# prof.rename(items={'%s_tcor' % var: var}, inplace=True) #
#
if hasnames(prof, '%s_tcor' % var):
journal(funcid + "Running t-correct %s D&C ..." % var, isonde.history, verbose)
prof['%s_tcor_era' % var] = prof['%s_era' % var]
if not daynight:
prof, dbreaks = detect_and_correct_data(prof, var='%s_tcor' % var, report=isonde.history,
verbose=verbose - 1, **kwargs)
else:
prof, dbreaks = detect_and_correct_daynight(prof, var='%s_tcor' % var, report=isonde.history,
verbose=verbose - 1, **kwargs)
dbreaks['breaks'] = dbreaks['00Z'] + dbreaks['12Z'] # Combine
del prof['%s_tcor_era' % var]
# prof.rename(items={var: '%s_tcor' % var,
# '%s_mcor' % var: '%s_tcor_mcor' % var,
# '%s_qcor' % var: '%s_tcor_qcor' % var,
# '%s_qecor' % var: '%s_tcor_qecor' % var,
# '%s_dep' % var: '%s_tcor_dep' % var,
# '%s_dep_breaks' % var: '%s_tcor_dep_breaks' % var,
# '%s_dep_snht' % var: '%s_tcor_dep_snht' % var,
# '%s_orig' % var: var}, inplace=True)
else:
journal(funcid + "No T breakpoints. ", isonde.history, verbose)
# prof = panel_to_database(prof) # Convert to DataFrame (after Detection) > Deprecated now
journal(funcid + "Running %s D&C" % var, isonde.history, verbose)
if not daynight:
prof, stat = detect_and_correct_data(prof, var=var, report=isonde.history, verbose=verbose, **kwargs)
else:
prof, stat = detect_and_correct_daynight(prof, var=var, report=isonde.history, verbose=verbose, **kwargs)
stat['breaks'] = stat['00Z'] + stat['12Z'] # Combine lists
#
isonde.add_data(savename, prof, replace=True, verbose=verbose) # DID anything change ?
#
# Options
#
thres = kwargs.get('thres', 50)
borders = kwargs.get('borders', 180)
sample_size = kwargs.get('sample_size', 730)
quantilen = kwargs.get('quantilen', np.arange(0, 101, 10)) # missing from history > add
journal(funcid + "%s (T:%d, N:%d, B:%d, Q:%d)" % (var, int(thres), sample_size, borders, len(quantilen)),
isonde.history, 0)
if tbreaks is not None:
if len(tbreaks['breaks']) > 0:
stat['t-breaks'] = tbreaks['breaks']
if len(dbreaks['breaks']) > 0:
stat['t-dpd-breaks'] = dbreaks['breaks']
stat['thres'] = thres
stat['borders'] = borders
stat['sample_size'] = sample_size
stat['quantilen'] = quantilen
stat['source'] = data
stat['savename'] = savename
stat['variable'] = var
stat['daynight'] = daynight
if 'detect_opts' not in isonde.attrs:
isonde.add_attr('detect_opts', {"%s_%s" % (savename, var): stat}) # sdcor_dpd
else:
isonde.detect_opts["%s_%s" % (savename, var)] = stat # update
if save:
isonde.save(var=savename, update=True, verbose=verbose)
return stat
def save(self, var=None, directory=None, update=False, hdfformat='table', hdfcomplevel=9, hdfcomplib='blosc', verbose=0):
""" Write Radiosonde Class Object to Directory
Parameters
----------
var str/list
directory str
update bool
force bool
mode str
hdfformat str
hdfcomplevel int
hdfcomplib str
verbose int
"""
from raso.config import rasodir
import pickle
if directory is None:
if self.directory is None:
directory = rasodir
else:
directory = self.directory
default = directory + '/%s/' % self.id
if not os.path.isdir(default):
os.makedirs(default)
if var is None:
varis = self.vars # All Variables
else:
if isinstance(var, str):
varis = [var]
else:
varis = var
update = True # Make sure we write it
journal("Saving to dir: %s" % default, self.history, verbose)
for ivar in varis:
written = False
if self.var_saved[ivar] and not update:
if os.path.isfile(default + "%s.h5" % ivar) or os.path.isfile(default + "%s.pickle" % ivar):
print_verbose("%20s [%5s]" % (ivar, color_boolean(written)), verbose)
continue
tmp = getattr(self, ivar)
#
# Try as HDF
#
if isinstance(tmp, (pd.Series, pd.DataFrame, pd.Panel)):
try:
tmp.to_hdf(default + "%s.h5" % ivar, ivar, mode='a', format=hdfformat, complevel=hdfcomplevel, complib=hdfcomplib)
fio = 'H5'
written = True
n = tmp.shape
except Exception as e:
if verbose:
print ivar, " >> ", repr(e)
#
# And as Pickle
#
if not written:
try:
pickle.dump(tmp, open(default + "%s.pickle" % ivar, 'w'))
fio = 'PKL'
written = True
n = len(tmp)
except Exception as e:
if verbose:
print ivar, " >> ", repr(e)
print_verbose("%20s %20s [%5s] %s" % (ivar, n, color_boolean(written), fio), verbose)
attrs = {}
for iatt in self.attrs:
attrs[iatt] = getattr(self, iatt)
pickle.dump(attrs, open(default + 'attributes.pickle', 'w'))
attrs = {}
for s in self.history:
if 'Saved' in s:
self.history.pop(self.history.index(s))
journal("Saved", self.history, 0)
for iatt in ['id', 'is_saved', 'is_empty']:
attrs[iatt] = getattr(self, iatt)
pickle.dump(attrs, open(default + 'radiosonde.pickle', 'w'))
f = open(default+'history.txt', 'w')
f.write("\n".join(self.history))
f.close()
if self.notes != "":
f = open(default + 'notes.txt', 'w')
f.write(self.notes)
f.close()
self.directory = directory
