def del_data(self, name, fromdisk=False, directory=None): """remove variable from radiosonde """ from raso.config import rasodir if not hasattr(name, '__iter__'): name = [name] if directory is None: if self.directory is None: directory = rasodir else: directory = self.directory default = directory + '/%s' % self.id for j in name: if hasattr(self, j): delattr(self, j) self.vars.remove(j) self.is_saved = False self.var_saved.pop(j) journal('Variable deleted: %s' % j, self.history, 0) if fromdisk: if os.path.isfile(default+'/%s.h5' % j): os.remove(default+'/%s.h5' % j) self.is_saved = True elif os.path.isfile(default+'/%s.pickle' % j): os.remove(default + '/%s.pickle' % j) self.is_saved = True else: print "%s not saved in %s" % (j, default)
def from_igra(ident, var=['p', 't', 'r', 'dpd', 'wd', 'ws'], filename=None, attach=None, save=False, verbose=0, **kwargs): """Read IGRA data from text Parameters ---------- ident Radiosonde ID (WMO or IGRA ID) wmo convert WMO ID to IGRA variables select variables filename filename of txt.gz attach radiosonde object to attach with verbose Verbosity Returns ------- radiosonde object / save to HDF5 """ wmoid = None igraid = wmo2igra(ident) # try to convert WMO to IGRA ? if igraid is not None: wmoid = ident # WMO ID ident = igraid # found IGRA ID print_verbose("[IGRA] WMO %s to IGRA %s"%(wmoid, ident), verbose) data = from_igra_to_dataframe(ident, filename=filename, verbose=verbose) lon = data.ix[-1, 'lon'] / 10000. # IGRA specific lat = data.ix[-1, 'lat'] / 10000. data = data[var] # if isinstance(attach, radiosonde): attach.add_data('igra', data) attach.add_attr('igraid', ident) if 'lon' not in attach.attrs: attach.add_attr('lon', lon) if 'lat' not in attach.attrs: attach.add_attr('lat', lat) journal("[IGRA] data read with %s" % ident, attach.history, verbose) else: # try to convert to WMO ID if wmoid is None: wmoid = igra2wmo(ident) if wmoid is None: out = radiosonde(id=ident) else: out = radiosonde(id=wmoid) out.add_attr('igraid', ident) out.add_data('igra', data) out.add_attr('lon', lon) out.add_attr('lat', lat) # convert to float out.history.append(now() + " IGRA Data read") if save: out.save() else: return out
def gather_information(directory, pattern, force=False, report=None, verbose=0): """ Read Information for NetCDF global attributes about time and position of GPS RO event Parameters ---------- directory search directory for GPS RO NetCDF files pattern search pattern Returns ------- pandas.DataFrame Notes ----- Can take quite long to collect (2h) stores info in outdir/GPS_info.h5 """ from ncio import read_history from raso.config import outdir # check midpoints of GPS profiles and create a list with filename = outdir + '/GPS_info.h5' if os.path.isfile(filename) and not force: tmp = pd.read_hdf(filename, 'data') journal('GPSRO_info : %s %s' % (filename, tmp.shape), report, verbose) return tmp # filenames , lon, lat, time files = find_files(directory, pattern, recursive=True) data = {} pbar = ProgressBar(maxval=len(files)) pbar.start() for i, ifile in enumerate(files): hist = read_history(ifile) data[i] = {'file': ifile, 'lon': hist['lon'], 'lat': hist['lat'], 'date': datetime(hist['year'], hist['month'], hist['day'], hist['hour'], hist['minute'])} pbar.update(i + 1) pbar.finish() data = pd.DataFrame.from_dict(data, orient='index') # save data.to_hdf(filename, 'data', format='table', append=False) journal('GPSRO_info : %s %s' % (filename, data.shape), report, verbose) return data
def __init__(self, id=None, verbose=0): """Initialize Radiosonde """ self.attrs = [] # if isinstance(id, (float, int)): id = "%06d" % id self.id = str(id) # always convert this to a string # # Information self.history = [] self.notes = "" # # Variables self.vars = [] self.is_saved = False self.var_saved = {} self.is_empty = True self.directory = None # journal("%s Radiosonde created! " % id, self.history, verbose)
def add_data(self, name, data, replace=False, verbose=0): """add variable to radiosonde class """ if name in self.vars: # replace it ? if replace: self.vars.remove(name) else: print "[%s] Variable %s already attached!" % (self.id, name) return # setattr(self, name, data) self.vars.append(name) self.is_empty = False self.var_saved[name] = False # if replace: journal("Variable replaced: %s" % name, self.history, verbose) else: journal("Variable added: %s" % name, self.history, verbose) self.is_saved = False
def standard(isonde, var='data', save=True, update=False, force=False, verbose=0, **kwargs): """Standardize radiosonde data 1. Quality Control 2. Datetime fix (0/12 Soundings) 3. Interpolation / std levels (ERA Levels) 4. Variable conversion (td, vp, dpd) 5. Final Quality Control 6. Merge with ERA-Interim Data if availbale Parameters ---------- isonde radiossonde Radiosonde class object var str Radiosonde variable to use save bool Save after complettion? update bool Restandard? force bool force write file verbose int verboseness Additional Parameters --------------------- interpolate bool Interpolate to levels (default: era_plevels) esat str saturation water vapor pressure formulation levels list pressure levels for interpolation remove bool Repalce Flagged values with nan ? Returns ------- radiosonde class object / or nothing (save=True) Raises ------ """ funcid = "[STD] Sonde " if not isinstance(isonde, radiosonde): raise ValueError(funcid+"requires a radiosonde object") if not isinstance(var, str): raise ValueError(funcid + "Requires a string") if var not in isonde.vars: raise ValueError(funcid + "No Variable found: %s in %s" % (var,isonde.id) ) svar = "std_%s" % var if not hasattr(isonde, svar) or update: journal(funcid+"Creating (%s) %s" %(update, svar), isonde.history, verbose) tmp = getattr(isonde, var) # Get Data tmp = control(tmp, replace=False) # MAke Report isonde.add_attr('qc_%s' % var, report(tmp, show=False)) # Save Report newdata = standard_data(tmp, report=isonde.history, verbose=verbose - 1, **kwargs) isonde.add_attr('qc_%s' % svar, report(newdata, show=False)) # Save Report isonde.add_data(svar, newdata, replace=True) else: # Check if Quality Reports are missing if 'qc_%s' % var not in isonde.attrs or update: journal(funcid + "QC %s : %s " % (isonde.id, var), isonde.history, verbose) tmp = control(getattr(isonde, var), replace=False) isonde.add_attr('qc_%s' % var, report(tmp, show=False)) # Add if 'qc_%s' % svar not in isonde.attrs or update: journal(funcid + "QC %s : %s " % (isonde.id, svar), isonde.history, verbose) isonde.add_attr('qc_%s' % svar, report(getattr(isonde, svar), show=False)) # Add journal(funcid + "%s : %s unchanged" % (isonde.id, var), isonde.history, verbose) if hasattr(isonde, svar) and hasattr(isonde, 'eradata'): if not getattr(isonde, svar).columns.str.contains('era').any() or update: newdata = merge(getattr(isonde, svar), getattr(isonde, 'eradata'), report=isonde.history, verbose=verbose - 1) isonde.add_data('%s' % svar, newdata, replace=True) journal(funcid + "Merged ERA: %s" % svar, isonde.history, verbose) else: journal(funcid + "%s : %s not merged" % (str(isonde.id), var), isonde.history, verbose) # batch = kwargs.get('batch', False) # if batch: # isonde.save(update=update, force=force, verbose=verbose - 1) # return inquire_extended(isonde) elif save: isonde.save(update=update, force=force, verbose=verbose - 1) else: return isonde
def from_gpsro(directory, pattern, ident, radius, save=True, esat='murphy_koop', verbose=0): """ Read GPS RO events into a timeseries for a certain location Parameters ---------- directory str search directory pattern str search pattern ident int/str radiosonde ID radius float allowed distance to ilon, ilat of GPS RO events [km] save bool save radiosonde class object verbose int verbosity Raises ------ ValueError wrong input types RuntimeError no GPS RO event within specified distance """ from . import from_store from raso.config import rasodir from raso import radiosonde if not isinstance(directory, str): raise ValueError("Require a string: directory") if not isinstance(pattern, str): raise ValueError("Require a string: pattern") if not isinstance(ident, (int, str, float)): raise ValueError("Require a int, str, float: ident") if not isinstance(radius, (float, int)): raise ValueError("Require a int, float: radius") if isinstance(ident, (int, float)): ident = "%06d" % ident name = "gpsro_%06d" % int(radius) iname = "gpsro_infos_%06d" % int(radius) filename = rasodir + "/%s/%s.h5" % (ident, name) # RECOVER if os.path.isfile(filename): print_verbose("Recovering previous: %s" % filename, verbose) sonde = from_store(ident, variables=[name, iname]) return sonde if not os.path.isdir(rasodir + "/%s" % ident): os.makedirs(rasodir + "/%s" % ident) try: sonde = from_store(ident, variables=[]) # Reads the empty Radiosonde class >> IOError ilon = sonde.lon # >> Attribute Error ilat = sonde.lat except IOError: sonde = radiosonde(ident) # Make a new one sondes = radiosondelist() ilon = sondes.ix[ident, 'lon'].values ilat = sondes.ix[ident, 'lat'].values except AttributeError: sondes = radiosondelist() ilon = sondes.ix[ident, 'lon'].values ilat = sondes.ix[ident, 'lat'].values if ilon is None or ilat is None: raise RuntimeError("Can't find any useful lon, lat Information on Radiosonde: %s" % ident) infos, data = read_nearest_events(directory, pattern, ilon, ilat, radius, esat=esat, verbose=verbose) journal("GPSRO Data: r=%d km [%s/%s]" % (radius, directory, pattern), sonde.history, verbose) sonde.add_attr('lon', ilon) sonde.add_attr('lat', ilat) sonde.add_data(iname, infos) sonde.add_data(name, data) if save: sonde.save(verbose=verbose) else: return sonde
def datasets( isonde, mars_std=True, igra_std=False, update_mars=False, update_igra=False, update_era=False, save=True, mars_vars=None, verbose=0, ): """ Create standard radiosonde datasets 1. Read Radiosonde data (MARS / IGRA) 2. Read ERA-Interim data at RS location 3. Read IGRAv2 data if available 4. Standardize the data 4.1. Quality Control 4.2. Interpolate to std levels 4.3. Convert to td, dpd, vp 4.4. Final Quality Control 5. Save as store Parameters ---------- isonde radiosonde Radiosonde class mars_std bool Standardize MARS Data ? igra_std bool Standardize IGRA Data ? update_mars bool Update MARS data ? update_igra bool Update IGRA data? update_era bool Update ERA data ? save bool save radiosonde ? mars_vars list MARS variables: data, an, fg, bias debug bool raises Errors verbose int verboseness kwargs dict Returns ------- attach to isonde """ from raso.read import from_era, from_igra, from_mars funcid = "[DS] " if not isinstance(isonde, radiosonde): raise ValueError(funcid + "requires a radiosonde object") std_vars = [] status = True if mars_vars is None: mars_vars = ["data", "an"] try: if "data" not in isonde.vars or update_mars: from_mars(isonde.id, var=mars_vars, attach=isonde, verbose=verbose - 1) journal(funcid + "from mars: %s" % str(mars_vars), isonde.history, verbose) if mars_std: std_vars.append("data") except Exception, e: journal(funcid + " MARS failed: %s" % repr(e), isonde.history, verbose) status = False
try: if "data" not in isonde.vars or update_mars: from_mars(isonde.id, var=mars_vars, attach=isonde, verbose=verbose - 1) journal(funcid + "from mars: %s" % str(mars_vars), isonde.history, verbose) if mars_std: std_vars.append("data") except Exception, e: journal(funcid + " MARS failed: %s" % repr(e), isonde.history, verbose) status = False # IGRA try: if "igra" not in isonde.vars or update_igra: from_igra(isonde.id, attach=isonde, verbose=verbose - 1) journal(funcid + "from igra: %s" % isonde.igraid, isonde.history, verbose) if igra_std: std_vars.append("igra") except Exception, e: journal(funcid + " IGRA failed: %s" % repr(e), isonde.history, verbose) status = False # ERA try: if "eradata" not in isonde.vars or "surface" not in isonde.vars or update_era: surf, prof = from_era( isonde.id, alt=getattr(isonde, "alt", 0), lon=getattr(isonde, "lon", None), lat=getattr(isonde, "lat", None),
def detect_and_correct(isonde, data='std_data', var='dpd', quality_controlled=True, savename='sdcor', save=True, tfirst=True, daynight=False, verbose=0, **kwargs): """Detect and Correct Radiosonde biases from Departure Statistics Use ERA-Interim departures to detect breakpoints and correct these with a mean and a quantile adjustment going back in time. uses raso.timeseries.breakpoint.detection / correction Parameters ---------- isonde radiosonde Radiosonde class object data str Radiosonde variable var str Variable quality_controlled bool Use QC to remove flagged values? savename str store name save bool Save? tfirst bool Correct Temperature first ? daynight bool Correct Day and Night Soundings separate? verbose int verbosness Additional Parameters --------------------- thres int [50] SNHT Threshold quantilen list/array [0-100] Quantile Ranges levels list [None] Pressure levels sample_size int [730] minimum Sample size borders int [180] biased sample before and after a break bias30k bool [T] remove 30K Bias for dpd ? verbose int [0] verboseness kwargs dict breakpoint.detection, breakpoint.correction ... Returns ------- """ from ..detect_and_correct import detect_and_correct as detect_and_correct_data from ..detect_and_correct import detect_and_correct_daynight from .. import standard_dewpoint_depression, standard_water_vapor funcid = "[DC] Sonde " if not isinstance(isonde, radiosonde): raise ValueError(funcid + "requires a radiosonde object!") if isonde.is_empty: raise ValueError(funcid + "Radiosonde is empty!") funcid = "[DC] %s " % isonde.id if data not in isonde.vars: raise ValueError(funcid + "Required variable (%s) not present" % data) prof = getattr(isonde, data).copy() # GET DATA print_verbose(funcid+"Savename: %s" % savename, verbose) if hasnames(prof, 'qual'): # # drop all the values that have a qual flag # if quality_controlled: journal(funcid + "QC of %s " % data, isonde.history, verbose) prof = enforcer(prof) # Work on FLAGS, but not DPD 30, makes a copy # # set DPD30 to missing # if hasnames(prof, 'dpd'): prof['dpd'] = np.where(prof.qual.str.contains('3'), np.nan, prof.dpd.values) # Apply? BAD, GOOD if hasnames(prof, 'td'): prof['td'] = np.where(prof.qual.str.contains('3'), np.nan, prof.td.values) # Apply? BAD, GOOD del prof['qual'] # prof.drop('qual', 1, inplace=True) # remove all flag information print_verbose(funcid + " dropping qual ...", verbose) if hasnames(prof, 'orig'): del prof['orig'] # prof.drop('orig', 1, inplace=True) # indicates interpolated or not if 'dpd' in var: if not hasnames(prof, 'dpd_era'): prof['dpd_era'] = prof['t_era'] - prof['td_era'] print_verbose(funcid + " Calculating dpd_era ...", verbose) tbreaks = None if var == 't' and tfirst: tfirst = False print_verbose(funcid + "tfirst=True only with temperature dependent variables", 1) if tfirst: journal(funcid + "Running T D&C first! ", isonde.history, verbose) # # Only Mean Adjustment for Temperature # if not daynight: prof, tbreaks = detect_and_correct_data(prof, var='t', correct_q=False, bounded=None, report=isonde.history, verbose=verbose - 1, **kwargs) else: prof, tbreaks = detect_and_correct_daynight(prof, var='t', correct_q=False, bounded=None, report=isonde.history, verbose=verbose - 1, **kwargs) tbreaks['breaks'] = tbreaks['00Z'] + tbreaks['12Z'] # 00Z and 12Z breaks # # new Columns: t_mcor, t_dep, t_dep_breaks, t_dep_snht # if len(tbreaks['breaks']) > 0: journal(funcid + "T-breaks: %s" % str(tbreaks['breaks']), isonde.history, verbose) # prof.major_axis.name = 'date' # prof.minor_axis.name = 'p' # prof = panel_to_database(prof) # to DataFrame -> Function requires it > Deprecated now # Recalculate Temperature Dependent Variables: prof = standard_water_vapor(prof, tvar='t_mcor', vpvar='vp_tcor', replace=True, report=isonde.history) prof = standard_dewpoint_depression(prof, tvar='t_mcor', dpdvar='dpd_tcor', vpvar='vp_tcor', tdvar='td_tcor', replace=True, report=isonde.history) del prof['qual'] # prof.drop('qual', 1, inplace=True) # remove quality Flag again # prof.rename(items={var: '%s_orig' % var}, inplace=True) # Rochade # prof.rename(items={'%s_tcor' % var: var}, inplace=True) # # if hasnames(prof, '%s_tcor' % var): journal(funcid + "Running t-correct %s D&C ..." % var, isonde.history, verbose) prof['%s_tcor_era' % var] = prof['%s_era' % var] if not daynight: prof, dbreaks = detect_and_correct_data(prof, var='%s_tcor' % var, report=isonde.history, verbose=verbose - 1, **kwargs) else: prof, dbreaks = detect_and_correct_daynight(prof, var='%s_tcor' % var, report=isonde.history, verbose=verbose - 1, **kwargs) dbreaks['breaks'] = dbreaks['00Z'] + dbreaks['12Z'] # Combine del prof['%s_tcor_era' % var] # prof.rename(items={var: '%s_tcor' % var, # '%s_mcor' % var: '%s_tcor_mcor' % var, # '%s_qcor' % var: '%s_tcor_qcor' % var, # '%s_qecor' % var: '%s_tcor_qecor' % var, # '%s_dep' % var: '%s_tcor_dep' % var, # '%s_dep_breaks' % var: '%s_tcor_dep_breaks' % var, # '%s_dep_snht' % var: '%s_tcor_dep_snht' % var, # '%s_orig' % var: var}, inplace=True) else: journal(funcid + "No T breakpoints. ", isonde.history, verbose) # prof = panel_to_database(prof) # Convert to DataFrame (after Detection) > Deprecated now journal(funcid + "Running %s D&C" % var, isonde.history, verbose) if not daynight: prof, stat = detect_and_correct_data(prof, var=var, report=isonde.history, verbose=verbose, **kwargs) else: prof, stat = detect_and_correct_daynight(prof, var=var, report=isonde.history, verbose=verbose, **kwargs) stat['breaks'] = stat['00Z'] + stat['12Z'] # Combine lists # isonde.add_data(savename, prof, replace=True, verbose=verbose) # DID anything change ? # # Options # thres = kwargs.get('thres', 50) borders = kwargs.get('borders', 180) sample_size = kwargs.get('sample_size', 730) quantilen = kwargs.get('quantilen', np.arange(0, 101, 10)) # missing from history > add journal(funcid + "%s (T:%d, N:%d, B:%d, Q:%d)" % (var, int(thres), sample_size, borders, len(quantilen)), isonde.history, 0) if tbreaks is not None: if len(tbreaks['breaks']) > 0: stat['t-breaks'] = tbreaks['breaks'] if len(dbreaks['breaks']) > 0: stat['t-dpd-breaks'] = dbreaks['breaks'] stat['thres'] = thres stat['borders'] = borders stat['sample_size'] = sample_size stat['quantilen'] = quantilen stat['source'] = data stat['savename'] = savename stat['variable'] = var stat['daynight'] = daynight if 'detect_opts' not in isonde.attrs: isonde.add_attr('detect_opts', {"%s_%s" % (savename, var): stat}) # sdcor_dpd else: isonde.detect_opts["%s_%s" % (savename, var)] = stat # update if save: isonde.save(var=savename, update=True, verbose=verbose) return stat
def save(self, var=None, directory=None, update=False, hdfformat='table', hdfcomplevel=9, hdfcomplib='blosc', verbose=0): """ Write Radiosonde Class Object to Directory Parameters ---------- var str/list directory str update bool force bool mode str hdfformat str hdfcomplevel int hdfcomplib str verbose int """ from raso.config import rasodir import pickle if directory is None: if self.directory is None: directory = rasodir else: directory = self.directory default = directory + '/%s/' % self.id if not os.path.isdir(default): os.makedirs(default) if var is None: varis = self.vars # All Variables else: if isinstance(var, str): varis = [var] else: varis = var update = True # Make sure we write it journal("Saving to dir: %s" % default, self.history, verbose) for ivar in varis: written = False if self.var_saved[ivar] and not update: if os.path.isfile(default + "%s.h5" % ivar) or os.path.isfile(default + "%s.pickle" % ivar): print_verbose("%20s [%5s]" % (ivar, color_boolean(written)), verbose) continue tmp = getattr(self, ivar) # # Try as HDF # if isinstance(tmp, (pd.Series, pd.DataFrame, pd.Panel)): try: tmp.to_hdf(default + "%s.h5" % ivar, ivar, mode='a', format=hdfformat, complevel=hdfcomplevel, complib=hdfcomplib) fio = 'H5' written = True n = tmp.shape except Exception as e: if verbose: print ivar, " >> ", repr(e) # # And as Pickle # if not written: try: pickle.dump(tmp, open(default + "%s.pickle" % ivar, 'w')) fio = 'PKL' written = True n = len(tmp) except Exception as e: if verbose: print ivar, " >> ", repr(e) print_verbose("%20s %20s [%5s] %s" % (ivar, n, color_boolean(written), fio), verbose) attrs = {} for iatt in self.attrs: attrs[iatt] = getattr(self, iatt) pickle.dump(attrs, open(default + 'attributes.pickle', 'w')) attrs = {} for s in self.history: if 'Saved' in s: self.history.pop(self.history.index(s)) journal("Saved", self.history, 0) for iatt in ['id', 'is_saved', 'is_empty']: attrs[iatt] = getattr(self, iatt) pickle.dump(attrs, open(default + 'radiosonde.pickle', 'w')) f = open(default+'history.txt', 'w') f.write("\n".join(self.history)) f.close() if self.notes != "": f = open(default + 'notes.txt', 'w') f.write(self.notes) f.close() self.directory = directory