def _2d_detection(data, var, window, dist, thres, min_levels, verbose=0): """ Detect Breakpoints in 2D (time x pressure levels) 1. Run SNHT per level 2. Count significant peaks from all levels 2.1. Local Maxima (>END) 2.2. Above Threshold 3. add breaks variable with 1 and 2 (break in that level) Parameters ---------- data Panel Inputdata (vars, time, p) var str Variable to consider window float Window size of Detection dist float distance between breakpoints thres int SNHT threshold min_levels int Minimum required levels to detect breakpoint verbose Returns ------- """ stest = np.squeeze(np.apply_along_axis(snht, 0, data[var].values, window, window / 4)) # per level data['%s_snht' % var] = stest data['%s_breaks' % var] = 0 imax = np.asarray(local_maxima(np.sum(stest, 1), dist=dist)) # for all levels if len(imax) == 0: print_verbose( "No Breakpoints detected: %s min_levels (%d) found: %f (%f)" % (var, min_levels, np.max(stest), thres), verbose) return False print_verbose("Local Maxima (%s): %d" % (var, len(imax)), verbose) # how many above threshold tstat = np.sum(stest[imax, :] > thres, 1) # could weight levels ? upper levels are less relevant? if not np.any(tstat >= min_levels): print_verbose("No Breakpoints detected: %s min_levels (%d) found: %d" % (var, min_levels, np.max(tstat)), verbose) return False else: if verbose > 1: print "Breaks (%s): " % var for i, ib in enumerate(tstat): print "%s (%d) %s" % (str(data.major_axis[imax[i]]), ib, color_boolean(ib >= min_levels)) print_verbose("Final Breaks (%s): %d" % (var, np.sum(tstat >= min_levels)), verbose) itx = imax[tstat >= min_levels] # indices of breaks ivar = data.items.get_loc("%s_breaks" % var) # index of variable iarray = 1 + np.int_(stest[itx, :] > thres) data.iloc[ivar, itx, :] = iarray[np.newaxis, ::] # (1,...) is required to set return True
def from_store(ident, variables=None, directory=None, verbose=0): """ Read Radiosonde from Store Skips any directory or file with extract in their name As well as mars_dump.h5 Parameters ---------- ident str variables list directory str verbose int Returns ------- radiosonde Examples -------- >>>import raso >>>raso.read.from_store('011035') """ from raso.sonde.radiosonde import radiosonde from raso.config import rasodir if directory is None: directory = rasodir default = directory + '/%s/' % ident if not os.path.isdir(default): raise IOError("[from_store] Requires a valid directory: %s" % default) if variables is not None: if isinstance(variables, str): variables = [variables] out = radiosonde(id=ident) for ifile in os.listdir(default): if ifile == 'attributes.pickle': attrs = pickle.load(open(default + 'attributes.pickle')) for ikey, ival in attrs.items(): out.add_attr(ikey, ival) elif ifile == 'radiosonde.pickle': attrs = pickle.load(open(default + 'radiosonde.pickle')) for ikey, ival in attrs.items(): setattr(out, ikey, ival) elif ifile == 'history.txt': f = open(default + 'history.txt') setattr(out, 'history', f.read().splitlines()) f.close() elif ifile == 'notes.txt': f = open(default + 'notes.txt') setattr(out, 'notes', f.read()) f.close() elif ifile == 'mars_dump.h5': print_verbose("%20s Skipped" % ifile, verbose) continue elif os.path.isdir(default+ifile): continue # any directory is ignored elif 'extract' in ifile: print_verbose("%20s Skipped" % ifile, verbose) # extract is a codename for temp files continue else: varname = ifile.split('.')[0] if variables is not None: if varname not in variables: print_verbose("%20s [%5s]" % (varname, color_boolean(False)), verbose) continue # TODO add file timestamp as info # print varname, "created: %s" % time.ctime(os.path.getctime(default + ifile)), time.ctime(os.path.getmtime(default + ifile)) if 'h5' in ifile: out.add_data(varname, pd.read_hdf(default + ifile, varname), verbose=-1) # not reported else: out.add_data(varname, pickle.load(open(default + ifile)), verbose=-1) # not reported out.var_saved[varname] = True print_verbose("%20s [%5s]" % (varname, color_boolean(True),), verbose) out.directory = directory out.is_saved = True return out
def show(self): """Representation """ from raso.config import wkdir # if not hasattr(self, 'id'): text = "Empty Radiosonde " return # if self.id is None: text = "Empty Radiosonde " return text else: text = "Radiosonde %s [%s]" % (self.id, color_boolean(self.is_saved)) # # Variables for ivar in self.vars: itype = type(getattr(self, ivar)) # 1+20+3+20+2+?+1 = 47+? == 80 # 46 offset # text += "\n %20s : [%5s] " % (ivar, color_boolean(self.var_saved[ivar])) if itype == pd.DataFrame: text += "%20s (%s)" % (getattr(self, ivar).shape, format_varnames(getattr(self, ivar).columns.tolist(), maxs=pd.options.display.width - 54, offset=54)) elif itype == pd.Series: text += "%20s " % getattr(self, ivar).shape elif itype == pd.Panel: text += "%20s (%s)" % (getattr(self, ivar).shape, format_varnames(getattr(self, ivar).items.tolist(), maxs=pd.options.display.width - 54, offset=54)) else: text += "%s " % itype # # Info if hasattr(self, 'lon'): text += "\n Station: \n Lon: %5.2fE Lat: %5.2fN Alt: %dm" % (tuple((getattr(self, 'lon', -999), getattr(self, 'lat', -999), getattr(self, 'alt', -999)))) # Other Attributes ? ilen = len(text) for iatt in self.attrs: if iatt in ['lon', 'lat', 'alt']: continue if isinstance(getattr(self, iatt), (str, bool, float, int)): text += "\n %15s : %s" % (iatt, str(getattr(self, iatt))) else: text += "\n %15s : %d" % (iatt, len(getattr(self, iatt))) if len(text) > ilen: text = text[:ilen] + "\n Attributes:" + text[ilen:] if self.directory is not None: directory = self.directory + '/%s/' % self.id dirsize = 0 current = 0 for ifile in os.listdir(directory): dirsize += os.path.getsize(directory+ifile) name, ext = os.path.splitext(ifile) if name in self.vars: current += os.path.getsize(directory+ifile) dirsize = dirsize / 1024 / 1024 current = current / 1024 / 1024 text += "\n Directory: " + directory.replace(wkdir, '.') + ' [%d mb] (%d mb)' % (dirsize, current) print text
def merge(isonde, jsonde, update=False, save=False, verbose=0): """ Read and merge two radiosonde class to one radiosonde Parameters ---------- isonde radiosonde jsonde radiosonde update bool save bool verbose int Returns ------- radiosonde / save """ from . import radiosonde if not isinstance(isonde, radiosonde): raise ValueError("[MERGE] Requires a radiosonde class: isonde") if not isinstance(jsonde, radiosonde): raise ValueError("[MERGE] Requires a radiosonde class: jsonde") for ivar in jsonde.vars: if ivar in isonde.vars: if update: setattr(isonde, ivar, getattr(jsonde, ivar)) print_verbose("[MERGE] %s replaced [%s]" % (ivar, color_boolean(update)), verbose) else: isonde.add(ivar, getattr(jsonde, ivar)) print_verbose("[MERGE] New variable: %s" % ivar, verbose) # Attributes ? for iatt in jsonde.attrs: if iatt in isonde.attrs: if getattr(isonde, iatt) == getattr(jsonde, iatt): continue elif iatt in ['history', 'is_saved']: continue elif update: isonde.add_attr(iatt, getattr(jsonde, iatt)) else: isonde.add_attr("new_" + iatt, getattr(jsonde, iatt)) print_verbose("[MERGE] %s replaced [%s]" % (iatt, color_boolean(update)), verbose) else: isonde.add_attr(iatt, getattr(jsonde, iatt)) print_verbose("[MERGE] New Attribute: %s " % ("new_" + iatt), verbose) # History ? isonde.history = history_duplicates(isonde.history + jsonde.history) if save: isonde.save(verbose=verbose) else: isonde.is_saved = False return isonde
def save(self, var=None, directory=None, update=False, hdfformat='table', hdfcomplevel=9, hdfcomplib='blosc', verbose=0): """ Write Radiosonde Class Object to Directory Parameters ---------- var str/list directory str update bool force bool mode str hdfformat str hdfcomplevel int hdfcomplib str verbose int """ from raso.config import rasodir import pickle if directory is None: if self.directory is None: directory = rasodir else: directory = self.directory default = directory + '/%s/' % self.id if not os.path.isdir(default): os.makedirs(default) if var is None: varis = self.vars # All Variables else: if isinstance(var, str): varis = [var] else: varis = var update = True # Make sure we write it journal("Saving to dir: %s" % default, self.history, verbose) for ivar in varis: written = False if self.var_saved[ivar] and not update: if os.path.isfile(default + "%s.h5" % ivar) or os.path.isfile(default + "%s.pickle" % ivar): print_verbose("%20s [%5s]" % (ivar, color_boolean(written)), verbose) continue tmp = getattr(self, ivar) # # Try as HDF # if isinstance(tmp, (pd.Series, pd.DataFrame, pd.Panel)): try: tmp.to_hdf(default + "%s.h5" % ivar, ivar, mode='a', format=hdfformat, complevel=hdfcomplevel, complib=hdfcomplib) fio = 'H5' written = True n = tmp.shape except Exception as e: if verbose: print ivar, " >> ", repr(e) # # And as Pickle # if not written: try: pickle.dump(tmp, open(default + "%s.pickle" % ivar, 'w')) fio = 'PKL' written = True n = len(tmp) except Exception as e: if verbose: print ivar, " >> ", repr(e) print_verbose("%20s %20s [%5s] %s" % (ivar, n, color_boolean(written), fio), verbose) attrs = {} for iatt in self.attrs: attrs[iatt] = getattr(self, iatt) pickle.dump(attrs, open(default + 'attributes.pickle', 'w')) attrs = {} for s in self.history: if 'Saved' in s: self.history.pop(self.history.index(s)) journal("Saved", self.history, 0) for iatt in ['id', 'is_saved', 'is_empty']: attrs[iatt] = getattr(self, iatt) pickle.dump(attrs, open(default + 'radiosonde.pickle', 'w')) f = open(default+'history.txt', 'w') f.write("\n".join(self.history)) f.close() if self.notes != "": f = open(default + 'notes.txt', 'w') f.write(self.notes) f.close() self.directory = directory