def _2d_detection(data, var, window, dist, thres, min_levels, verbose=0):
""" Detect Breakpoints in 2D (time x pressure levels)
1. Run SNHT per level
2. Count significant peaks from all levels
2.1. Local Maxima (>END)
2.2. Above Threshold
3. add breaks variable with 1 and 2 (break in that level)
Parameters
----------
data Panel Inputdata (vars, time, p)
var str Variable to consider
window float Window size of Detection
dist float distance between breakpoints
thres int SNHT threshold
min_levels int Minimum required levels to detect breakpoint
verbose
Returns
-------
"""
stest = np.squeeze(np.apply_along_axis(snht, 0, data[var].values, window, window / 4)) # per level
data['%s_snht' % var] = stest
data['%s_breaks' % var] = 0
imax = np.asarray(local_maxima(np.sum(stest, 1), dist=dist)) # for all levels
if len(imax) == 0:
print_verbose(
"No Breakpoints detected: %s min_levels (%d) found: %f (%f)" % (var, min_levels, np.max(stest), thres),
verbose)
return False
print_verbose("Local Maxima (%s): %d" % (var, len(imax)), verbose)
# how many above threshold
tstat = np.sum(stest[imax, :] > thres, 1) # could weight levels ? upper levels are less relevant?
if not np.any(tstat >= min_levels):
print_verbose("No Breakpoints detected: %s min_levels (%d) found: %d" % (var, min_levels, np.max(tstat)),
verbose)
return False
else:
if verbose > 1:
print "Breaks (%s): " % var
for i, ib in enumerate(tstat):
print "%s (%d) %s" % (str(data.major_axis[imax[i]]), ib, color_boolean(ib >= min_levels))
print_verbose("Final Breaks (%s): %d" % (var, np.sum(tstat >= min_levels)), verbose)
itx = imax[tstat >= min_levels] # indices of breaks
ivar = data.items.get_loc("%s_breaks" % var) # index of variable
iarray = 1 + np.int_(stest[itx, :] > thres)
data.iloc[ivar, itx, :] = iarray[np.newaxis, ::] # (1,...) is required to set
return True
def from_store(ident, variables=None, directory=None, verbose=0):
""" Read Radiosonde from Store
Skips any directory or file with extract in their name
As well as mars_dump.h5
Parameters
----------
ident str
variables list
directory str
verbose int
Returns
-------
radiosonde
Examples
--------
>>>import raso
>>>raso.read.from_store('011035')
"""
from raso.sonde.radiosonde import radiosonde
from raso.config import rasodir
if directory is None:
directory = rasodir
default = directory + '/%s/' % ident
if not os.path.isdir(default):
raise IOError("[from_store] Requires a valid directory: %s" % default)
if variables is not None:
if isinstance(variables, str):
variables = [variables]
out = radiosonde(id=ident)
for ifile in os.listdir(default):
if ifile == 'attributes.pickle':
attrs = pickle.load(open(default + 'attributes.pickle'))
for ikey, ival in attrs.items():
out.add_attr(ikey, ival)
elif ifile == 'radiosonde.pickle':
attrs = pickle.load(open(default + 'radiosonde.pickle'))
for ikey, ival in attrs.items():
setattr(out, ikey, ival)
elif ifile == 'history.txt':
f = open(default + 'history.txt')
setattr(out, 'history', f.read().splitlines())
f.close()
elif ifile == 'notes.txt':
f = open(default + 'notes.txt')
setattr(out, 'notes', f.read())
f.close()
elif ifile == 'mars_dump.h5':
print_verbose("%20s Skipped" % ifile, verbose)
continue
elif os.path.isdir(default+ifile):
continue # any directory is ignored
elif 'extract' in ifile:
print_verbose("%20s Skipped" % ifile, verbose) # extract is a codename for temp files
continue
else:
varname = ifile.split('.')[0]
if variables is not None:
if varname not in variables:
print_verbose("%20s [%5s]" % (varname, color_boolean(False)), verbose)
continue
# TODO add file timestamp as info
# print varname, "created: %s" % time.ctime(os.path.getctime(default + ifile)), time.ctime(os.path.getmtime(default + ifile))
if 'h5' in ifile:
out.add_data(varname, pd.read_hdf(default + ifile, varname), verbose=-1) # not reported
else:
out.add_data(varname, pickle.load(open(default + ifile)), verbose=-1) # not reported
out.var_saved[varname] = True
print_verbose("%20s [%5s]" % (varname, color_boolean(True),), verbose)
out.directory = directory
out.is_saved = True
return out
def show(self):
"""Representation
"""
from raso.config import wkdir
#
if not hasattr(self, 'id'):
text = "Empty Radiosonde "
return
#
if self.id is None:
text = "Empty Radiosonde "
return text
else:
text = "Radiosonde %s [%s]" % (self.id, color_boolean(self.is_saved))
#
# Variables
for ivar in self.vars:
itype = type(getattr(self, ivar))
# 1+20+3+20+2+?+1 = 47+? == 80
# 46 offset
#
text += "\n %20s : [%5s] " % (ivar, color_boolean(self.var_saved[ivar]))
if itype == pd.DataFrame:
text += "%20s (%s)" % (getattr(self, ivar).shape, format_varnames(getattr(self, ivar).columns.tolist(),
maxs=pd.options.display.width - 54,
offset=54))
elif itype == pd.Series:
text += "%20s " % getattr(self, ivar).shape
elif itype == pd.Panel:
text += "%20s (%s)" % (getattr(self, ivar).shape, format_varnames(getattr(self, ivar).items.tolist(),
maxs=pd.options.display.width - 54,
offset=54))
else:
text += "%s " % itype
#
# Info
if hasattr(self, 'lon'):
text += "\n Station: \n Lon: %5.2fE Lat: %5.2fN Alt: %dm" % (tuple((getattr(self, 'lon', -999),
getattr(self, 'lat', -999),
getattr(self, 'alt', -999))))
# Other Attributes ?
ilen = len(text)
for iatt in self.attrs:
if iatt in ['lon', 'lat', 'alt']:
continue
if isinstance(getattr(self, iatt), (str, bool, float, int)):
text += "\n %15s : %s" % (iatt, str(getattr(self, iatt)))
else:
text += "\n %15s : %d" % (iatt, len(getattr(self, iatt)))
if len(text) > ilen:
text = text[:ilen] + "\n Attributes:" + text[ilen:]
if self.directory is not None:
directory = self.directory + '/%s/' % self.id
dirsize = 0
current = 0
for ifile in os.listdir(directory):
dirsize += os.path.getsize(directory+ifile)
name, ext = os.path.splitext(ifile)
if name in self.vars:
current += os.path.getsize(directory+ifile)
dirsize = dirsize / 1024 / 1024
current = current / 1024 / 1024
text += "\n Directory: " + directory.replace(wkdir, '.') + ' [%d mb] (%d mb)' % (dirsize, current)
print text
def merge(isonde, jsonde, update=False, save=False, verbose=0):
"""
Read and merge two radiosonde class to one radiosonde
Parameters
----------
isonde radiosonde
jsonde radiosonde
update bool
save bool
verbose int
Returns
-------
radiosonde / save
"""
from . import radiosonde
if not isinstance(isonde, radiosonde):
raise ValueError("[MERGE] Requires a radiosonde class: isonde")
if not isinstance(jsonde, radiosonde):
raise ValueError("[MERGE] Requires a radiosonde class: jsonde")
for ivar in jsonde.vars:
if ivar in isonde.vars:
if update:
setattr(isonde, ivar, getattr(jsonde, ivar))
print_verbose("[MERGE] %s replaced [%s]" % (ivar, color_boolean(update)), verbose)
else:
isonde.add(ivar, getattr(jsonde, ivar))
print_verbose("[MERGE] New variable: %s" % ivar, verbose)
# Attributes ?
for iatt in jsonde.attrs:
if iatt in isonde.attrs:
if getattr(isonde, iatt) == getattr(jsonde, iatt):
continue
elif iatt in ['history', 'is_saved']:
continue
elif update:
isonde.add_attr(iatt, getattr(jsonde, iatt))
else:
isonde.add_attr("new_" + iatt, getattr(jsonde, iatt))
print_verbose("[MERGE] %s replaced [%s]" % (iatt, color_boolean(update)), verbose)
else:
isonde.add_attr(iatt, getattr(jsonde, iatt))
print_verbose("[MERGE] New Attribute: %s " % ("new_" + iatt), verbose)
# History ?
isonde.history = history_duplicates(isonde.history + jsonde.history)
if save:
isonde.save(verbose=verbose)
else:
isonde.is_saved = False
return isonde
def save(self, var=None, directory=None, update=False, hdfformat='table', hdfcomplevel=9, hdfcomplib='blosc', verbose=0):
""" Write Radiosonde Class Object to Directory
Parameters
----------
var str/list
directory str
update bool
force bool
mode str
hdfformat str
hdfcomplevel int
hdfcomplib str
verbose int
"""
from raso.config import rasodir
import pickle
if directory is None:
if self.directory is None:
directory = rasodir
else:
directory = self.directory
default = directory + '/%s/' % self.id
if not os.path.isdir(default):
os.makedirs(default)
if var is None:
varis = self.vars # All Variables
else:
if isinstance(var, str):
varis = [var]
else:
varis = var
update = True # Make sure we write it
journal("Saving to dir: %s" % default, self.history, verbose)
for ivar in varis:
written = False
if self.var_saved[ivar] and not update:
if os.path.isfile(default + "%s.h5" % ivar) or os.path.isfile(default + "%s.pickle" % ivar):
print_verbose("%20s [%5s]" % (ivar, color_boolean(written)), verbose)
continue
tmp = getattr(self, ivar)
#
# Try as HDF
#
if isinstance(tmp, (pd.Series, pd.DataFrame, pd.Panel)):
try:
tmp.to_hdf(default + "%s.h5" % ivar, ivar, mode='a', format=hdfformat, complevel=hdfcomplevel, complib=hdfcomplib)
fio = 'H5'
written = True
n = tmp.shape
except Exception as e:
if verbose:
print ivar, " >> ", repr(e)
#
# And as Pickle
#
if not written:
try:
pickle.dump(tmp, open(default + "%s.pickle" % ivar, 'w'))
fio = 'PKL'
written = True
n = len(tmp)
except Exception as e:
if verbose:
print ivar, " >> ", repr(e)
print_verbose("%20s %20s [%5s] %s" % (ivar, n, color_boolean(written), fio), verbose)
attrs = {}
for iatt in self.attrs:
attrs[iatt] = getattr(self, iatt)
pickle.dump(attrs, open(default + 'attributes.pickle', 'w'))
attrs = {}
for s in self.history:
if 'Saved' in s:
self.history.pop(self.history.index(s))
journal("Saved", self.history, 0)
for iatt in ['id', 'is_saved', 'is_empty']:
attrs[iatt] = getattr(self, iatt)
pickle.dump(attrs, open(default + 'radiosonde.pickle', 'w'))
f = open(default+'history.txt', 'w')
f.write("\n".join(self.history))
f.close()
if self.notes != "":
f = open(default + 'notes.txt', 'w')
f.write(self.notes)
f.close()
self.directory = directory
