def store_read(self, tablename, default_value=None, nocache_value=None):
"""Read from hdf store if using caching."""
if self.use_cache:
ensure_dir(self.cache_dir())
try:
with pd.HDFStore(self.store_path()) as store:
return store.get(tablename)
except KeyError as err:
warnings.warn("KeyError: {0} Using default value.".format(err))
return default_value
return nocache_value
def msg_io(self, name, func, **kwargs):
"""Read data from msgpack. If not available, calculate and store."""
cd = self.cache_dir()
msgpath = os.path.join(cd, name + MSGTLD)
if os.path.isfile(msgpath):
data = pd.read_msgpack(msgpath)
else:
ensure_dir(cd)
data = func(**kwargs)
data.to_msgpack(msgpath)
return data
def store_read(self, tablename, default_value=None, nocache_value=None):
"""Read from hdf store if using caching."""
if self.use_cache:
ensure_dir(self.cache_dir())
try:
with pd.HDFStore(self.store_path()) as store:
return store.get(tablename)
except KeyError as err:
warnings.warn("KeyError: {0} Using default value.".format(err))
return default_value
return nocache_value
def msg_io(self, name, func, **kwargs):
"""Read data from msgpack. If not available, calculate and store."""
cd = self.cache_dir()
msgpath = os.path.join(cd, name + MSGTLD)
if os.path.isfile(msgpath):
data = pd.read_msgpack(msgpath)
else:
ensure_dir(cd)
data = func(**kwargs)
data.to_msgpack(msgpath)
return data
def pkl_io(self, name, func, **kwargs):
cd = self.cache_dir()
pklpath = os.path.join(cd, name + PICKLETLD)
if os.path.isfile(pklpath):
with open(pklpath, 'rb') as cachefile:
data = pickle.load(cachefile)
else:
ensure_dir(cd)
data = func(**kwargs)
with open(pklpath, 'wb') as cachefile:
pickle.dump(data, cachefile, pickle.HIGHEST_PROTOCOL)
return data
def pkl_io(self, name, func, **kwargs):
cd = self.cache_dir()
pklpath = os.path.join(cd, name + PICKLETLD)
if os.path.isfile(pklpath):
with open(pklpath, 'rb') as cachefile:
data = pickle.load(cachefile)
else:
ensure_dir(cd)
data = func(**kwargs)
with open(pklpath, 'wb') as cachefile:
pickle.dump(data, cachefile, pickle.HIGHEST_PROTOCOL)
return data
def batch_interpolate(filepaths_good, outpath, data_site=None, save_png=False,
interval_s=10.):
interval_dt = datetime.timedelta(seconds=interval_s)
for f0, f1 in itertools.izip(filepaths_good, filepaths_good[1:]):
if data_site is None:
for site in radx.SITES:
if site in f0:
data_site = site
nc0 = radx.RADXgrid(f0)
nc1 = radx.RADXgrid(f1)
if data_site == 'KER':
nc0, nc1 = radx.equalize_ker_zmin(nc0, nc1)
r0 = nc0.rainrate()
r1 = nc1.rainrate()
t0 = nc0.elevation_end_time()
t1 = nc1.elevation_end_time()
dt = t1-t0
n = int(round(dt.total_seconds()/interval_s))
intrp_timestamps = [t0+i*interval_dt for i in range(1, n+1, 1)]
selection = [t.second<10 for t in intrp_timestamps]
print(dt.total_seconds())
intrp = np.array(interp(r0, r1, n))
for i in np.where(selection)[0]:
t = intrp_timestamps[i]
r = intrp[i]
datedir = t.strftime('%Y%m%d')
fbasename = t.strftime('intrp_%Y%m%d_%H%M%S')
matfname = fbasename + '.mat'
matsitepath = ensure_dir(path.join(outpath, data_site, 'R', 'mat', datedir))
matfilepath = path.join(matsitepath, matfname)
mdict = {'time': np.array(str(t)), 'R': r}
scipy.io.savemat(matfilepath, mdict, do_compression=True)
if save_png:
pngfname = fbasename + '.png'
pngsitepath = ensure_dir(path.join(outpath, data_site, 'R', 'png', datedir))
pngfilepath = path.join(pngsitepath, pngfname)
fig, ax = radx.plot_rainmap(r)
ax.set_title(str(t))
fig.savefig(pngfilepath, bbox_inches="tight")
plt.close(fig)
def plot_flip(self, axarr, f, vfit, data, datai, origdata, perr):
filterstr = ['D-binned filter', 'v-binned filter', 'unfiltered']
for ax in axarr:
vfit.plot(ax=ax, label='original %.4f' % perr[1])
self.plot(ax=axarr[0], data=data, ymax=3)
self.plot(ax=axarr[1], data=datai, ymax=3)
self.plot(ax=axarr[2], data=origdata, ymax=3)
for i, ax in enumerate(axarr):
ax.set_title(ax.get_title() + ', ' + filterstr[i])
plt.legend()
f.tight_layout()
fname = data.index[-1].strftime('%H%M.eps')
datedir = data.index[-1].strftime('%Y%m%d')
f.savefig(path.join(ensure_dir(path.join(baecc.RESULTS_DIR, 'pip2015',
'fitcomparison',
datedir)), fname))
return axarr
def plot_flip(self, axarr, f, vfit, data, datai, origdata, perr):
filterstr = ['D-binned filter', 'v-binned filter', 'unfiltered']
for ax in axarr:
vfit.plot(ax=ax, label='original %.4f' % perr[1])
self.plot(ax=axarr[0], data=data, ymax=3)
self.plot(ax=axarr[1], data=datai, ymax=3)
self.plot(ax=axarr[2], data=origdata, ymax=3)
for i, ax in enumerate(axarr):
ax.set_title(ax.get_title() + ', ' + filterstr[i])
plt.legend()
f.tight_layout()
fname = data.index[-1].strftime('%H%M.eps')
datedir = data.index[-1].strftime('%Y%m%d')
f.savefig(
path.join(
ensure_dir(
path.join(baecc.RESULTS_DIR, 'pip2015', 'fitcomparison',
datedir)), fname))
return axarr
save = True
plot_by_class = False
plt.ion()
#plt.close('all')
n_eigens = 25
n_clusters = 25
reduced = True
use_temperature = True
t_weight_factor = 0.8
radar_weight_factors = dict(zdr=0.5)
cases = case.read_cases('14-16by_hand')
name = classification.scheme_name(basename='14-16',
n_eigens=n_eigens,
n_clusters=n_clusters,
reduced=reduced,
use_temperature=use_temperature,
t_weight_factor=t_weight_factor,
radar_weight_factors=radar_weight_factors)
results_dir = ensure_dir(path.join(RESULTS_DIR, 'classes_summary', name))
c = case.Case.by_combining(cases)
c.load_classification(name)
f_cen, axarr_cen = c.plot_cluster_centroids(cmap='viridis',
colorful_bars='blue')
#c.classes[c.classes==14]
from j24 import ensure_dir, ensure_join
N_COMB_INTERVALS = 2
dtformat_default = '%Y-%m-%d %H:%M'
dtformat_snex = '%Y %d %B %H UTC'
dtformat_paper = '%Y %b %d %H:%M'
cond = lambda df: (df.intensity>0.2) & (df.D_0_gamma>0.6) & \
(df.density==df.density) & (df['count']>800)
RHO_LIMITS = (0, 100, 200, 1000)
#rholimits = (0, 150, 300, 800)
resultspath = path.join(RESULTS_DIR, 'pip2015')
paperpath = path.join(resultspath, 'paper')
paths = {
'results': ensure_dir(resultspath),
'paper': paperpath,
'tables': ensure_join(paperpath, 'tables')
}
files = {
'h5nov14': path.join(DATA_DIR, '2014nov1-23.h5'),
'h5w1415': path.join(DATA_DIR, 'dec-jan1415.h5'),
'h5baecc': H5_PATH,
'params_cache': path.join(caching.CACHE_DIR,
'param_table' + caching.MSGTLD)
}
def cases_filepath(name):
return path.join(USER_DIR, 'cases', name + '.csv')
from collections import OrderedDict
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import decomposition
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score
from radcomp import learn, USER_DIR
from radcomp.vertical import preprocessing, plotting
from j24 import ensure_dir, limitslist
from j24.learn import pca_stats
META_SUFFIX = '_metadata'
MODEL_DIR = ensure_dir(path.join(USER_DIR, 'class_schemes'))
def weight_factor_str(param, value):
out = '_' + param
if value != 1:
out += str(value).replace('.', '')
return out
def scheme_name(basename='',
n_eigens=30,
n_clusters=20,
reduced=True,
extra_weight=0,
**kws):
def store_write(self, tablename, data):
"""Write data to hdf store."""
ensure_dir(self.cache_dir())
with pd.HDFStore(self.store_path()) as store:
store[tablename] = data
def store_write(self, tablename, data):
"""Write data to hdf store."""
ensure_dir(self.cache_dir())
with pd.HDFStore(self.store_path()) as store:
store[tablename] = data
plt.ioff()
plt.close('all')
case_set = '14-16by_hand'
n_eigens = 25
n_clusters = 20
reduced = True
save = True
cases = case.read_cases(case_set)
name = classification.scheme_name(basename='14-16_t',
n_eigens=n_eigens,
n_clusters=n_clusters,
reduced=reduced)
results_dir = ensure_dir(
path.join(RESULTS_DIR, 'z-lwe_comparison2', name, case_set))
def plot_ze_lwe_comparison(c):
z = c.cl_data.ZH.iloc[:, 1]
i = c.lwe(offset_half_delta=False)
iz = 10 * i**1.2
ioz = 10 * c.pluvio.intensity()**1.2
fig = plt.figure()
z.plot(drawstyle='steps')
iz.plot(drawstyle='steps')
#ioz.plot(drawstyle='steps')
return fig
for i, c in cases.case.iteritems():
from baecc import caching
from j24 import ensure_dir, ensure_join
N_COMB_INTERVALS = 2
dtformat_default = '%Y-%m-%d %H:%M'
dtformat_snex = '%Y %d %B %H UTC'
dtformat_paper = '%Y %b %d %H:%M'
cond = lambda df: (df.intensity>0.2) & (df.D_0_gamma>0.6) & \
(df.density==df.density) & (df['count']>800)
RHO_LIMITS = (0, 100, 200, 1000)
#rholimits = (0, 150, 300, 800)
resultspath = path.join(RESULTS_DIR, 'pip2015')
paperpath = path.join(resultspath, 'paper')
paths = {'results': ensure_dir(resultspath),
'paper': paperpath,
'tables': ensure_join(paperpath, 'tables')}
files = {'h5nov14': path.join(DATA_DIR, '2014nov1-23.h5'),
'h5w1415': path.join(DATA_DIR, 'dec-jan1415.h5'),
'h5baecc': H5_PATH,
'params_cache': path.join(caching.CACHE_DIR, 'param_table'+ caching.MSGTLD)}
def cases_filepath(name):
return path.join(USER_DIR, 'cases', name + '.csv')
def find_interval(x, limits=(0, 100, 200, 1000)):
"""Find rightmost value less than x and leftmost value greater than x."""
i = bisect.bisect_right(limits, x)
return limits[i-1:i+1]