def make_hdf():
datadir = path.join(home(), 'DATA', 'HYY_DenData')
p200dir = path.join(datadir, 'Pluvio200')
p400dir = path.join(datadir, 'Pluvio400')
pipdir = path.join(datadir, 'PIP')
psddir = path.join(pipdir, 'f_1_4_DSD_Tables_ascii')
vdir = path.join(pipdir, 'f_2_2_Velocity_Tables')
h5file = path.join(datadir, 'annakaisa15-18.h5')
pluv_paths = dict(pluvio200=p200dir, pluvio400=p400dir)
instr = dict()
# read pluvio data
for p in ['pluvio200', 'pluvio400']:
fnames = glob(path.join(pluv_paths[p], '*.txt'))
fnames.sort()
pluv = pluvio.Pluvio(fnames, name=p)
selection = pluv.data.i_rt.apply(lambda x: type(x) == float)
pluv.data = pluv.data[selection].astype(float)
instr[p] = pluv
# read pip data
vfiles = glob(path.join(vdir, '*', '*.dat'))
vfiles.sort()
psdfiles = glob(path.join(psddir, '*.dat'))
psdfiles.sort()
instr['vel'] = pip_v.PipV(vfiles)
instr['dsd'] = pip_psd.PipPSD(psdfiles)
for i in instr.values():
i.to_hdf(filename=h5file)
def make_hdf():
datadir = path.join(home(), 'DATA', 'HYY_DenData')
p200dir = path.join(datadir, 'Pluvio200')
p400dir = path.join(datadir, 'Pluvio400')
pipdir = path.join(datadir, 'PIP')
psddir = path.join(pipdir, 'f_1_4_DSD_Tables_ascii')
vdir = path.join(pipdir, 'f_2_2_Velocity_Tables')
h5file = path.join(datadir, 'annakaisa15-18.h5')
pluv_paths = dict(pluvio200=p200dir, pluvio400=p400dir)
instr = dict()
# read pluvio data
for p in ['pluvio200', 'pluvio400']:
fnames = glob(path.join(pluv_paths[p], '*.txt'))
fnames.sort()
pluv = pluvio.Pluvio(fnames, name=p)
selection = pluv.data.i_rt.apply(lambda x: type(x)==float)
pluv.data = pluv.data[selection].astype(float)
instr[p] = pluv
# read pip data
vfiles = glob(path.join(vdir, '*', '*.dat'))
vfiles.sort()
psdfiles = glob(path.join(psddir, '*.dat'))
psdfiles.sort()
instr['vel'] = pip_v.PipV(vfiles)
instr['dsd'] = pip_psd.PipPSD(psdfiles)
for i in instr.values():
i.to_hdf(filename=h5file)
def fr(self):
"""rime mass fraction"""
t_end = self.t_end() + pd.Timedelta(minutes=15)
hdfpath = path.join(home(), 'DATA', 'FR_haoran.h5')
if not path.exists(hdfpath):
return pd.Series()
fr = pd.read_hdf(hdfpath, 'data')[self.t_start():t_end]
return self.time_weighted_mean(fr, offset_half_delta=False)
def load_pluvio(start=None, end=None, kind='400'):
"""Load Pluvio data from hdf5 database."""
import baecc.instruments.pluvio as pl
name = 'pluvio{}'.format(str(kind))
hdfpath = path.join(home(), 'DATA', 'pluvio14-16.h5')
data = pd.read_hdf(hdfpath, key=name)[start:end]
pluv = pl.Pluvio(data=data, name=name)
return pluv
def t_surface(self, use_arm=False, interp_gaps=True):
"""resampled ground temperature
Returns:
Series: resampled temperature
"""
t_end = self.t_end() + pd.Timedelta(minutes=15)
if use_arm:
t = arm.var_in_timerange(self.t_start(), t_end, var='temp_mean')
else:
hdfpath = path.join(home(), 'DATA', 't_fmi_14-17.h5')
if not path.exists(hdfpath):
return pd.Series()
t = pd.read_hdf(hdfpath, 'data')['TC'][self.t_start():t_end]
t.name = 'temp_mean'
tre = t.resample('15min', base=self.base_minute()).mean()
if interp_gaps:
tre = tre.interpolate()
return tre
import pandas as pd
import xarray as xr
import matplotlib as mpl
import matplotlib.pyplot as plt
from scipy.io import loadmat
from radcomp.vertical import (filtering, classification, plotting, insitu, ml,
deriv, NAN_REPLACEMENT)
from radcomp import arm, azs
from radcomp.tools import strftime_date_range, cloudnet
from j24 import home, daterange2str
USE_LEGACY_DATA = False
if USE_LEGACY_DATA:
DATA_DIR = path.join(home(), 'DATA', 'vprhi')
DATA_FILE_FMT = '%Y%m%d_IKA_VP_from_RHI.mat'
else:
DATA_DIR = path.join(home(), 'DATA', 'vprhi2')
DATA_FILE_FMT = '%Y%m%d_IKA_vprhi.mat'
DEFAULT_PARAMS = ['zh', 'zdr', 'kdp']
def case_id_fmt(t_start,
t_end=None,
dtformat='{year}{month}{day}{hour}',
day_fmt='%d',
month_fmt='%m',
year_fmt='%y',
hour_fmt='T%H'):
"""daterange2str wrapper for date range based IDs"""
# coding: utf-8
import pyart
import matplotlib.pyplot as plt
from os import path
from radcomp.vertical import RESULTS_DIR
from j24 import home
if __name__ == '__main__':
#plt.close('all')
rawfile = path.join(home(), 'DATA', '201402152010_IKA.RHI_HV.raw')
outfile = path.join(RESULTS_DIR, 'rhi_sample.png')
radar = pyart.io.read(rawfile)
fig, ax = plt.subplots()
display = pyart.graph.RadarDisplay(radar)
display.plot('differential_reflectivity', vmin=-0.25, vmax=2)
ax.set_ylim(bottom=0, top=10)
ax.set_xlim(left=0, right=80)
fig.savefig(outfile)
# coding: utf-8
import numpy as np
import pandas as pd
from scipy.io import loadmat
from os import path
from j24 import home, mldatenum2datetime
fpath = path.join(home(), 'DATA', 'FMI_meteo_data_2014_2017.mat')
hdfpath = path.join(home(), 'DATA', 't_fmi_14-17.h5')
data = loadmat(fpath)['meteo_fmi']
class TFMI:
"""FMI temperature mat struct"""
def __init__(self, data):
self.data = data # struct from mat
def get(self, param):
return data[param][0][0].flatten()
def time(self):
return np.array(list(map(mldatenum2datetime, self.get('ObsTime'))))
def fields(self):
return list(self.data[0].dtype.fields)
def to_dataframe(self):
f = self.fields() # copy
f.remove('ObsTime')
index = self.time()
# -*- coding: utf-8 -*-
"""
@author: Jussi Tiira
"""
import matplotlib.pyplot as plt
from os import path
from j24 import home, ensure_join
from scr_snowfall import pip2015events
#plt.close('all')
plt.ion()
basepath = ensure_join(home(),'results','pip2015','density')
dtfmt = '%Y%m%d'
e = pip2015events()
rho_label = 'bulk density (kg m$^{-3}$)'
t_label = 'time'
for c in e.events.paper.values:
savepath = basepath
rho = c.density()
rho.to_csv(path.join(savepath, c.dtstr(dtfmt) + '.csv'))
c.instr['pluvio'].tdelta().to_csv(path.join(savepath, 'timedelta_' + c.dtstr(dtfmt) + '.csv'))
plt.figure(dpi=120)
rho.plot(drawstyle='steps')
plt.title(c.dtstr())
plt.xlabel(t_label)
plt.ylabel(rho_label)
# coding: utf-8
import numpy as np
import netCDF4 as nc
import pandas as pd
from glob import glob
from datetime import datetime
from os import path
from j24 import home
arm_dir = path.join(home(), 'DATA', 'arm')
SOUNDING_DIR = path.join(arm_dir, 'sounding')
GROUND_DIR = path.join(arm_dir, 'ground')
MWR_DIR = path.join(arm_dir, 'MWR')
all_soundings_f = 'tmpsondewnpnM1.b1.20140121.125200..20140330.172000.custom.cdf'
sounding_f = 'tmpsondewnpnM1.b1.20140131.115000.cdf' # sample
all_soundings_path = path.join(SOUNDING_DIR, all_soundings_f)
sounding_path = path.join(SOUNDING_DIR, sounding_f)
SOUNDING_GLOB = path.join(SOUNDING_DIR,
'tmpsondewnpnM1.b1.20??????.??????.cdf')
GROUND_GLOB = path.join(GROUND_DIR, 'tmpmetM1.b1.20??????.??????.cdf')
MWR_GLOB = path.join(MWR_DIR, '*.cdf')
#s = nc.Dataset(soundings_f)
def time(ncdata, as_np=False):
"""time from ARM netCDF"""
t0 = ncdata.variables['base_time'][0]
if as_np:
return np.array(t0 + ncdata.variables['time_offset'][:]).astype(
'datetime64[s]')
return pd.to_datetime(t0 + ncdata.variables['time_offset'][:], unit='s')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import clustering as cl
import sonde
from os import path
from sklearn.cluster import KMeans
from j24 import home, learn
import datetime
np.random.seed(42)
plt.ion()
plt.close('all')
datadir = path.join(home(), 'DATA', 'pangaea', 'sonde')
storage_path = path.join(datadir, '96-16prep.h5')
#data = sonde.read_all_data(datadir)
#data.to_hdf(storage_path, 'data')
data = pd.read_hdf(storage_path, 'data')
#times = sonde.launch_times(data)
ww = cl.wind4clus(data)
tt = cl.t4clus(data, col='t')
hh = cl.t4clus(data, col='rh')
clus_vars = (ww, tt, hh)
km = KMeans(init='k-means++', n_clusters=6, n_init=40, n_jobs=-1)
wtr, isplit = cl.concat(clus_vars)
classes = learn.fit_predict(wtr, km)
cen = learn.centroids(wtr, km)
cw, ct, ch = cl.split(cen, isplit)
# coding: utf-8
import os
import warnings
import pickle
import hashlib
import pandas as pd
from j24 import home, ensure_dir
MSGTLD = '.msg'
PICKLETLD = '.pkl'
CACHE_DIR = os.path.join(home(), '.cache', 'baecc')
def fingerprint(string):
return hashlib.sha256(string.encode('utf-8')).hexdigest()[-12:]
def hash_dict(d):
return fingerprint(str(sorted(d.items())))
def combine2str(*identifiers):
return ''.join(tuple(map(str, identifiers)))
class Cacher:
"""common methods to use msg cache"""
def __init__(self, use_cache=True, storefilename='store.h5', parent=None):
self.use_cache = use_cache
self.storefilename = storefilename
self.parent = parent
border = basemap.border()
x0 = 1.1e5
y0 = 6.55e6
x1 = 6.5e5
y1 = 7e6
#urls = fmi.available_maps(**t_range)
fakeindex = pd.DatetimeIndex(freq='5min',
start=t_range['starttime'],
end=t_range['endtime'])
urls = pd.Series(index=fakeindex) # fake urls
#url = fmi.gen_url(timestamp='2017-10-17T07:00:00Z')
dl = fmi.available_maps().tail(2)
#fmi.download_maps(urls)
paths = fmi.download_maps(dl)
savedir = ensure_join(home(), 'results', 'sataako')
### FORECAST AND SAVE LOGIC ###
rads = paths.apply(rasterio.open)
crops, tr, meta = raster.crop_rasters(rads, **raster.DEFAULT_CORNERS)
dtype = meta['dtype']
rad_crs = rads.iloc[0].read_crs().data
rads.apply(lambda x: x.close())
rr = raster.raw2rr(crops)
fcast = forecast.forecast(rr)
savepaths = fcast.copy()
pngpaths = fcast.copy()
for t, fc in fcast.iteritems():
savepath = path.join(savedir, t.strftime(fmi.FNAME_FORMAT))
savepaths.loc[t] = savepath
raster.write_rr_geotiff(fc, meta, savepath)
# coding: utf-8
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from os import path
from datetime import timedelta
from j24 import home, ensure_join
CACHE_DIR = ensure_join(home(), '.pysonde', 'cache')
CACHE_KEY_FMT = 'wyo%Y%m%d%H'
def round_hours(timestamp, hres=12):
"""round timestamp to hres hours"""
tt = timestamp + timedelta(hours=hres / 2)
dt = timedelta(hours=tt.hour % hres, minutes=tt.minute, seconds=tt.second)
return tt - dt
def sounding_url(t, dtype='text'):
out_type = dict(pdf='PDF%3ASTUVE', text='TEXT%3ASIMPLE')
baseurl = 'http://weather.uwyo.edu/cgi-bin/sounding'
query = '?region=europe&TYPE={type}&YEAR={year}&MONTH={month:02d}&FROM={day:02d}{hour:02d}&TO={day:02d}{hour:02d}&STNM=02963'
urlformat = baseurl + query
return urlformat.format(type=out_type[dtype],
year=t.year,
month=t.month,
day=t.day,
hour=t.hour)
#import numpy as np
#import pandas as pd
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import radcomp.visualization as vis
from matplotlib import gridspec
from os import path
from scipy.io import loadmat
from j24 import home
from radcomp.qpe.radar import RADARS
I_RADAR = dict(ker=0, kum=1, van=2, com=3)
NAMES = dict(ker='KER', kum='KUM', van='VAN', com='Composite')
RADAR = dict(ker=RADARS['KER'], kum=RADARS['KUM'], van=RADARS['VAN'])
RESULTSDIR = path.join(home(), 'results', 'radcomp', 'brandon')
PROJECTION = ccrs.Orthographic(central_longitude=25, central_latitude=60)
def datalist4radar(i_radar, data, param='kdp'):
dat = data['{}_three_radars'.format(param)]
n_timesteps = dat.shape()[3]
return [dat[:, :, i_radar, i] for i in range(n_timesteps)]
def datalist4timestep(i_timestep, data, param='kdp'):
if param=='r':
sitestrs = ['ker', 'kum', 'van', 'c']
return [data['rain_{}'.format(site)][:, :, i_timestep] for site in sitestrs]
dat = data['{}_three_radars'.format(param)]
n_radars = dat.shape[2]
# coding: utf-8
import os
import warnings
import pickle
import hashlib
import pandas as pd
from j24 import home, ensure_dir
MSGTLD = '.msg'
PICKLETLD = '.pkl'
CACHE_DIR = os.path.join(home(), '.cache', 'baecc')
def fingerprint(string):
return hashlib.sha256(string.encode('utf-8')).hexdigest()[-12:]
def hash_dict(d):
return fingerprint(str(sorted(d.items())))
def combine2str(*identifiers):
return ''.join(tuple(map(str, identifiers)))
class Cacher:
"""common methods to use msg cache"""
def __init__(self, use_cache=True, storefilename='store.h5',
parent=None):
self.use_cache = use_cache
self.storefilename = storefilename
# coding: utf-8
from __future__ import absolute_import, division, print_function, unicode_literals
__metaclass__ = type
import pandas as pd
from os import path
from glob import glob
from baecc.instruments import pluvio, pip_psd, pip_v
from j24 import home, ensure_join
RESULTS_DIR = ensure_join(home(), 'results', 'density18')
def make_hdf():
datadir = path.join(home(), 'DATA', 'HYY_DenData')
p200dir = path.join(datadir, 'Pluvio200')
p400dir = path.join(datadir, 'Pluvio400')
pipdir = path.join(datadir, 'PIP')
psddir = path.join(pipdir, 'f_1_4_DSD_Tables_ascii')
vdir = path.join(pipdir, 'f_2_2_Velocity_Tables')
h5file = path.join(datadir, 'annakaisa15-18.h5')
pluv_paths = dict(pluvio200=p200dir, pluvio400=p400dir)
instr = dict()
# read pluvio data
for p in ['pluvio200', 'pluvio400']:
fnames = glob(path.join(pluv_paths[p], '*.txt'))
fnames.sort()
pluv = pluvio.Pluvio(fnames, name=p)
selection = pluv.data.i_rt.apply(lambda x: type(x) == float)
pluv.data = pluv.data[selection].astype(float)
# coding: utf-8
import pandas as pd
from glob import glob
from datetime import datetime, timedelta
from os import path
from scipy.io import loadmat
from j24 import home
DATAPATH = path.join(home(), 'DATA')
hdfpath = path.join(DATAPATH, 'FR_haoran.h5')
def path_dstr(fpath, start_char=4):
fname = path.basename(fpath)
return fname[start_char:start_char + 8]
def parse_date_from_fpath(fpath, **kws):
dstr = path_dstr(fpath, **kws)
return datetime.strptime(dstr, '%Y%m%d')
def times_from_file(fpath, **kws):
date = parse_date_from_fpath(fpath, **kws)
data = loadmat(fpath)
h = data['PIP_time'].flatten()
return list(map(lambda hh: date + timedelta(hours=hh), h))
# coding: utf-8
from __future__ import absolute_import, division, print_function, unicode_literals
__metaclass__ = type
import pandas as pd
from os import path
from glob import glob
from baecc.instruments import pluvio, pip_psd, pip_v
from j24 import home, ensure_join
RESULTS_DIR = ensure_join(home(), 'results', 'density18')
def make_hdf():
datadir = path.join(home(), 'DATA', 'HYY_DenData')
p200dir = path.join(datadir, 'Pluvio200')
p400dir = path.join(datadir, 'Pluvio400')
pipdir = path.join(datadir, 'PIP')
psddir = path.join(pipdir, 'f_1_4_DSD_Tables_ascii')
vdir = path.join(pipdir, 'f_2_2_Velocity_Tables')
h5file = path.join(datadir, 'annakaisa15-18.h5')
pluv_paths = dict(pluvio200=p200dir, pluvio400=p400dir)
instr = dict()
# read pluvio data
for p in ['pluvio200', 'pluvio400']:
fnames = glob(path.join(pluv_paths[p], '*.txt'))
fnames.sort()
pluv = pluvio.Pluvio(fnames, name=p)
selection = pluv.data.i_rt.apply(lambda x: type(x)==float)
pluv.data = pluv.data[selection].astype(float)
# coding: utf-8
import pandas as pd
from glob import glob
from os import path
from scipy.io import loadmat
from j24 import home, mldatenum2datetime
datadir = path.join(home(), 'DATA', 'BAECC_1308_AVL')
dataset = set(glob(path.join(datadir, 'Snow_*.mat')))
P200SET = set(glob(path.join(datadir, 'Snow_*PL200.mat')))
P400SET = dataset - P200SET
def mat_data(filename):
dat = loadmat(filename)
ind = next(i for i, x in enumerate(dat.keys()) if 'Snow_' in x)
data = list(dat.values())[ind][0]
keys = data.dtype.names
values = data[0]
return dict(zip(keys, values))
def mat2series(filename, key='azs'):
data = mat_data(filename)
values = data[key].flatten()
t = list(map(mldatenum2datetime, data['time'].flatten()))
series = pd.Series(data=values, index=t)
series.name = key
return series
# coding: utf-8
import locale
from os import path
from j24 import home
locale.setlocale(locale.LC_ALL, 'C')
# general configuration
DEBUG = False
CGS_UNITS = True # display units in cgs instead of SI
# CONFIG default paths
HOME = home()
DATA_DIR = path.join(HOME, 'DATA')
USER_DIR = path.join(HOME, '.baecc')
RESULTS_DIR = path.join(HOME, 'results')
H5_PATH = path.join(DATA_DIR, 'baecc.h5')
# constants
if CGS_UNITS:
RHO_SCALE = 1e-3
RHO_UNITS = 'g$\,$cm$^{-3}$'
else:
RHO_SCALE = 1
RHO_UNITS = 'kg$\,$m$^{-3}$'
from baecc.tools import merge_series, merge_multiseries
from baecc import events
# coding: utf-8
import locale
from os import path
from j24 import home
locale.setlocale(locale.LC_ALL, 'C')
# general configuration
DEBUG = False
CGS_UNITS = True # display units in cgs instead of SI
# CONFIG default paths
HOME = home()
DATA_DIR = path.join(HOME, 'DATA')
USER_DIR = path.join(HOME, '.baecc')
RESULTS_DIR = path.join(HOME, 'results')
H5_PATH = path.join(DATA_DIR, 'baecc.h5')
# constants
if CGS_UNITS:
RHO_SCALE = 1e-3
RHO_UNITS = 'g$\,$cm$^{-3}$'
else:
RHO_SCALE = 1
RHO_UNITS = 'kg$\,$m$^{-3}$'
from baecc.tools import merge_series, merge_multiseries
from baecc import events
