def csat_no_cal(files, outname, where):
v = vcm.VCM(files, verbose=False)
cal = v.get_vcm('cal333+cal05+cal20+cal80')
csat = v.get_vcm('csat')
calcloudy = np.sum(cal, axis=1)
csatcloudy = np.sum(csat, axis=1)
csatonly = np.where((csatcloudy > 0) & (calcloudy == 0))[0]
print 100. * csatonly.shape[0] / calcloudy.shape[0]
vcm_csatonly = np.zeros_like(csat)
vcm_csatonly[csatonly, :] = csat[csatonly, :]
csatcloudy[calcloudy > 0] = 0
dset = da.Dataset()
time_axis = da.Axis(v.time, 'tai_time')
alt_axis = da.Axis(v.altitude, 'altitude')
dset['lon'] = da.DimArray(v.lon, [time_axis])
dset['lat'] = da.DimArray(v.lat, [time_axis])
dset['csat'] = da.DimArray(vcm_csatonly, [time_axis, alt_axis])
dset['cloudpts'] = da.DimArray(csatcloudy, [time_axis])
check_out_dir(where)
dset.write_nc(where + outname, 'w', zlib=True, complevel=9)
def percentile(a, pct, axis=0, newaxis=None, out=None, overwrite_input=False):
""" calculate percentile along an axis
parameters:
-----------
pct: float, percentile or sequence of percentiles (0< <100)
axis, optional, default 0: axis along which to compute percentiles
newaxis, optional: name of the new percentile axis, if more than one pct.
By default, append "_percentile" to the axis name on which the transformation
is applied.
out, overwrite_input: passed to numpy's percentile method (see documentation)
outputs:
--------
pctiles: DimArray or scalar whose required axis has been reduced or replaced by percentiles
Examples:
---------
>>> np.random.seed(0) # for reproductibility of results
>>> a = DimArray(np.random.randn(1000), dims=['sample'])
>>> percentile(a, 50)
-0.058028034799627745
>>> percentile(a, [50, 95])
dimarray: 2 non-null elements (0 null)
dimensions: 'sample_percentile'
0 / sample_percentile (2): 50 to 95
array([-0.05802803, 1.66012041])
"""
if not isinstance(a, da.DimArray):
raise TypeError("Expected DimArray instance got {} of type {}".format(
a, type(a)))
pos, nm = a._get_axis_info(axis)
results = np.percentile(a.values,
pct,
axis=pos,
out=out,
overwrite_input=overwrite_input)
# If the result is scalar (pct is scalar and ), just return it
if np.isscalar(results):
return results
# for scalar pct, results is a numpy array. Just reduce the axis.
subaxes = [ax for ax in a.axes if ax.name != nm]
if np.isscalar(pct):
results = da.DimArray(results, axes=subaxes)
# pct is array-like, recreate a Dimarray
else:
if newaxis is None:
newaxis = nm + '_percentile'
results = [da.DimArray(res, axes=subaxes)
for res in results] # list of DimArrays
results = da.from_arrays(results, keys=pct,
axis=newaxis) # join in a larger DimArray
return results
def reindex_vcms(vcm, vcm5, vcmc):
# vcmc can be None
mintime5, maxtime5 = vcm5['time_min'].values, vcm5['time_max'].values
nprof5 = mintime5.shape[0]
time333 = vcm['time']
nprof333 = time333.shape[0]
# first find 333m profiles indexes for a given 5km profile
n1, n2 = np.zeros(nprof5, 'int16'), np.zeros(nprof5, 'int16')
n = 0
for i in np.r_[0:nprof5]:
n1[i] = n
while n < nprof333 and time333[n] < maxtime5[i]:
n += 1
n2[i] = n
# print 'Range for profile5 %d = %d - %d' % (i, n1[i], n2[i])
# reindex all flags on the same 333m coordinates
time_axis = da.Axis(vcm['time'], 'tai_time')
alt_axis = da.Axis(vcm['altitude'], 'altitude')
reindexed = da.Dataset()
reindexed['lon'] = da.DimArray(vcm['lon'], [
time_axis,
])
reindexed['lat'] = da.DimArray(vcm['lat'], [
time_axis,
])
reindexed['cal333'] = da.DimArray(vcm['cal333'], [time_axis, alt_axis])
# remap CALIPSO flag
for vcm_name in 'cal05', 'cal20', 'cal80':
this_vcm = remap_profiles(vcm5[vcm_name].values, n1, n2, nprof333)
reindexed[vcm_name] = da.DimArray(this_vcm,
labels=reindexed['cal333'].labels,
dims=reindexed['cal333'].dims)
# remap cloudsat flag
if vcmc is None:
this_vcm = np.ones_like(vcm['cal333'], 'int8') * -1.
else:
vcmc.values = remove_ground_clutter(vcmc.values,
vcm5['elevation'].values,
vcm5['cal05'].altitude)
this_vcm = remap_profiles(vcmc.values, n1, n2, nprof333)
reindexed['csat'] = da.DimArray(this_vcm,
labels=reindexed['cal333'].labels,
dims=reindexed['cal333'].dims)
# Now we have
# cal333, cal05, cal20, cal80, csat in reindexed.
return reindexed
def create_nc_monthly_variable_1d(exp, varibale, fname, dim_name):
months = np.arange(len(exp))
slr_po = da.DimArray(exp.tolist(), axes=[months], dims=['months'])
month = da.DimArray(months.tolist(), axes=[months], dims=['months'])
# month_co=da.DimArray(mons_cons.tolist(),axes = [mons_cons],dims = ['months'])
# year_date=da.DimArray(years_mon.tolist(),axes = [mons_cons],dims = ['months'])
# dim_name = my_vari_name(variable)
dataset = da.Dataset({variable: slr_po})
# print(clu+fname+dim_name)
dataset.write_nc(clu_path + fname + dim_name)
def my_read_vcm_nc(f=f):
nc = netCDF4.Dataset(f, 'r')
time = nc.variables['tai_time'][:]
alt = nc.variables['altitude'][:]
lon = nc.variables['lon'][:]
lat = nc.variables['lat'][:]
cal333 = nc.variables['cal333'][:]
time_axis = da.Axis(time, 'time')
alt_axis = da.Axis(alt, 'alt')
lon = da.DimArray(lon, time_axis)
lat = da.DimArray(lon, time_axis)
cal333 = da.DimArray(cal333, [time_axis, alt_axis])
def test():
""" Test Dataset functionality
>>> data = test()
>>> data['test2'] = da.DimArray([0,3],('source',['greenland','antarctica'])) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
ValueError: axes values do not match, align data first.
Dataset: source(1)=greenland:greenland,
Got: source(2)=greenland:antarctica
>>> data['ts']
dimarray: 5 non-null elements (5 null)
dimensions: 'time'
0 / time (10): 1950 to 1959
array([ 0., 1., 2., 3., 4., nan, nan, nan, nan, nan])
>>> data.to_array(axis='items')
dimarray: 12250 non-null elements (1750 null)
dimensions: 'items', 'lon', 'lat', 'time', 'source'
0 / items (4): mymap to test
1 / lon (50): -180.0 to 180.0
2 / lat (7): -90.0 to 90.0
3 / time (10): 1950 to 1959
4 / source (1): greenland to greenland
array(...)
"""
import dimarray as da
axes = da.Axes.from_tuples(('time', [1, 2, 3]))
ds = da.Dataset()
a = da.DimArray([[0, 1], [2, 3]], dims=('time', 'items'))
ds['yo'] = a.reindex_like(axes)
np.random.seed(0)
mymap = da.DimArray.from_kw(np.random.randn(50, 7),
lon=np.linspace(-180, 180, 50),
lat=np.linspace(-90, 90, 7))
ts = da.DimArray(np.arange(5), ('time', np.arange(1950, 1955)))
ts2 = da.DimArray(np.arange(10), ('time', np.arange(1950, 1960)))
# Define a Dataset made of several variables
data = da.Dataset({'ts': ts, 'ts2': ts2, 'mymap': mymap})
#data = da.Dataset([ts, ts2, mymap], keys=['ts','ts2','mymap'])
assert np.all(
data['ts'].time == data['ts2'].time), "Dataset: pb data alignment"
data['test'] = da.DimArray([0], ('source', ['greenland'])) # new axis
#data
return data
def load_path(path, variables=None, method="after"):
"""Load variables along a path
Parameters
----------
path : list of [(x0,y0), (x1, y1), ...] coordinates
variables : list, optional
of variables to be loaded (by default, all in a dataset)
method : str, optional
method to sample the data, by default "after", which indicates
the grid point at or just after the match, as returned
by searchsorted
"""
if method == "after":
pass
elif method == "nearest":
raise NotImplementedError()
elif method == "linear":
raise NotImplementedError()
else:
raise ValueError("Invalid method: " + method)
xs, ys = zip(
*path) # [(x0, y0), (x1, ...)] into [[x0,x1..], [y0, y1..]]
xs = np.asarray(xs)
ys = np.asarray(ys)
l = np.min(xs)
r = np.max(xs)
b = np.min(ys)
t = np.max(ys)
data2d = load_map_func(variables=variables, bbox=[l, r, b, t])
# add a new coordinate s
diff_s = np.sqrt(np.square(np.diff(xs)) + np.square(np.diff(ys)))
s = np.concatenate(([0], np.cumsum(diff_s)))
datapath = da.Dataset()
# add x and y as variables in the dataset
datapath['x'] = da.DimArray(xs, axes=[s], dims=['s'])
datapath['x'].long_name = "x-coordinate along sample path"
datapath['y'] = da.DimArray(ys, axes=[s], dims=['s'])
datapath['y'].long_name = "y-coordinate along sample path"
# now extract dataset...
i = np.searchsorted(data2d.y, ys)
j = np.searchsorted(data2d.x, xs)
i[i == data2d.y.size] -= 1 # out-of-bound
j[j == data2d.x.size] -= 1
for v in data2d.keys():
pathvals = data2d[v].values[i, j]
datapath[v] = da.DimArray(pathvals, axes=[s], dims=['s'])
datapath[v].attrs.update(data2d[v].attrs)
return datapath
def cf_from_vcm_orbit(vcm_orbit, layers):
print 'Creating vcm from ', vcm_orbit
# read data
v = vcm.VCM(vcm_orbit, verbose=False)
print '%d profiles' % (v.lon.shape[0])
lon_axis = da.Axis(lonbins[:-1], 'lon')
lat_axis = da.Axis(latbins[:-1], 'lat')
# gridded number of profiles
h, xx, yy = np.histogram2d(v.lon, v.lat, bins=(lonbins, latbins))
out = da.Dataset()
out['nprof'] = da.DimArray(h * 3, [lon_axis, lat_axis])
out['nprof'].longname = 'Number of measured profiles'
for layer in layers:
altrange = layers[layer]
altidx = np.where((v.altitude >= altrange[0])
& (v.altitude < altrange[1]))[0]
for vcm_name in vcm_names:
# number of cloudy profiles in grid and altitude range
this_vcm = v.get_vcm(vcm_name)
assert this_vcm is not None
if layer == 'total':
t = this_vcm
else:
t = np.take(this_vcm, altidx, axis=1)
cloudy_profile = np.sum(t, axis=1)
np.clip(cloudy_profile, 0, 3, out=cloudy_profile)
h, xx, yy = np.histogram2d(v.lon,
v.lat,
bins=(lonbins, latbins),
weights=cloudy_profile)
outname = vcm_name + '_cprof_%s' % (layer)
out[outname] = da.DimArray(h, [lon_axis, lat_axis])
out[outname].longname = 'Number of cloudy profiles from cloud mask = ' + vcm_name + ' at altitudes %5.2f - %5.2f' % (
altrange[0], altrange[1])
return out
def test_operations():
a = da.DimArray([[1, 2, 3], [3, 4, 5]], dims=('x0', 'x1'))
assert np.all(a == a)
assert np.all(a + 2 == a + np.ones(a.shape) * 2)
assert np.all(a + a == a * 2)
assert np.all(a * a == a**2)
assert np.all((a - a.values) == a - a)
def save_track(self, tk, t_i, i_f, i_b, y_ini, x_ini, plot):
pos_ar = np.array([tk['time'], tk['lat'], tk['lon']]).T
#if True:
if i_f + i_b > 1 and len(
unique(self.all_pos.tolist() +
pos_ar.tolist())) - self.all_pos.shape[0] > 3:
self.tcs[str(self.id_)] = da.DimArray(
pos_ar,
axes=[tk['time'], ['time', 'lat', 'lon']],
dims=['time', 'pos'])
self.all_pos = np.concatenate((self.all_pos, pos_ar))
if plot: self.plot_track(tk, t_i, i_f, i_b, y_ini, x_ini)
print '--------', t_i
print np.max(self.wind[self.time[t_i], y_ini + 3:y_ini - 3,
x_ini - 3:x_ini + 3])
print np.min(self.mslp[self.time[t_i], y_ini + 3:y_ini - 3,
x_ini - 3:x_ini + 3])
y, x = local_extrem(self.mslp[self.time[t_i], :, :],
'min',
threshold=101000)
print y_ini, self.lat[y]
print x_ini, self.lon[x]
self.id_ += 1
else:
pass
def compound_precip_temp_index(combinations, out_file):
"""
Not documented yet
"""
out = {}
for name, conditions in combinations.items():
conds = []
description = []
for condition in conditions:
nc = da.read_nc(condition[0])
conds.append(nc[condition[1]].squeeze())
description.append(nc[condition[1]].description)
compound_state = conds[0].copy()
compound_state[:] = False
for cond in conds:
compound_state += cond
compound_state /= len(conds)
out[name] = da.DimArray(np.array(compound_state.values, dtype=np.byte),
axes=compound_state.axes,
dims=compound_state.dims,
dtype=np.byte)
out[name].description = ' AND '.join(description)
da.Dataset(out).write_nc(out_file)
def gather_info_track(self, overwrite=False):
out_file = self._working_dir + 'surrounding_info.nc'
if overwrite and os.path.isfile(out_file):
os.system('rm ' + out_file)
elif overwrite == False and os.path.isfile(out_file):
self._track_info = da.read_nc(out_file)
return self._track_info
track_info = {}
for id_, track in self._tcs.items():
track = track[np.isfinite(track[:, 't']), :]
info = np.zeros([
6, track.shape[0], self._win2 * 2 + 1, self._win2 * 2 + 1
]) * np.nan
for i, p in enumerate(track.values.tolist()):
box_2 = [
int(bb) for bb in self.get_box(p[1], p[2], self._win2)
]
info[0, i,
abs(p[1] - box_2[0] - 12):box_2[1] - p[1] + 12,
abs(p[2] - box_2[2] - 12):box_2[3] - p[2] +
12] = self._VO[int(p[0]), box_2[0]:box_2[1],
box_2[2]:box_2[3]]
info[1, i,
abs(p[1] - box_2[0] - 12):box_2[1] - p[1] + 12,
abs(p[2] - box_2[2] - 12):box_2[3] - p[2] +
12] = self._MSLP[int(p[0]), box_2[0]:box_2[1],
box_2[2]:box_2[3]]
info[2, i,
abs(p[1] - box_2[0] - 12):box_2[1] - p[1] + 12,
abs(p[2] - box_2[2] - 12):box_2[3] - p[2] +
12] = self._Wind10[int(p[0]), box_2[0]:box_2[1],
box_2[2]:box_2[3]]
if self._SST is not None:
info[3, i,
abs(p[1] - box_2[0] - 12):box_2[1] - p[1] + 12,
abs(p[2] - box_2[2] - 12):box_2[3] - p[2] +
12] = self._SST[int(p[0]), box_2[0]:box_2[1],
box_2[2]:box_2[3]]
if self._T is not None:
info[4, i,
abs(p[1] - box_2[0] - 12):box_2[1] - p[1] + 12,
abs(p[2] - box_2[2] - 12):box_2[3] - p[2] +
12] = self._T[int(p[0]), 0, box_2[0]:box_2[1],
box_2[2]:box_2[3]]
info[5, i,
abs(p[1] - box_2[0] - 12):box_2[1] - p[1] + 12,
abs(p[2] - box_2[2] - 12):box_2[3] - p[2] +
12] = self._T[int(p[0]), 1, box_2[0]:box_2[1],
box_2[2]:box_2[3]]
track_info[str(id_)] = da.DimArray(
info,
axes=[['VO', 'MSLP', 'Wind10', 'SST', 'T850', 'T500'],
range(len(track.time)),
range(self._win2 * 2 + 1),
range(self._win2 * 2 + 1)],
dims=['time_id', 'variable', 'lat', 'lon'])
self._track_info = da.Dataset(track_info)
self._track_info.write_nc(out_file, mode='w')
def precip_to_index(
in_file,
out_file,
var_name='pr',
unit_multiplier=1,
states={
'dry': {
'mod': 'below',
'threshold': 1
},
'wet': {
'mod': 'above',
'threshold': 1
},
'5mm': {
'mod': 'above',
'threshold': 5
},
'10mm': {
'mod': 'above',
'threshold': 10
}
}):
"""
Classifies daily precipitation into 'wet' and 'dry' days based on a `threshold`
Parameters
----------
anom_file: str
filepath of a daily precipitation file. The variable that is read in can be specified with `var_name`.
out_file: str
filepath of a state file
var_name: str
name of the variable read in `anom_file`
threshold: float,default=0.5
threshold used to differentiate between wet and dry days
unit_multiplier: float,default=1
factor to multiply daily precipiation with to get mm as units
overwrite: bool
overwrites existing files
"""
nc = da.read_nc(in_file)
pr = nc[var_name].squeeze() * unit_multiplier
out = {}
for name, state_dict in states.items():
state = nc[var_name].squeeze().copy()
state[:] = False
if state_dict['mod'] == 'above':
state[pr >= state_dict['threshold']] = True
if state_dict['mod'] == 'below':
state[pr <= state_dict['threshold']] = True
out[name] = da.DimArray(np.array(state.values, dtype=np.byte),
axes=state.axes,
dims=state.dims,
dtype=np.byte)
out[name].description = 'days with precipitation ' + state_dict[
'mod'] + ' ' + str(state_dict['threshold']) + 'mm'
da.Dataset(out).write_nc(out_file)
def calc_residuals(param_to_calibrate):
""" Only residuals in calibration period play a role for fit.
Residuals start always with zero in the first year of cal_period. """
proj_period = self.gmt.time
contrib = da.DimArray(cl.calc_contribution(
self.gmt, proj_period, param_to_calibrate),
axes=proj_period,
dims="time")
residuals = da.DimArray(np.zeros_like(contrib.values),
axes=proj_period,
dims="time")
residuals[self.cal_period] = self.sl_observation[
self.cal_period] - contrib[self.cal_period]
residuals[self.cal_period] -= residuals[self.cal_period[0]]
return residuals.values
def my_read_nc_lon(f=f):
nc = netCDF4.Dataset(f, 'r')
time = nc.variables['tai_time'][:]
lon = nc.variables['lon'][:]
nc.close()
axis = da.Axis(time, 'time')
d1 = da.DimArray(lon, axis)
return d1
def combine_tracks(self, plot=True):
detected = self.detect[self.detect[:, 'tc_cond'] == 4].values
mslp_found = self.detect[((self.detect[:, 'mslp'] == 1) &
(self.detect[:, 'tc_cond'] != 4))].values
candidates = detected[:, :].tolist()
weak_candidates = mslp_found[:, :].tolist()
used_pos = []
for p in candidates:
if p[7] and p[8]:
if p not in used_pos:
used_pos.append(p)
track = [p]
candidates.remove(p)
for p in track:
tmp = np.array(candidates)
loc_candidates = tmp[(
(tmp[:, 0] <= p[0] + 1) &
(tmp[:, 0] >= p[0] - 1)), :].tolist()
for nei in self.get_surrounding(
int(p[1]), int(p[2]), 4):
for candi in loc_candidates:
if tuple(candi[1:3]) == nei:
track.append(candi)
candidates.remove(candi)
if len(candidates) == 0:
break
for p in track:
tmp = np.array(weak_candidates)
loc_candidates = tmp[(
(tmp[:, 0] <= p[0] + 1) &
(tmp[:, 0] >= p[0] - 1)), :].tolist()
for nei in self.get_surrounding(
int(p[1]), int(p[2]), 4):
for candi in loc_candidates:
if tuple(candi[1:3]) == nei:
track.append(candi)
weak_candidates.remove(candi)
if len(weak_candidates) == 0:
break
track = self.rearange_track(np.array(track))
track = da.DimArray(track,
axes=[
track[:, 0],
[
't', 'y', 'x', 'vort', 'mslp',
'wind', 'ta', 'sst',
'tropical', 'tc_cond'
]
],
dims=['time', 'z'])
if track[track[:, 'tc_cond'] ==
4].shape[0] > 6 or track.shape[0] > 10:
self.tcs[self.id_] = track
if plot: self.plot_track(track)
self.id_ += 1
def calc_residuals(self, param, sl_function):
""" Only residuals in calibration period play a role for fit.
Residuals start always with zero in the first year of cal_period. """
contrib = da.DimArray(sl_function(self.gmt, param),
axes=self.gmt.time,
dims="time")
residuals = da.DimArray(np.zeros_like(contrib.values),
axes=self.gmt.time,
dims="time")
residuals[self.cal_period] = self.sl_observation[
self.cal_period] - contrib[self.cal_period]
residuals[self.cal_period] -= residuals[self.cal_period[0]]
# alternative, but untested, residuals relative to their mean, may reduce
# dependency on the very first value.
# residuals[self.cal_period] -= residuals[self.cal_period].mean()
return residuals.values
def save_track(self,tk,t_i,i_f,i_b,y_ini,x_ini):
pos_ar=np.array([tk['time'],tk['lat'],tk['lon']]).T
#if True:
if i_f+i_b>4 and len(unique(self.all_pos.tolist()+pos_ar.tolist()))-self.all_pos.shape[0]>3:
self.tcs[str(self.id_)]=da.DimArray(pos_ar,axes=[tk['time'],['time','lat','lon']],dims=['time','pos'])
self.all_pos=np.concatenate((self.all_pos,pos_ar))
self.plot_track(tk,t_i,i_f,i_b,y_ini,x_ini)
self.id_+=1
else:
pass
def cflon(f, altmin, latbounds):
# altmin is an array
v = vcm.VCM(f, verbose=False)
out = da.Dataset()
lon_axis = da.Axis(lonbins[:-1], 'lon')
# number of profiles per lon bin
h, xx = np.histogram(v.lon, bins=lonbins)
out['nprof'] = da.DimArray(h, [lon_axis])
out['nprof'].longname = 'Number of measured profiles'
for n in names:
cv = v.get_vcm(n)
assert cv is not None
# clip latitudes
latidx = (v.lat >= latbounds[0]) & (v.lat < latbounds[1])
cv = np.take(cv, latidx, axis=0)
lon = np.take(v.lon, latidx, axis=0)
outdict = dict()
for a in altmin:
idx = np.where(v.altitude >= a)[0]
cloudy = np.take(cv, idx, axis=1)
cloudy = np.sum(cloudy, axis=1)
np.clip(cloudy, 0, 1, out=cloudy)
h, xx = np.histogram(v.lon, bins=lonbins, weights=cloudy)
outdict[a] = da.DimArray(h, [
lon_axis,
])
outname = n + '_cprof'
out[outname] = da.stack(outdict, axis='altmin')
out[outname].longname = 'Number of cloudy profiles from cloud mask = ' + n
return out
def detect_dieng(self,
overwrite=False,
dis_VO_max=8,
min_number_cells=6,
thr_VO=1 * 10**(-5),
thr_RH=50):
out_file = self._working_dir + str(
self._identifier) + '_detected_positions.nc'
if overwrite and os.path.isfile(out_file):
os.system('rm ' + out_file)
elif overwrite == False and os.path.isfile(out_file):
self._detected = da.read_nc(out_file)['detected']
return self._detected
# convert distances from degrees into grid-cells
dis_VO_max = self.degree_to_step(dis_VO_max)
detect = np.array([[np.nan] * 6])
print('detecting\n10------50-------100')
for t, progress in zip(
self._time_i,
np.array([['-'] + [''] * (len(self._time_i) / 20 + 1)] *
20).flatten()[0:len(self._time_i)]):
sys.stdout.write(progress)
sys.stdout.flush()
coords = peak_local_max(self._VO[t, :, :],
min_distance=int(dis_VO_max))
#print(coords)
for y_, x_ in zip(coords[:, 0], coords[:, 1]):
if self._VO[t, y_, x_] > thr_VO:
yy, xx = self.find_group(field=self._VO[t, :, :],
y=y_,
x=x_,
thresh=thr_VO)
if len(yy) >= min_number_cells:
if self._RH[t, y_,
x_] >= thr_RH and self._lat[y_] < 35:
#y_,x_ = sum(yy) / len(yy), sum(xx) / len(yy)
tmp = [
t, y_, x_, self._VO[t, y_, x_], self._RH[t, y_,
x_],
len(yy)
]
detect = np.concatenate((detect, np.array([tmp])))
self._detected = da.DimArray(
np.array(detect[1:, :]),
axes=[
range(detect.shape[0] - 1),
['t', 'y', 'x', 'VO', 'RH', 'members']
],
dims=['ID', 'z'])
da.Dataset({'detected': self._detected}).write_nc(out_file, mode='w')
print('\ndone')
return self._detected
def save_track(self,tk,t_i,i_f,i_b,y_ini,x_ini,plot):
pos_ar=np.array([tk['time'],tk['lat'],tk['lon']]).T
#if i_f+i_b>1 and len(unique(self.all_pos.tolist()+pos_ar.tolist()))-self.all_pos.shape[0]>4:
if i_f+i_b>2:
if sum([bb in self.all_pos.tolist() for bb in pos_ar.tolist()])<9999:
self.tcs[str(self.id_)]=da.DimArray(pos_ar,axes=[tk['time'],['time','lat','lon']],dims=['time','pos'])
self.all_pos=np.concatenate((self.all_pos,pos_ar))
if plot: self.plot_track(tk,t_i,i_f,i_b,y_ini,x_ini)
self.detection_t_i.append(t_i)
self.id_+=1
def vcm_from_geoprof_file(geoprof_file, vcm_alt):
geoprof_vcm, geoprof_alt, geoprof_time = _geoprof_vcm_from_geoprof_file(
geoprof_file)
if geoprof_vcm is None:
return None
vcm = _geoprof_vcm_on_altitudes(geoprof_vcm, geoprof_alt, vcm_alt)
vcm_da = da.DimArray(vcm,
labels=(geoprof_time, vcm_alt),
dims=['tai_time', 'altitude'])
return vcm_da
def save_track(self, track, t_i, plot=True):
if track.shape[0] > 1:
if self.sst[track[0, 0],
self.lat[np.argmin(abs(self.lat - track[0, 1]))],
self.lon[np.argmin(abs(self.lon -
track[0, 2]))]] > self.thr_sst:
self.tcs[str(self.id_)] = da.DimArray(
track,
axes=[track[:, 0], ['time', 'lat', 'lon', 't_i']],
dims=['time', 'pos'])
if plot: self.plot_track(track, t_i)
self.id_ += 1
def vcm_dataset_from_l2_orbit(filename):
#print 'Creating vcm dataset from l2 file ' + filename
l2 = calipso_l2.Cal2(filename)
lon, lat = l2.coords()
tai_time = l2.time()
nl, base, top = l2.layers()
havg = l2.horizontal_averaging()
ltype = l2.layer_type()
tai_time_min, tai_time_max = l2.time_bounds()
tropo = l2.tropopause_height()
elevation = l2.dem_surface_elevation()[:,1]
l2.close()
tropo[lat < -60] = 11.
tropo[lat > 60] = 11.
dset = da.Dataset()
time_axis = da.Axis(tai_time, 'tai_time')
alt_axis = da.Axis(vcm_alt, 'altitude')
for havg_vcm in havgs_vcm:
vcm = vcm_from_layers(nl, base, top, havg, ltype, tropo, only_havg=havg_vcm)
vcm_name = 'cal%02d' % (havg_vcm)
dset[vcm_name] = da.DimArray(vcm, [time_axis, alt_axis])
dset['lon'] = da.DimArray(lon, [time_axis])
dset['lat'] = da.DimArray(lat, [time_axis])
dset['time_min'] = da.DimArray(tai_time_min, [time_axis])
dset['time_max'] = da.DimArray(tai_time_max, [time_axis])
dset['elevation'] = da.DimArray(elevation, [time_axis])
return dset
def test_diff():
np.random.seed(0)
v = da.DimArray(np.random.randn(5, 7), {
'time': np.arange(1950, 1955),
'lat': np.linspace(-90, 90, 7)
})
v.diff(axis='time', keepaxis=False)
v.diff(axis=0, keepaxis=False, scheme='centered')
v.diff(axis=0, keepaxis=False, scheme='backward')
v.diff(axis=0, keepaxis=False, scheme='forward')
v.diff(axis=0, keepaxis=True, scheme='backward')
v.diff(axis=0, keepaxis=True, scheme='forward')
v.diff(n=2, axis=('time'), scheme='centered')
def zone_vcm_from_vcm_orbit(vcm_orbit, latbins=latbins):
# read data
#print 'opening ' + vcm_orbit
v = vcm.VCM(vcm_orbit, verbose=False)
nlat = latbins.shape[0]
nalt = v.altitude.shape[0]
out = da.Dataset()
# ilatbins = vector with nprof indexes containing bin numbers
ilatbins = np.digitize(v.lat, latbins)
lat_axes = da.Axis(latbins[:-1], 'lat')
nprof, xx = np.histogram(v.lat, bins=latbins)
out['nprof'] = da.DimArray(nprof * 3, [lat_axes])
alt_axes = da.Axis(v.altitude, 'altitude')
for name in vcm_names:
this_vcm = v.get_vcm(name)
zone_vcm = np.zeros([nlat-1, nalt], dtype='uint16')
prof_is_cloudy = np.sum(this_vcm, axis=1)
np.clip(prof_is_cloudy, 0, 3, out=prof_is_cloudy)
cprof, xx = np.histogram(v.lat, bins=latbins, weights=prof_is_cloudy)
out[name + '_cprof'] = da.DimArray(cprof, [lat_axes])
for i,ilatbin in enumerate(ilatbins[:-1]):
if prof_is_cloudy[i] > 0:
zone_vcm[ilatbin,:] += np.take(this_vcm, i, axis=0)
out[name] = da.DimArray(zone_vcm, [lat_axes, alt_axes], longname='Number of cloudy points in lat-z bin, considering ' + name)
return out
def convert_raw_mb_to_sl(rawdata):
fn = scipy.interpolate.interp1d(rawdata[:, 0], rawdata[:, 1])
# in Gt/y
mb = da.DimArray(fn(np.arange(1871, 2009, 1)),
axes=np.arange(1871, 2009, 1),
dims="time")
# assume "equilibrium" in years 1961-1990 or 1971-1988, see discussion in Hanna et al 2011
# and in Rignot et al. 2008
# doi:10.1029/2011JD016387
# assume "equilibrium" before 1900, see the zero before 1900 in Fig. 7,
# Box and Colgan 2013
mb -= mb[1870:1900].mean()
# convert to m SLE
return mb.cumsum() / 360. / 1.e3
def project_slr(scen, gmt, settings):
projection_data = {}
temp_anomaly_years = pd.read_csv(os.path.join(settings.calibfolder,
"temp_anomaly_years.csv"),
index_col=[0, 1])
temp_anomaly_years = temp_anomaly_years.where(
pd.notnull(temp_anomaly_years), None)
for i, contrib_name in enumerate(settings.project_these):
print "conribution", contrib_name
realizations = np.arange(settings.nrealizations)
calibdata = pd.read_csv(os.path.join(settings.calibfolder,
contrib_name + ".csv"),
index_col=[0])
temp_anomaly_year = temp_anomaly_years.loc[contrib_name]
sl_contributor = cf.contributor_functions[contrib_name]
proj = np.zeros([len(settings.proj_period), settings.nrealizations])
for n in realizations:
slr, gmt_n, obs_choice, params = project(gmt, settings.proj_period,
calibdata,
temp_anomaly_year,
sl_contributor, n,
contrib_name)
proj[:, n] = slr
pdata = da.DimArray(proj,
axes=[settings.proj_period, realizations],
dims=["time", "runnumber"])
projection_data[contrib_name] = pdata
if not os.path.exists(settings.projected_slr_folder):
os.makedirs(settings.projected_slr_folder)
fname = "projected_slr_" + scen + "_n" + str(
settings.nrealizations) + ".nc"
da.Dataset(projection_data).write_nc(
os.path.join(settings.projected_slr_folder, fname))
print "Sea level projection data written to"
print settings.projected_slr_folder
def detect(self):
detect = np.array([[np.nan] * 10])
for t in self.time_i:
y_p, x_p = self.local_min(self.mslp.values[t, :, :],
threshold=self.thr_mslp,
neighborhood_size=self.neighborhood_size)
p_mins = [(y, x) for y, x in zip(y_p, x_p)]
# i vort max
y_v, x_v = self.local_max(self.vort.values[t, 0, :, :],
threshold=self.thr_vort,
neighborhood_size=self.neighborhood_size)
for y, x in zip(y_v, x_v):
box_1 = self.get_box(y, x, self.win1)
box_2 = self.get_box(y, x, self.win2)
points_1 = self.points_in_box(box_1)
tmp = [t, y, x, 1, 0, 0, 0, 0, 0, 0]
# ii pressure min
if sum([(yy, xx) in p_mins for yy, xx in points_1]):
tmp[4] = 1
# iii wind speed
if self.wind.ix[t, box_2[0]:box_2[1],
box_2[2]:box_2[3]].max() > self.thr_wind:
tmp[5] = 1
# iv warm core
if self.ta.ix[t, box_1[0]:box_1[1], box_1[2]:box_1[3]].max(
) - self.ta.ix[t, box_2[0]:box_2[1],
box_2[2]:box_2[3]].mean() > self.thr_ta:
tmp[6] = 1
# v warm sea
if self.sst.ix[t, y, x] > self.thr_sst:
tmp[7] = 1
# vi tropical
if self.lat[y] <= 30:
tmp[8] = 1
tmp[9] = sum(tmp[3:7])
detect = np.concatenate((detect, np.array([tmp])))
self.detect = da.DimArray(detect[1:, :],
axes=[
range(detect.shape[0] - 1),
[
't', 'y', 'x', 'vort', 'mslp',
'wind', 'ta', 'sst', 'tropical',
'tc_cond'
]
],
dims=['ID', 'z'])
def detect(self):
detect=np.array([[np.nan]*10])
for t in self.time_i:
# get mslp min
y_p,x_p = self.local_min(self.mslp.values[t,:,:],threshold=self.thr_mslp,neighborhood_size=self.neighborhood_size)
p_mins=[(y,x) for y,x in zip(y_p,x_p)]
# get vort clusters
clustered=self.max_clusters(self.vort.values[t,0,:,:],self.thr_vort,7)
for group in clustered.values():
if len(group['x'])>4:
axes[0].plot(group['x'],group['y'],marker='.')
lr=stats.linregress(group['x'],group['y'])
xxx=np.array(list(set(group['x'])))
axes[0].plot(xxx,xxx*lr.slope+lr.intercept)
for step in range(4):
x=int(round(min(xxx)+len(xxx)/4.*step))
y=int(round(x*lr.slope+lr.intercept))
if 0<x<len(self.lon) and 0<y<len(self.lat):
box_1=self.get_box(y,x,self.win1)
box_2=self.get_box(y,x,self.win2)
points_1=self.points_in_box(box_1)
tmp=[t,y,x,1,0,0,0,0,0,0]
# ii pressure min
if sum([(yy,xx) in p_mins for yy,xx in points_1]):
tmp[4]=1
# iii wind speed
if self.wind.ix[t,box_2[0]:box_2[1],box_2[2]:box_2[3]].max()>self.thr_wind:
tmp[5]=1
# iv warm core
if self.ta.ix[t,box_1[0]:box_1[1],box_1[2]:box_1[3]].max()-self.ta.ix[t,box_2[0]:box_2[1],box_2[2]:box_2[3]].mean()>self.thr_ta:
tmp[6]=1
# v warm sea
if self.sst.ix[t,y,x]>self.thr_sst:
tmp[7]=1
# vi tropical
if self.lat[y]<=30:
tmp[8]=1
tmp[9]=sum(tmp[3:7])
detect=np.concatenate((detect,np.array([tmp])))
self.detect=da.DimArray(detect[1:,:],axes=[range(detect.shape[0]-1),['t','y','x','vort','mslp','wind','ta','sst','tropical','tc_cond']],dims=['ID','z'])
