def test_min():
out = xr.DataArray(df.i64.values.reshape((2, 2, 5)).min(axis=2).astype('f8').T,
coords=coords, dims=dims)
assert_eq(c.points(df, 'x', 'y', ds.min('i32')), out)
assert_eq(c.points(df, 'x', 'y', ds.min('i64')), out)
assert_eq(c.points(df, 'x', 'y', ds.min('f32')), out)
assert_eq(c.points(df, 'x', 'y', ds.min('f64')), out)
def test_min():
out = xr.DataArray(df.i64.values.reshape((2, 2, 5)).min(axis=2).astype('f8').T,
coords=coords, dims=dims)
assert_eq(c.points(ddf, 'x', 'y', ds.min('i32')), out)
assert_eq(c.points(ddf, 'x', 'y', ds.min('i64')), out)
assert_eq(c.points(ddf, 'x', 'y', ds.min('f32')), out)
assert_eq(c.points(ddf, 'x', 'y', ds.min('f64')), out)
def tests_datashader():
import datashader as ds
import datashader.transfer_functions as tf
import pandas as pd
df = pd.read_csv('/Users/iregon/C3S/dessaps/test_data/imma_converter/observations-sst-2014-6.psv',usecols=[6,7,14],sep="|",skiprows=0)
agg_mean = cvs.points(df, 'longitude', 'latitude', ds.mean('observation_value'))
agg_max = cvs.points(df, 'longitude', 'latitude', ds.max('observation_value'))
agg_min = cvs.points(df, 'longitude', 'latitude', ds.min('observation_value'))
agg_count = cvs.points(df, 'longitude', 'latitude', ds.count('observation_value'))
def make_image(data, time_range, y_range, size, scale=None, width=0):
"Flatten the given range of the data into a 2d image"
time_range = (time_range[0].timestamp() * 1e6,
time_range[1].timestamp() * 1e6)
cvs = datashader.Canvas(x_range=time_range,
y_range=y_range,
plot_width=size[0],
plot_height=size[1],
y_axis_type=scale or "linear")
# aggregate some useful measures
agg_line = cvs.line(source=data["data"], x="t", y="value_r")
agg_points = cvs.points(source=data["data"],
x="t",
y="value_r",
agg=datashader.summary(
count=datashader.count("value_r"),
vmean=datashader.mean("value_r"),
vmin=datashader.min("value_r"),
vmax=datashader.max("value_r")))
color = data["info"].get("color", "red")
image = datashader.transfer_functions.shade(agg_line, cmap=[color])
if width > 0:
image = datashader.transfer_functions.spread(image, px=width)
# image = datashader.transfer_functions.spread(
# image, mask=np.matrix([[False, False, False],
# [False, True, True],
# [False, True, True]]))
with timer("Making hover info"):
indices = np.where(np.nanmax(agg_points["count"].values, axis=0))[0]
vmin = np.take(np.nanmin(agg_points["vmin"].values, axis=0), indices)
vmax = np.take(np.nanmax(agg_points["vmax"].values, axis=0), indices)
# vmean = np.take(np.nanmax(agg_points["vmean"].values, axis=0), indices)
# TODO: aggregating the mean is not quite this simple...
timestamps = np.take(agg_points["x_axis"].values, indices)
count = np.take(np.sum(agg_points["count"].values, axis=0), indices)
desc = {
"total_points": data["points"],
"indices": indices.tolist(),
"min": np.where(np.isnan(vmin), None, vmin).tolist(),
"max": np.where(np.isnan(vmax), None, vmax).tolist(),
"timestamp": [float(t) for t in timestamps],
# "mean": np.where(np.isnan(vmean), None, vmean).tolist(),
"count": np.where(np.isnan(count), None, count).tolist()
}
return image, desc
def gen_water_spec(self):
opts_spec = dict(width=self.plot_width,
height=self.plot_height,
xticks=10,
yticks=10,
xrotation=10,
fontsize=font_dict) # colorbar=False,
heat_ds = datashade(self.points,
aggregator=ds.min('PowerdB'),
dynamic=False,
cmap=Viridis256,
clims=self.zrange,
min_alpha=min_alpha,
normalization='linear',
x_sampling=self.step_size,
y_sampling=1).options(**opts_spec)
return heat_ds
def draw_figure(df, fileout):
geodetic = ccrs.Geodetic(globe=ccrs.Globe(datum='WGS84'))
fig=plt.figure(frameon=False)
tiler = StamenTerrain()
ax = plt.axes(projection=tiler.crs)
ax.add_feature(cartopy.feature.OCEAN,zorder=1)
ax.add_feature(cartopy.feature.COASTLINE,edgecolor='green',linewidth=0.5,zorder=4)
ax.add_feature(cartopy.feature.BORDERS,edgecolor='green',linewidth=0.5,zorder=4)
tra = tiler.crs.transform_points(geodetic,df.lon.values,df.lat.values)
x = tra[:,0]
y = tra[:,1]
tra = pd.DataFrame({'lon':x,'lat':y,'baroaltitude':df.baroaltitude.values})
target = 6000
ratio = target/(np.max(tra.lon)-np.min(tra.lon))
cvs = ds.Canvas(plot_width=target, plot_height=int((np.max(tra.lat)-np.min(tra.lat))*ratio))
agg = cvs.points(tra, 'lon', 'lat',ds.min('baroaltitude'))#, ds.mean('baroaltitude'))
img = tf.shade(agg, cmap=inferno,how='linear')
img = tf.set_background(img, 'black')
r = img.to_pil()
datas = r.getdata()
newData = []
for item in datas:
if item[0] == 0 and item[1] == 0 and item[2] == 0:
newData.append((255, 255, 255, 0))
else:
newData.append(item)
r.putdata(newData)
cax = plt.imshow(r,zorder=3,origin='upper',interpolation='gaussian',extent=(np.min(tra.lon),np.max(tra.lon),np.min(tra.lat),np.max(tra.lat)))
ax1 = fig.add_axes([0.05, 0.18, 0.9, 0.025])
norm = mpl.colors.Normalize(vmin=np.min(df.baroaltitude.values)/FEET2METER, vmax=np.max(df.baroaltitude.values)/FEET2METER)
cb1 = mpl.colorbar.ColorbarBase(ax1, cmap=infernompl,norm=norm,orientation='horizontal')
cb1.set_label('$H_p$ [ft]')
size = fig.get_size_inches()
h_over_w = size[1]/size[0]
fig.set_tight_layout({'pad':0})
fig.set_figwidth(TEXT_WIDTH)
fig.set_figheight(TEXT_WIDTH*h_over_w)
plt.savefig(fileout,format="pdf",pad_inches=0,dpi=2000, bbox_inches='tight')
def tests_datashader():
import datashader as ds
import datashader.transfer_functions as tf
import pandas as pd
df = pd.read_csv(
'/Users/iregon/C3S/dessaps/test_data/imma_converter/observations-sst-2014-6.psv',
usecols=[6, 7, 14],
sep="|",
skiprows=0)
agg_mean = cvs.points(df, 'longitude', 'latitude',
ds.mean('observation_value'))
agg_max = cvs.points(df, 'longitude', 'latitude',
ds.max('observation_value'))
agg_min = cvs.points(df, 'longitude', 'latitude',
ds.min('observation_value'))
agg_count = cvs.points(df, 'longitude', 'latitude',
ds.count('observation_value'))
for vari in vars_in:
descriptors[descriptor][vari] = dict()
i = 1
for yr in range(y_ini,y_end + 1):
for mo in range(1,13):
logging.info('{0}-{1}'.format(yr,mo))
mm = '{:02d}'.format(mo)
# Read monthly data (header and observed vars) to df indexed by report_id
df_mo = read_monthly_data()
# For each observed variable in df, compute stats and aggregate to its monhtly composite.
for vari in vars_in:
if mm in descriptors['counts'][vari].keys():
descriptors['counts'][vari][mm] = descriptors['counts'][vari][mm] + cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.count(vari))
descriptors['max'][vari][mm] = np.fmax(descriptors['max'][vari][mm],cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.max(vari)))
descriptors['min'][vari][mm] = np.fmin(descriptors['min'][vari][mm],cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.min(vari)))
# All this mess, because addition propagates nan's in xarrays!
mean_mm = cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.mean(vari))
max_frame = np.fmax(descriptors['ave'][vari][mm],mean_mm)
min_frame = np.fmin(descriptors['ave'][vari][mm],mean_mm)
descriptors['ave'][vari][mm] = 0.5*max_frame + 0.5*min_frame
else:
descriptors['counts'][vari][mm] = cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.count(vari)) # <class 'xarray.core.dataarray.DataArray'>
descriptors['max'][vari][mm] = cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.max(vari))
descriptors['min'][vari][mm] = cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.min(vari))
mean_mm = cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.mean(vari))
descriptors['ave'][vari][mm] = mean_mm
# Also add monthly stats to the global aggregate
if 'global' in descriptors['counts'][vari].keys():
descriptors['counts'][vari]['global'] = descriptors['counts'][vari]['global'] + cvs.points(df_mo[['latitude','longitude',vari]], 'longitude', 'latitude', ds.count(vari))
def test_min():
out = df.i32.reshape((2, 2, 5)).min(axis=2).T
eq(c.points(ddf, 'x', 'y', agg=ds.min('i32')).agg, out)
eq(c.points(ddf, 'x', 'y', agg=ds.min('i64')).agg, out.astype('i8'))
eq(c.points(ddf, 'x', 'y', agg=ds.min('f32')).agg, out.astype('f4'))
eq(c.points(ddf, 'x', 'y', agg=ds.min('f64')).agg, out.astype('f8'))
aggregator=agg_data,
width=50,
height=25,
streams=[RangeXY]),
operation=hv.QuadMesh)
hover = hover.options(cmap=cmap)
img = geomap * zip_codes * hover
img = img.relabel(label)
return img
# In[ ]:
plot_map(brazil,
'Zip Codes in Brazil',
ds.min(agg_name),
agg_name,
cmap=rainbow)
# In[ ]:
# plot wtih datashader - image with black background
import datashader as ds
from datashader import transfer_functions as tf
from functools import partial
from datashader.utils import export_image
from IPython.core.display import HTML, display
from colorcet import fire, rainbow, bgy, bjy, bkr, kb, kr
background = "black"
cm = partial(colormap_select, reverse=(background != "black"))
def test_min():
out = df.i32.reshape((2, 2, 5)).min(axis=2).T
eq(c.points(ddf, 'x', 'y', agg=ds.min('i32')).agg, out)
eq(c.points(ddf, 'x', 'y', agg=ds.min('i64')).agg, out.astype('i8'))
eq(c.points(ddf, 'x', 'y', agg=ds.min('f32')).agg, out.astype('f4'))
eq(c.points(ddf, 'x', 'y', agg=ds.min('f64')).agg, out.astype('f8'))