def plot_ts(filename, time_range=[None, None]):
'''
Plot the time series in z3d file filename
return a holoviews dynspread object
time_range must be tz-aware
'''
# print('reading...')
z3d = read_z3d(filename)
sampling_rate = z3d['metadata']['A/D Rate']
if 'TX.SENSE' in z3d['metadata']:
# NOTE: this only ids channel 1 TX files
station_type = 'TX'
else:
station_type = 'RX'
# These items were in schedule_info for RX
try:
station = z3d['metadata']['CH.STN']
except KeyError:
try:
station = z3d['metadata']['RX.STN']
except KeyError:
station = ''
try:
station = str(int(station))
except ValueError:
station = str(station)
channel = z3d['metadata']['Channel']
component = z3d['metadata']['CH.CMP']
if z3d['metadata']['CH.CMP'][0] == 'H':
# mag field
# antenna_number = z3d['metadata']['CH.ANTSN']
pass
elif z3d['metadata']['CH.CMP'][0] == 'E':
# electric field
# antenna_number = None
pass
elif z3d['metadata']['CH.CMP'][0] == 'R':
# electric field, transmitter ref channel
# antenna_number = None
pass
else:
raise ValueError('CH.CMP not recognized as either E or H component')
if z3d['num_records'] == 0:
valid = False
else:
[dt_start, dt_end
] = timeio.get_start_and_end_times_mountain(z3d,
include_final_second=True,
astype='timestamp')
# print('getting times...')
# dt_start_naive = dt_start.replace(tzinfo=None)
# dt_end_naive = dt_end.replace(tzinfo=None)
# dt_start_naive = pd.Timestamp(dt_start.replace(tzinfo=None))
# dt_end_naive = pd.Timestamp(dt_end.replace(tzinfo=None))
# period = timedelta(seconds=1)/sampling_rate
# sampling_increment = timedelta(seconds=1./sampling_rate)
# dt_times = pd.date_range(start=dt_start_naive, periods=len(z3d['data']),
# freq=sampling_increment, tz=None,
# normalize=False, name=None, closed=None)
dt_times = pd.date_range(start=dt_start,
end=dt_end,
periods=len(z3d['data']),
tz=None,
normalize=False,
name=None,
closed=None) #.values
# print('windowing data...')
# print(type(dt_start))
# print(type(time_range[0]))
# window data to time_range
if time_range[0]:
dt_start = time_range[0]
if time_range[1]:
dt_end = time_range[1]
window_slice = timeio.slice_to_window_series(dt_times, dt_start, dt_end)
curve_data = z3d['data'][window_slice]
dt_times = dt_times[window_slice]
# ... or don't
# curve_data = z3d['data']
# print('windowed!')
# holoviews seems to want timezone-naive type datetime64[ns]. Throws warning otherwise
try:
dt_times = dt_times.tz_localize(None).astype('datetime64[ns]')
# time_range[0] = time_range[0].tz_localize(None)
# time_range[1] = time_range[1].tz_localize(None)
# hv_time_range = (time_range[0].tz_localize(None), time_range[1].tz_localize(None))
hv_time_range = (dt_start.tz_localize(None), dt_end.tz_localize(None))
# print(type(dt_times))
# print(type(dt_times[0]))
# print(dt_times[0])
# print(type(dt_start))
# print(dt_start)
dt_start = np.datetime64(dt_start.tz_localize(None))
# print(type(dt_start))
# print(dt_start)
dt_end = np.datetime64(dt_end.tz_localize(None))
except Exception as e:
print(e)
traceback.print_exc()
# print(len(curve_data))
# print(len(dt_times))
# print(hv_time_range)
# print('plotting...')
title_str = '{} Station: {}, '+\
'Channel {:.0f} ({}), {} to {} Mountain'
title_str = title_str.format(station_type, station, channel, component,
np.datetime_as_string(dt_start, unit='s'),
np.datetime_as_string(dt_end, unit='s'))
ts_tools = ['save', 'pan', 'xwheel_zoom', 'box_zoom', 'undo', 'reset']
signal = hv.Dimension('signal', label='Signal', unit='V')
time = hv.Dimension('time', label='Time', range=hv_time_range)
curve = Curve((dt_times, curve_data), time, signal,
label=title_str) #.redim.range(time=time_range)
# kdims=['Time'],vdims=['Signal, V']
# dscurve = datashade(curve, cmap=["blue"]).opts(width=800)
# dscurve = datashade(curve,name=title_str).opts(width=800)
dscurve = dynspread(
datashade(curve).opts( #framewise=True,
xformatter=verbose_formatter(),
default_tools=ts_tools))
dsfinal = dscurve.opts(plot=dict(hooks=[no_logo]))
# print('done!')
return dsfinal
def get_baseplot(sel_appl, **kwargs):
# Get abbreviation for this application
appl = appl_dict[sel_appl]
# Get the data set for this application
data = all_data[appl]
# Set some common options for all the figures
opts = {'width': 600, 'height': 400, 'colorbar': True, 'tools': ['hover']}
xylab = [hv.Dimension('xval'), hv.Dimension('yval')]
hvargs = {'kdims': xylab}
levCost = [10, 12.5, 15, 17.5, 20, 25, 30, 35, 40, 50]
# First two plots: cost at optimal capture ratio and 90%, with contour lines
im1 = hv.Image(data,
**hvargs,
label='Cost ($) at optimal CCR',
vdims=[hv.Dimension('costOpt', range=(0, 50))
]).options(**opts, cmap='RdYlGn_r')
im2 = hv.Image(data,
**hvargs,
group=appl,
label='Cost ($) at 90% CCR',
vdims=[hv.Dimension('cost90', range=(0, 50))
]).options(**opts, cmap='RdYlGn_r')
im1 = im1 * hv.operation.contours(
im1, levels=levCost).options(show_legend=False)
im2 = im2 * hv.operation.contours(
im2, levels=levCost).options(show_legend=False)
# Next two plots: cost reduction optimal vs. 90 and capture ratio optimal
im3 = hv.Image(data,
**hvargs,
group=appl,
label='Cost reduction, optimal vs. 90% CCR',
vdims=[hv.Dimension('costRedOpt90', range=(-1, 0))
]).options(**opts, cmap='Greens_r')
im4 = hv.Image(data,
**hvargs,
group=appl,
label='CCR for the optimal case',
vdims=[hv.Dimension('ccrOpt',
range=(0, 1))]).options(**opts,
cmap='RdYlGn')
im3 = im3 * hv.operation.contours(im3, levels=np.arange(
-0.5, 0, 0.1)).options(show_legend=False)
im4 = im4 * hv.operation.contours(im4, levels=np.arange(
0, 1, 0.05)).options(show_legend=False)
# Final two plots: number of stages for optimal and 90% capture ratio
im5 = hv.operation.contours(hv.Image(data,
**hvargs,
group=appl,
label='No. of stages, optimal CCR',
vdims='nstagesOpt'),
levels=[1.0, 2.0, 3.0],
filled=True).options(**opts,
cmap='Wistia',
color_levels=3)
im6 = hv.operation.contours(hv.Image(data,
**hvargs,
group=appl,
label='No. of stages, 90% CCR',
vdims='nstages90'),
levels=[1.0, 2.0, 3.0],
filled=True).options(**opts,
cmap='Wistia',
color_levels=3)
image = im1 + im2 + im3 + im4 + im5 + im6
image = image.cols(2)
return image #im1+im2
def interactive_map(v1, options):
"""
Creates and interactive map of the variable v1 on the cruise track contained in options['gps_file']
:param v1: dataframe / series containing the variable to be visualized
:param options:
- options['plot_size'] : global figure scaling (make all of them larger of smaller by this multiplier), except the interactive plots\n",
- options['figsize'] : width, height for the interactive plot\n",
- options['scatter_markersize'] : size of the markers in the scatterplot\n",
- options['map_scatter_markersize'] : size of the markers in the static geographical map\n",
- options['map_temporal_aggregation'] : Hours to aggregate in the static and interactive geographical map.\n",
- options['resampling_operation'] : resample points on map temporally,
- options['colormap'] : colormap from plt.cm
- options['gps_file'] : file containing the gps coordinates
:returns: interactive figure object
"""
import cartopy.crs as ccrs
import holoviews as hv
from holoviews import opts, dim
import geoviews as gv
import geoviews.feature as gf
import simplekml
hv.extension("bokeh", "matplotlib")
vname = v1.columns.tolist()[0]
stretch = [0, 100]
tres = np.median(np.diff(v1.index.tolist()))
tres = 1 * int(tres.total_seconds() / 60)
# vf1 = dataset.ts_aggregate_timebins(v1.to_frame(), time_bin=tres, operations={'': np.nanmean}, index_position='middle')
# come up with leg coloring
leg_series = dataset.add_legs_index(v1)["leg"]
vcol = dataset.ts_aggregate_timebins(
v1,
time_bin=tres,
operations={"": options["resampling_operation"]},
index_position="initial",
)
vcol.columns = ["color"]
min_ = np.percentile(vcol.dropna(), stretch[0])
max_ = np.percentile(vcol.dropna(), stretch[1])
# print(min_,max_)
vcol[vcol < min_] = min_
vcol[vcol > max_] = max_
coordinates_raw = dataset.read_standard_dataframe(
options["gps_file"])[["latitude", "longitude"]]
# mode_ = lambda x : stats.mode(x)[0]
# Deal with coordinates
coordinates = dataset.ts_aggregate_timebins(
coordinates_raw,
time_bin=int(np.floor(options["map_temporal_aggregation"] * 60)),
operations={"": np.nanmedian},
index_position="initial",
)
# Resample and merge coordinates + data
to_plot = pd.merge(coordinates, v1, left_index=True, right_index=True)
to_plot = pd.merge(
to_plot,
pd.DataFrame(data=to_plot.index.tolist(),
columns=["date"],
index=to_plot.index),
left_index=True,
right_index=True,
)
to_plot = to_plot.dropna()
if options["kml_file"]:
kml_ = simplekml.Kml()
fol = kml_.newfolder(name="LV_KML")
colz = (to_plot.loc[:, vname] - to_plot.loc[:, vname].min()) / (
to_plot.loc[:, vname].max() - to_plot.loc[:, vname].min())
colz = np.floor(255 * plt.cm.Spectral_r(colz.values)).astype(int)
c = 0
for lat, lon, val, date in to_plot.values:
# print(lat, lon, val, date)
# print(row[1].loc['LV#11'])
pnt = fol.newpoint(name=str(date), coords=[(lon, lat)])
pnt.style.iconstyle.icon.href = (
"http://maps.google.com/mapfiles/kml/shapes/shaded_dot.png")
pnt.style.iconstyle.scale = 1 # Icon thrice as big
pnt.style.iconstyle.color = simplekml.Color.rgb(
colz[c, 0], colz[c, 1], colz[c, 2], 255)
pnt.style.labelstyle.scale = 0.5
c += 1
kml_.save(options["kml_file"])
# colorbar limits
min_v1 = min_ # np.percentile(to_plot.loc[:,vname].dropna(), stretch[0])
max_v1 = max_ # np.percentile(to_plot.loc[:,vname].dropna(), stretch[1])
# Create geoviews datasets
v1_tomap = hv.Dataset(
to_plot.loc[:, ["longitude", "latitude", "date", vname]],
kdims=["longitude", "latitude"],
vdims=[hv.Dimension(vname, range=(min_v1, max_v1))],
group=vname,
)
points_v1 = v1_tomap.to(gv.Points, kdims=["longitude", "latitude"])
gps_track = gv.Dataset(coordinates_raw)
track = gv.Path(gps_track, kdims=["longitude", "latitude"])
# land_ = gf.land#.options(facecolor='red')
point_map_v1 = points_v1.opts(
projection=ccrs.SouthPolarStereo(),
cmap=options["colormap"],
size=5,
tools=["hover"], # ['hover'],
width=500,
height=400,
color_index=2,
colorbar=True,
)
track_map = track.opts(projection=ccrs.SouthPolarStereo()).opts(
color="black")
return (gf.land * gf.coastline * track_map * point_map_v1).opts(
title=vname, width=options["figsize"][0], height=options["figsize"][1])
# create HoloMap object with varying slope
n_a = 20 # number of slope changes
lower_slope_limit = 0.4
upper_slope_limit = 1.2
# let a vary between lower and upper_slope_limit
a_var = np.linspace(lower_slope_limit,upper_slope_limit,n_a)
y_list = []
for i in range(n_a):
y_list.append(hv.Curve((x,fct(a_var[i],x,b))).options(opts_curve))
# create HoloMap object
dict_y = {round(a_var[i], 2):y_list[i] for i in range(n_a)}
dim_a = hv.Dimension(('a','a'), default=round(a_var[8], 2))
hmap_y = hv.HoloMap(dict_y, kdims = dim_a).options(title_format='')
# Find fixpoint (if it exists, which it doesn't if a >= 1)
x_0 = 8.0 # starting value
n_steps = 7
steps = range(0,n_steps)
hv_path_list = []
for i in range(n_a):
hv_path_list_j = []
x_n = x_0
path = []
for j in steps:
def test_holoview():
import holoviews as hv
hv.extension('bokeh')
model = Generate()
alloc = Amounts() + Simulation()
exec = sim.engine.BasicExecution(model, alloc)
tr = model['grow', 'const', 'alt']
ht = tr.take(6).to(sim.trace.Holotrace)
assert 'Holotrace' in str(ht)
assert str(ht.trace) in str(ht)
assert repr(ht.trace) in repr(ht)
assert list(ht.data.columns) == ['grow', 'const', 'alt']
assert list(ht.data.index.names) == ['trace', 'sample']
dm = ht.plot(hv.Curve)
hv.render(dm) # trigger rendering, so dynamic map gets filled
over, = dm
assert len(over) == 6 * 3
crv, *_ = over
assert isinstance(dm, hv.DynamicMap)
assert isinstance(over, hv.NdOverlay)
assert isinstance(crv, hv.Curve)
assert crv.kdims == [hv.Dimension('trace')]
assert crv.vdims == [hv.Dimension('value')]
bf = ht.buffer
assert len(bf.data) == 6
with exec.trace(ht):
exec.run(n_steps=2)
assert ht.buffer == bf
assert len(bf.data) == 3 * 6
ht = tr.take(6).to(sim.trace.Holotrace, skip=4, batch=5)
with exec.trace(ht):
exec.run()
print(ht.traces)
print(ht.data)
assert ht.data.shape == (6 * 5, 3)
assert ht.data.shape == (6 * 6, 3)
ht = tr.take(6).to(sim.trace.Holotrace, skip=4, batch=100, timeout=10)
with exec.trace(ht):
exec.run()
assert ht.data.shape == (0, 3)
assert ht.data.shape == (6 * 6, 3)
ht = tr.take(6).to(sim.trace.Holotrace, skip=4, batch=100, timeout=.001)
with exec.trace(ht):
exec.run()
assert ht.data.shape[0] > 0
assert ht.data.shape == (6 * 6, 3)
if len(sys.argv) < 2:
print('We need a run#!')
sys.exit()
run = int(sys.argv[1])
anaps = sdaps(expname, run)
ana = anaps.sda
#ana = sda(expname,run)
## Defining initial selection (laser-on events)
iniFilter = 'initial'
ana.addCut('lightStatus/xray', 0.5, 1.5, iniFilter)
ana.addCut('lightStatus/laser', 0.5, 1.5, iniFilter)
### Get data & define axis title&ranges.
ipmUpDim = hv.Dimension(('ipm4/sum', 'ipm4 Sum'))
ipmDownDim = hv.Dimension(('ipm5/sum', 'ipm5 Sum'))
scatterDim = hv.Dimension(('epix10k2M/ROI_0_sum', 'epix10k2M intensity'))
eventTimeDim = hv.Dimension(('eventTimeR', 'relative event time'))
l3eDim = hv.Dimension(('l3e', 'L3 Energy'))
scanVar = ana.getScanName()
try:
scanDim = hv.Dimension(('scan/%s' % scanVar, '%s' % scanVar))
except:
scanDim = None
nevtsDim = hv.Dimension(('nevents', 'N events / scan point'))
nevtsLxtDim = hv.Dimension(('neventslxt', 'N events / lxt'))
#timing vars.
lxtDim = hv.Dimension(('epics/lxt', 'lxt'))
def plot_ly(lon, lat, alt, time, power, density, weight, bins, file_path,
clear_fig, current_time_range, maps):
'''Plotting routine to view raw data. Select map=False or True to view a 3 panel or single panel
plot of the raw lma data. All other kwargs are used to display specific user selected traits of the data,
e.g. raw source locations, gridded source count densities, etc.
'''
fig = make_subplots(
rows=5,
cols=4,
shared_xaxes=True,
shared_yaxes=True,
specs=[[{
'colspan': 3,
"rowspan": 2
}, None, {}, {}], [None, None, None, None],
[{
'colspan': 3,
"rowspan": 3
}, {}, {}, {
"rowspan": 3
}], [None, None, None, None], [None, None, None, None]],
# print_grid=True
)
fig.update_layout(autosize=False, width=1000, height=900)
if maps == True:
if clear_fig == 'No Data Found':
print('Nope')
if density == 'Points' or density == 'Density':
if weight is not None:
fig.add_trace(go.Scatter(x=[0, 0, 0], y=[0, 0, 0]),
row=1,
col=1)
fig.add_trace(go.Scatter(x=[0, 0, 0], y=[0, 0, 0]),
row=3,
col=4)
fig.add_trace(go.Scatter(x=[0, 0, 0], y=[0, 0, 0]),
row=3,
col=1)
else:
if density == 'Points':
print('Picked Points')
fig.add_trace(go.Scatter(x=[0, 0, 0], y=[0, 0, 0]),
row=1,
col=1)
fig.add_trace(go.Scatter(x=[0, 0, 0], y=[0, 0, 0]),
row=3,
col=4)
fig.add_trace(go.Scatter(x=[0, 0, 0], y=[0, 0, 0]),
row=3,
col=1)
if weight == 'None':
fig.update_traces(go.Scatter(
x=lon,
y=alt,
mode='markers',
),
row=1,
col=1)
fig.update_traces(go.Scatter(
x=lon,
y=lat,
mode='markers',
),
row=3,
col=1)
fig.update_traces(go.Scatter(
x=alt,
y=lat,
mode='markers',
),
row=3,
col=4)
else:
if weight == 'alt':
weight = alt / 1e4
cmap = cm.cm.deep_r
elif weight == 'time':
weight = time / 1e9
cmap = plt.cm.magma
elif weight == 'power':
weight = power * 3e1
cmap = cm.cm.algae_r
fig.update_traces(go.Scatter(x=lon,
y=alt,
mode='markers',
marker=dict(size=weight,
color='black')),
row=1,
col=1)
fig.update_traces(go.Scatter(x=lon,
y=lat,
mode='markers',
marker=dict(size=weight,
color='black')),
row=3,
col=1)
fig.update_traces(go.Scatter(x=alt,
y=lat,
mode='markers',
marker=dict(size=weight,
color='black')),
row=3,
col=4)
else:
h0xy, xe1, ye1 = np.histogram2d(lon,
lat,
bins=(int(bins), int(bins)))
h0xz, xe2, ye2 = np.histogram2d(lon,
alt,
bins=(int(bins), int(bins)))
h0yz, xe3, ye3 = np.histogram2d(alt,
lat,
bins=(int(bins), int(bins)))
colors = fig.add_trace(go.Heatmap(z=h0xy.T,
showscale=True,
coloraxis='coloraxis'),
row=3,
col=1)
fig.add_trace(go.Heatmap(z=h0xz.T, coloraxis='coloraxis'),
row=1,
col=1)
fig.add_trace(go.Heatmap(z=h0yz.T, coloraxis='coloraxis'),
row=3,
col=4)
if weight == 'None':
cbar_title = r'LMA Sources per square km'
fig.update_traces(go.Heatmap(x=xe1,
y=ye1,
z=np.log10(h0xy).T,
showscale=True,
coloraxis='coloraxis'),
row=3,
col=1)
fig.update_traces(go.Heatmap(x=xe2,
y=ye2,
z=np.log10(h0xz).T,
coloraxis='coloraxis'),
row=1,
col=1)
fig.update_traces(go.Heatmap(x=xe3,
y=ye3,
z=np.log10(h0yz).T,
coloraxis='coloraxis'),
row=3,
col=4)
color_bar_pref = dict(
title=cbar_title,
thicknessmode="pixels",
thickness=40,
lenmode="pixels",
len=600,
yanchor="bottom",
y=0.05,
)
fig.update_layout(coloraxis_colorbar=color_bar_pref,
coloraxis={'colorscale': 'viridis'})
else:
if weight == 'alt':
cbar_title = 'LMA Source Altitude'
weight = alt
cmap = cm.cm.deep_r
elif weight == 'time':
cbar_title = 'LMA Source Time'
weight = time
cmap = plt.cm.magma
elif weight == 'power':
cbar_title = 'LMA Source Power'
weight = power
cmap = cm.cm.algae_r
h0xy0, xe1, ye1 = np.histogram2d(lon,
lat,
bins=(int(bins),
int(bins)),
weights=weight)
h0xz0, xe2, ye2 = np.histogram2d(lon,
alt,
bins=(int(bins),
int(bins)),
weights=weight)
h0yz0, xe3, ye3 = np.histogram2d(alt,
lat,
bins=(int(bins),
int(bins)),
weights=weight)
fig.update_traces(go.Heatmap(x=xe1,
y=ye1,
z=(h0xy0 / h0xy).T,
showscale=True,
coloraxis='coloraxis'),
row=3,
col=1)
fig.update_traces(go.Heatmap(x=xe2,
y=ye2,
z=(h0xz0 / h0xz).T,
coloraxis='coloraxis'),
row=1,
col=1)
fig.update_traces(go.Heatmap(x=xe3,
y=ye3,
z=(h0yz0 / h0yz).T,
coloraxis='coloraxis'),
row=3,
col=4)
color_bar_pref = dict(
title=cbar_title,
thicknessmode="pixels",
thickness=40,
lenmode="pixels",
len=600,
yanchor="bottom",
y=0.05,
)
fig.update_layout(coloraxis_colorbar=color_bar_pref,
coloraxis={'colorscale': 'viridis'})
fig.update_layout(showlegend=False,
title_text=r"Date: {0}-{1}-{2} {3}".format(
file_path.year, file_path.month,
file_path.day, current_time_range))
fig.update_xaxes(matches='x', row=1, col=1)
fig.update_xaxes(matches='x', row=3, col=1)
fig.update_yaxes(matches='y2', row=3, col=1)
fig.update_yaxes(matches='y2', row=3, col=4)
fig.update_yaxes(matches='y', row=1, col=1)
fig.update_xaxes(matches='y', row=3, col=4)
responsive = pn.pane.Plotly(fig, config={'responsive': True})
else:
#Original use of Plotl's MapBox
#fig.add_trace(go.Densitymapbox(lon=lon, lat=lat,
# radius=20))
#fig.update_layout(mapbox_style="carto-positron",mapbox_center_lon=-101.5,mapbox_center_lat=34,mapbox=dict(zoom=6))
#fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0})
#Switch to Holoviews for consistency:
raw_hist, xe, ye = np.histogram2d(lon,
lat,
bins=(int(bins), int(bins)))
# get xarray dataset, suited for handling raster data in pyviz
xr_dataset = gv.Dataset(
hv.Image(
(xe, ye,
np.log10(np.ma.MaskedArray(raw_hist.T, mask=raw_hist.T
== 0))),
bounds=(xe.min(), ye.min(), xe.max(), ye.max()),
kdims=[hv.Dimension('x'), hv.Dimension('y')],
datatype=['grid']))
# create contours from image
gv.FilledContours(xr_dataset)
fig = gv.tile_sources.Wikipedia.opts(
width=900, height=900) * xr_dataset.to.image(['x', 'y']).opts(
cmap='cubehelix', alpha=0.8
) #gv.FilledContours(xr_dataset).opts(cmap='viridis', alpha=0.5)
responsive = pn.pane.HoloViews(fig, config={'responsive': True})
return (responsive)
def distance_matrix(SBJ, RUN, WL_sec, x_dim, y_dim, z_dim):
"""
This fuction plots a heat map of the distnaces of each window for a given run.
The inputs for the fuction (subject, run, and window leght) allows the user to choose what run and window leghth
they with to plot for a given subject.
The distance between two windows (i.e. points on the 3D plot) is computed using numpys squareform(pdist()).
The fuction plots the heat map using holoviews hv.Image().
The x and y axes of the plot are the two windows in which you are finding the distance.
The z value is that distance.
A plot of the sleep staging segments are ploted along the x and y axis of the image using hv.Segments().
If all runs are being displayed a plot of the run segments are ploted along the x and y axis of the image using hv.Segments().
"""
LE3D_df = load_data(SBJ, RUN, WL_sec) # Load embedding data
LE3D_df = LE3D_df.infer_objects(
) # Infer objects to be int, float, or string apropriatly
LE3D_df = LE3D_df[[
'x' + str(x_dim).zfill(2), 'x' + str(y_dim).zfill(2),
'x' + str(z_dim).zfill(2), 'x' + str(x_dim).zfill(2) + '_norm',
'x' + str(y_dim).zfill(2) + '_norm',
'x' + str(z_dim).zfill(2) + '_norm', 'Run', 'Sleep Value',
'Sleep Stage', 'mean FD', 'label'
]]
LE3D_df.columns = [
'x', 'y', 'z', 'x_norm', 'y_norm', 'z_norm', 'Run', 'Sleep Value',
'Sleep Stage', 'mean FD', 'label'
]
data_path = osp.join(PRJDIR, 'PrcsData', SBJ, 'D02_Preproc_fMRI',
SBJ + '_' + RUN + '_WL_' + str(WL_sec) +
'sec_Sleep_Segments.pkl') # path to segment data
sleep_segments_df = pd.read_pickle(data_path) # Load segment data
data_df = LE3D_df[[
'x_norm', 'y_norm', 'z_norm', 'Sleep Stage'
]].copy() # New data frame of only x_norm, y_norm, and z_norm values
num_win = data_df.shape[0] # Number of windwos in data
data_array = data_df[['x_norm', 'y_norm',
'z_norm']].to_numpy() # Data as a numpy array
dist_array = squareform(pdist(
data_array,
'euclidean')) # Calculate distance matrix and rehape into one vecotr
dist_array = xr.DataArray(dist_array,
dims=['Time [Window ID]', 'Time [Window ID] Y'
]) # Distances as x_array data frame
sleep_color_map = {
'Wake': 'orange',
'Stage 1': 'yellow',
'Stage 2': 'green',
'Stage 3': 'blue',
'Undetermined': 'gray'
} # Color key for sleep staging
# Plot of sleep staging segements along the x and y axis
# Range is from (-10, num_win) so we have space to display segments
sleep_seg_x = hv.Segments(sleep_segments_df, [
hv.Dimension('start', range=(-10, num_win)),
hv.Dimension('start_event', range=(-5, num_win)), 'end', 'end_event'
], 'stage').opts(color='stage',
cmap=sleep_color_map,
line_width=10,
show_legend=False)
sleep_seg_y = hv.Segments(sleep_segments_df, [
hv.Dimension('start_event', range=(-10, num_win)),
hv.Dimension('start', range=(-5, num_win)), 'end_event', 'end'
], 'stage').opts(color='stage',
cmap=sleep_color_map,
line_width=10,
show_legend=False)
# If plotting all runs add segent to x and y axis for coloring by run
if RUN == 'All':
run_list = [
SubDict[SBJ][i][0] for i in range(0,
len(SubDict[SBJ]) - 1)
] # List of all runs
time_list = [
SubDict[SBJ][i][1] for i in range(0,
len(SubDict[SBJ]) - 1)
] # List of all run lenghts in TR's (in the same order as runs in list above)
WL_trs = int(WL_sec / 2) # Window length in TR's (TR = 2.0 sec)
run_segments_df = pd.DataFrame(
columns=['run', 'start',
'end']) # Emptly data frame for segment legths of runs
# For each run a row is appended into the data frame created above (run_segments_df) with the run name and the start and end window of the run
# For the windows that overlap runs the run will be called 'Inbetween Runs'
x = 0 # Starting at 0th window
for i in range(len(run_list)):
time = time_list[i] # Number of windows in run
run = run_list[i] # Name of run
end = x + time - WL_trs # Last window of run
if i == len(
run_list
) - 1: # If its the last run no need to append inbetween run
run_segments_df = run_segments_df.append(
{
'run': run,
'start': x,
'end': end
}, ignore_index=True) # Append run info
else:
run_segments_df = run_segments_df.append(
{
'run': run,
'start': x,
'end': end
}, ignore_index=True) # Append run info
x = end + 1
run_segments_df = run_segments_df.append(
{
'run': 'Inbetween Runs',
'start': x,
'end': (x - 1) + (WL_trs - 1)
},
ignore_index=True) # Append inbetween run info
x = x + (WL_trs - 1)
# Add 0.5 to each end of segment to span entire heat map
run_segments_df['start'] = run_segments_df['start'] - 0.5
run_segments_df['end'] = run_segments_df['end'] + 0.5
# 'start_event' and 'end_event' represent the axis along which the segments will be (-50 so it is not on top of the heat map or sleep segments)
run_segments_df['start_event'] = -50
run_segments_df['end_event'] = -50
# Color key for runs
run_color_map = {
'SleepAscending': '#DE3163',
'SleepDescending': '#FF7F50',
'SleepRSER': '#FFBF00',
'WakeAscending': '#6495ED',
'WakeDescending': '#40E0D0',
'WakeRSER': '#CCCCFF',
'Inbetween Runs': 'gray'
}
# Plot of run segements along the x and y axis
# Range is from (-80, num_win) so we have space to display both segments
run_seg_x = hv.Segments(run_segments_df, [
hv.Dimension('start', range=(-80, num_win)),
hv.Dimension('start_event', range=(-80, num_win)), 'end',
'end_event'
], 'run').opts(color='run',
cmap=run_color_map,
line_width=10,
show_legend=False)
run_seg_y = hv.Segments(run_segments_df, [
hv.Dimension('start_event', range=(-80, num_win)),
hv.Dimension('start', range=(-80, num_win)), 'end_event', 'end'
], 'run').opts(color='run',
cmap=run_color_map,
line_width=10,
show_legend=False)
segment_plot = (
sleep_seg_x * sleep_seg_y * run_seg_x * run_seg_y).opts(
xlabel=' ', ylabel=' ',
show_legend=False) # All segments (run and sleep) overlayed
else:
segment_plot = (sleep_seg_x * sleep_seg_y).opts(
xlabel=' ', ylabel=' ',
show_legend=False) # All segments (not including runs)overlayed
# Plot heat map using hv.Image
# Set bounds to (-0.5,-0.5,num_win-0.5,num_win-0.5) to corespond with acurate windows
plot = hv.Image(dist_array,
bounds=(-0.5, -0.5, num_win - 0.5,
num_win - 0.5)).opts(cmap='jet',
colorbar=True,
ylabel='Time [Window ID]')
# Overlay segment plots and heat map
output = (plot * segment_plot).opts(width=600, height=390)
return output
def plot_histogram(self, data):
plot_data, xlim = data
return holoviews.Histogram(plot_data).redim(x=holoviews.Dimension("x", range=xlim))
def update(pattern, counter, x, y):
if x and y:
pattern = np.array(shapes[pattern])
r, c = pattern.shape
y, x = img.sheet2matrixidx(x, y)
img.data[y:y + r, x:x + c] = pattern[::-1]
else:
img.data = step(img.data)
return hv.Image(img)
# Set up plot which advances on counter and adds pattern on tap
title = 'Game of Life - Tap to place pattern, Doubletap to clear'
img = hv.Image(np.zeros((100, 200), dtype=np.uint8))
counter, tap = Counter(transient=True), Tap(transient=True),
pattern_dim = hv.Dimension('Pattern', values=sorted(shapes.keys()))
dmap = hv.DynamicMap(update, kdims=[pattern_dim], streams=[counter, tap])
plot = dmap.opts(
opts.Image(cmap='gray',
clim=(0, 1),
toolbar=None,
responsive=True,
min_height=800,
title=title,
xaxis=None,
yaxis=None))
# Add callback to clear on double tap
def reset_data(x, y):
def test_show_mask(self):
arr = np.random.rand(3, 3, 3)
cube1 = xr.DataArray(arr,
dims=['a', 'b', 'c'],
coords={
'a': [1, 2, 3],
'b': [1, 2, 3],
'c': [1, 2, 3]
})
arr2 = np.random.rand(4, 3, 2)
cube2 = xr.DataArray(arr2,
dims=['a', 'b', 'c'],
coords={
'a': [1, 2, 3, 4],
'b': [1, 2, 3],
'c': [1, 2]
})
cube2.M.reset(dims=['b', 'c'])
layout = cube1.visualize.show_mask()
layout2 = cube2.visualize.show_mask()
self.assertTrue(isinstance(layout, hv.Image))
self.assertTrue(isinstance(layout2, hv.Image))
dims = [hv.Dimension('c'), hv.Dimension('b'), hv.Dimension('Value')]
dims2 = [hv.Dimension('c'), hv.Dimension('b'), hv.Dimension('Value')]
self.assertEqual(layout.dimensions(), dims)
self.assertEqual(layout2.dimensions(), dims2)
cube2.M.reset(dims=['c', 'b'])
layout2 = cube2.visualize.show_mask()
self.assertTrue(isinstance(layout2, hv.Image))
self.assertEqual(layout2.dimensions(), dims2)
cube2.M.reset(dims=['a', 'c'])
layout2 = cube2.visualize.show_mask()
dims2 = [hv.Dimension('c'), hv.Dimension('a'), hv.Dimension('Value')]
self.assertTrue(isinstance(layout2, hv.Image))
self.assertEqual(layout2.dimensions(), dims2)
cube2.M.reset(dims=['b', 'a'])
layout2 = cube2.visualize.show_mask()
dims2 = [hv.Dimension('b'), hv.Dimension('a'), hv.Dimension('Value')]
self.assertTrue(isinstance(layout2, hv.Image))
self.assertEqual(layout2.dimensions(), dims2)
def _plot_tasks_activation(self, tasks, show_legend=None, cpu: CPU=None, alpha:
float=None, overlay: bool=False, duration: bool=False, duty_cycle:
bool=False, which_cpu: bool=False, height_duty_cycle: bool=False, best_effort=False):
logger = self.logger
def ensure_last_rectangle(df):
# Make sure we will draw the last rectangle, which could be
# critical for tasks that are never sleeping
if df.empty:
return df
else:
start = self.trace.start
last_duration = df['duration'].iat[-1]
if pd.isna(last_duration):
end = self.trace.end
else:
end = df.index[-1] + last_duration
# If the rectangle finishes before the beginning of the trace
# window, we ignore it
if start <= end:
# Clip the beginning so that plots don't extend to the
# left of the trace window.
return df_refit_index(df, window=(start, end))
else:
return df.iloc[0:0]
def make_twinx(fig, **kwargs):
return _hv_twinx(fig, **kwargs)
if which_cpu:
def make_rect_df(df):
half_height = df['active'] / 2
return pd.DataFrame(
dict(
Time=df.index,
CPU=df['cpu'] - half_height,
x1=df.index + df['duration'],
y1=df['cpu'] + half_height,
),
index=df.index
)
else:
def make_rect_df(df):
if duty_cycle or duration:
max_val = max(
df[col].max()
for select, col in (
(duty_cycle, 'duty_cycle'),
(duration, 'duration')
)
if select
)
height_factor = max_val
else:
height_factor = 1
return pd.DataFrame(
dict(
Time=df.index,
CPU=0,
x1=df.index + df['duration'],
y1=df['active'] * height_factor,
),
index=df.index,
)
def plot_extra(task, df):
figs = []
if duty_cycle:
figs.append(
plot_signal(df['duty_cycle'], name=f'Duty cycle of {task}')
)
if duration:
def plot_duration(active, label):
duration_series = df[df['active'] == active]['duration']
# Add blanks in the plot when the state is not the one we care about
duration_series = duration_series.reindex_like(df)
return plot_signal(duration_series, name=f'{label} duration of {task}')
figs.extend(
plot_duration(active, label)
for active, label in (
(True, 'Activations'),
(False, 'Sleep')
)
)
return figs
def check_df(task, df, empty_is_none):
if df.empty:
msg = f'Could not find events associated to task {task}'
if empty_is_none:
logger.debug(msg)
return None
else:
raise ValueError(msg)
else:
return ensure_last_rectangle(df)
def get_task_data(task, df):
df = df.copy()
# Preempted == sleep for plots
df['active'] = df['active'].fillna(0)
if height_duty_cycle:
df['active'] *= df['duty_cycle']
data = make_rect_df(df[df['active'] != 0])
name_df = self.trace.df_event('sched_switch')
name_df = name_df[name_df['next_pid'] == task.pid]
names = name_df['next_comm'].reindex(data.index, method='ffill')
# If there was no sched_switch with next_pid matching task.pid, we
# simply take the last known name of the task, which could
# originate from another field or another event.
#
# Note: This prevents an <NA> value, which makes bokeh choke.
last_comm = self.trace.get_task_pid_names(task.pid)[-1]
if last_comm not in names.cat.categories:
names = names.cat.add_categories([last_comm])
names = names.fillna(last_comm)
# Use a string for PID so that holoviews interprets it as
# categorical variable, rather than continuous. This is important
# for correct color mapping
data['pid'] = str(task.pid)
data['comm'] = names
data['start'] = data.index
data['cpu'] = df['cpu']
data['duration'] = df['duration']
data['duty_cycle'] = df['duty_cycle']
return data
def plot_rect(data):
if show_legend:
opts = {}
else:
# If there is no legend, we are gonna plot all the rectangles at once so we use colormapping to distinguish the tasks
opts = dict(
color='pid',
# Colormap from colorcet with a large number of color, so it is
# suitable for plotting many tasks
cmap='glasbey_hv',
)
return hv.Rectangles(
data,
kdims=[
hv.Dimension('Time'),
hv.Dimension('CPU'),
hv.Dimension('x1'),
hv.Dimension('y1'),
]
).options(
show_legend=show_legend,
alpha=alpha,
**opts,
).options(
backend='matplotlib',
linewidth=0,
).options(
backend='bokeh',
line_width=0,
tools=[self._BOKEH_TASK_HOVERTOOL],
)
if alpha is None:
if overlay or duty_cycle or duration:
alpha = 0.2
else:
alpha = 1
# For performance reasons, plot all the tasks as one hv.Rectangles
# invocation when we get too many tasks
if show_legend is None:
if overlay:
# TODO: twinx() breaks on hv.Overlay, so we are forced to use a
# single hv.Rectangles in that case, meaning no useful legend
show_legend = False
else:
show_legend = len(tasks) < 5
cpus_count = self.trace.cpus_count
task_dfs = {
task: check_df(
task,
self.df_task_activation(task, cpu=cpu),
empty_is_none=best_effort,
)
for task in tasks
}
if best_effort:
task_dfs = {
task: df
for task, df in task_dfs.items()
if df is not None
}
tasks = sorted(task_dfs.keys())
if show_legend:
fig = hv.Overlay(
[
plot_rect(get_task_data(task, df)).relabel(
f'Activations of {task.pid} (' +
', '.join(
task_id.comm
for task_id in self.trace.get_task_ids(task)
) +
')',
)
for task, df in task_dfs.items()
]
).options(
legend_limit=len(tasks) * 100,
)
else:
data = pd.concat(
get_task_data(task, df)
for task, df in task_dfs.items()
)
fig = plot_rect(data)
if overlay:
fig = make_twinx(
fig,
y_range=(-1, cpus_count),
display=False
)
else:
if which_cpu:
fig = fig.options(
'Rectangles',
ylabel='CPU',
yticks=[
(cpu, f'CPU{cpu}')
for cpu in range(cpus_count)
],
).redim(
y=hv.Dimension('y', range=(-0.5, cpus_count - 0.5))
)
elif height_duty_cycle:
fig = fig.options(
'Rectangles',
ylabel='Duty cycle',
)
if duty_cycle or duration:
if duty_cycle:
ylabel = 'Duty cycle'
elif duration:
ylabel = 'Duration (s)'
# TODO: twinx() on hv.Overlay does not work, so we unfortunately have a
# scaling issue here
fig = hv.Overlay(
[fig] +
[
fig
for task, df in task_dfs.items()
for fig in plot_extra(task, df)
]
).options(
ylabel=ylabel,
)
return fig.options(
title='Activations of {}'.format(
', '.join(map(str, tasks))
),
)
def res_diag(points_list, color='red'):
x = hv.Dimension('ν_x', range=(0, 1))
y = hv.Dimension('ν_y', range=(0, 1))
path = hv.Path(points_list, [x, y])
path.opts(width=700, height=700, color=color, line_width=1)
return path
hv.extension('bokeh')
#%%
#Create dataframe from files in folder.
data_path = '/Users/keatra/Galvanize/Projects/Instagram_NLP/users'
data = create_full_df(data_path)
data.head()
#%%
#Graph a histogram of the number of likes
num_likes = hv.Histogram(np.histogram(data['number_of_likes'], 250))
num_likes.opts(title='Number of Likes Frequency',
xlabel='Amount of Likes',
xticks=10)
num_likes.redim(x=hv.Dimension('x', range=(0, 200)))
#%%
#Graph relationship between number of likes and number of words
likes = data['number_of_likes'].values
words = data['number_of_words'].values
words_likes = hv.Scatter(
(words, likes)).opts(xlabel='Number of words in caption',
ylabel='Number of likes',
title='Words vs. Likes')
words_likes.redim(x=hv.Dimension('x', range=(0, 100)),
y=hv.Dimension('y', range=(0, 100)))
#%%
#Graph a histogram of the number of words per document
def task_fMRI_plots(SBJ, PURE, WL_sec, corr_range):
# Define segment data
# -------------------
if WL_sec == 30:
if PURE == 'pure':
seg_df = WL30pure_taskseg_df
else:
seg_df = WL30_taskseg_df
else:
if PURE == 'pure':
seg_df = WL45pure_taskseg_df
else:
seg_df = WL45_taskseg_df
# Define PURE varaible based on widget
# ------------------------------------
if PURE == 'not pure':
PURE = '' # Load data with non pure windows
# Load task fMRI data
# -------------------
file_name = SBJ + '_CTask001_WL0' + str(
WL_sec) + '_WS01' + PURE + '_NROI0200_dF.mat' # Data file name
data_path = osp.join(
'/data/SFIMJGC_HCP7T/PRJ_CognitiveStateDetection02/PrcsData_PNAS2015',
SBJ, 'D02_CTask001', file_name) # Path to data
data_df = loadmat(data_path)['CB']['snapshots'][0][0] # Read data
num_samp = data_df.shape[0] # Save number of samples as a varable
# Create sleep segments plots
# ---------------------------
task_color_map = {
'Rest': 'gray',
'Memory': 'blue',
'Video': 'yellow',
'Math': 'green',
'Inbetween': 'black'
} # Color key for task segments
seg_x = hv.Segments(seg_df, [
hv.Dimension('start', range=(-10, num_samp - 1.5)),
hv.Dimension('start_event', range=(-5, num_samp - 1.5)), 'end',
'end_event'
], 'task').opts(color='task',
cmap=task_color_map,
line_width=7,
show_legend=True) # x axis segments
seg_y = hv.Segments(seg_df, [
hv.Dimension('start_event', range=(-10, num_samp - 1.5)),
hv.Dimension('start', range=(-5, num_samp - 1.5)), 'end_event', 'end'
], 'task').opts(color='task',
cmap=task_color_map,
line_width=7,
show_legend=False) # y axis segments
seg_plot = (seg_x * seg_y).opts(xlabel=' ', ylabel=' ',
show_legend=False) # All segments
# Compute correlation and distance matrix
# ---------------------------------------
data_corr = np.corrcoef(data_df) # Correlation matrix
data_dist = pairwise_distances(data_df,
metric='euclidean') # Distance matrix
# Compute distribution of correlation and distance matrix
# -------------------------------------------------------
triangle = np.mask_indices(num_samp, np.triu,
k=1) # Top triangle mask for matricies
corr_freq, corr_edges = np.histogram(
np.array(data_corr)[triangle], 100
) # Compute histogram of top triangle of correlation matrix (100 bars)
dist_freq, dist_edges = np.histogram(
np.array(data_dist)[triangle],
100) # Compute histogram of top triangle of distance matrix (100 bars)
# Create matrix and histogram plots
# ---------------------------------
corr_img = hv.Image(
np.rot90(data_corr),
bounds=(-0.5, -0.5, num_samp - 1.5, num_samp - 1.5)).opts(
cmap='viridis',
colorbar=True,
height=300,
width=400,
title='Correlation Matrix').redim.range(z=corr_range)
dist_img = hv.Image(np.rot90(data_dist),
bounds=(-0.5, -0.5, num_samp - 1.5,
num_samp - 1.5)).opts(cmap='viridis',
colorbar=True,
height=300,
width=400,
title='Distance Matrix')
corr_his = hv.Histogram(
(corr_edges, corr_freq)).opts(xlabel='Correlation',
height=300,
width=400,
title='Correlation Histogram')
dist_his = hv.Histogram(
(dist_edges, dist_freq)).opts(xlabel='Distance',
height=300,
width=400,
title='Distance Histogram')
corr_img_wseg = (corr_img * seg_plot).opts(
width=600, height=300, legend_position='right'
) # Overlay task segemnt plot with correlation matrix
dist_img_wseg = (dist_img * seg_plot).opts(
width=600, height=300, legend_position='right'
) # Overlay task segemnt plot with distance matrix
dash = (corr_img_wseg + corr_his + dist_img_wseg + dist_his).opts(
opts.Layout(shared_axes=False)).cols(2) # Dashboard of all plots
return dash
def _heatmap(adata,
genes,
group,
sort_genes=True,
use_raw=False,
agg_fns=['mean'],
hover=True,
xrotation=90,
yrotation=0,
colorbar=True,
cmap=None,
plot_height=300,
plot_width=600):
'''
Internal heatmap function.
Params
-------
adata: anndata.AnnData
adata object
genes: List[Str]
genes in `adata.var_names`
group: Str
key in `adata.obs`, must be categorical
sort_genes: Bool, optional (default: `True`)
whether to sort the genes
use_raw: Bool, optional (default: `True`)
whether to use `.raw` attribute
agg_fns: List[Str], optional (default: `['mean']`)
list of pandas' aggregation functions
hover: Bool, optional (deault: `True`)
whether to show hover information
xrotation: Int, optional (default: `90`)
rotation of labels on x-axis
yrotation: Int, optional (default: `0`)
rotation of labels on y-axis
colorbar: Bool, optional (default: `True`)
whether to show colorbar
cmap: Union[List[Str], NoneType], optional (default, `None`)
colormap of the heatmap,
if `None`, use `Viridis256`
plot_height: Int, optional (default: `600`)
height of the heatmap
plot_width: Int, optional (default: `200`)
width of the heatmap
Returns
-------
plot: hv.HeatMap
a heatmap
'''
assert group in adata.obs
assert is_categorical(adata.obs[group])
assert len(agg_fns) > 0
for g in genes:
assert g in adata.var_names, f'Unable to find gene `{g}` in `adata.var_names`.'
genes = sorted(genes) if sort_genes else genes
groups = sorted(list(adata.obs[group].cat.categories))
adata_mraw = get_mraw(adata, use_raw)
common_subset = list(set(adata.obs_names) & set(adata_mraw.obs_names))
adata, adata_mraw = adata[common_subset, :], adata_mraw[common_subset, :]
ixs = np.in1d(adata.obs[group], groups)
adata, adata_mraw = adata[ixs, :], adata_mraw[ixs, :]
df = pd.DataFrame(adata_mraw[:, genes].X, columns=genes)
df['group'] = list(map(str, adata.obs[group]))
groupby = df.groupby('group')
vals = {agg_fn: groupby.agg(agg_fn) for agg_fn in agg_fns}
z_value = vals.pop(agg_fns[0])
x = hv.Dimension('x', label='Gene')
y = hv.Dimension('y', label='Group')
z = hv.Dimension('z', label='Expression')
vdims = [(k, k.capitalize()) for k in vals.keys()]
heatmap = hv.HeatMap(
{
'x': np.array(genes),
'y': np.array(groups),
'z': z_value,
**vals
},
kdims=[('x', 'Gene'), ('y', 'Group')],
vdims=[('z', 'Expression')] + vdims).opts(tools=['box_select'] +
(['hover'] if hover else []),
xrotation=xrotation,
yrotation=yrotation)
return heatmap.opts(frame_width=plot_width,
frame_height=plot_height,
colorbar=colorbar,
cmap=cmap)
class PlotConnCompSettings(MotherMachinePlotter):
init_niblack_k_def = -0.35
maxima_niblack_k_def = -0.45
init_smooth_sigma_def = 3
maxima_smooth_sigma_def = 2
maxima_niblack_window_size_def = 13
init_niblack_window_size_def = 13
param_kdims = [
hv.Dimension("init_niblack_k", range=(-1.5,0.5), step=0.05, default=init_niblack_k_def),
hv.Dimension("maxima_niblack_k", range=(-1.5,0.5), step=0.05, default=maxima_niblack_k_def),
hv.Dimension("init_smooth_sigma", range=(0,10), step=0.5, default=init_smooth_sigma_def),
hv.Dimension("maxima_smooth_sigma", range=(0,10), step=0.5, default=maxima_smooth_sigma_def),
hv.Dimension("init_niblack_window_size", range=(3,35), step=2, default=init_niblack_window_size_def),
hv.Dimension("maxima_niblack_window_size", range=(3,35), step=2, default=maxima_niblack_window_size_def)
hv.Dimension("flip_trenches", range=(0,1), step=1, default=0)
]
@staticmethod
def extract_image(t_frame,lane_num,pos_num, extract_conn_comp_func,MotherMachineFile,**kwargs):
#static is required for dask to pickle it !!!!
s_file = MotherMachineFile(lane_num, pos_num, t_frame)
img = s_file.getImage()
conn_comp = extract_conn_comp_func(img,**kwargs)
return img, conn_comp
def __init__(self, lane_indcs, pos_indcs, tindcs, MotherMachineFile, extract_conn_comp_func, client, has_two_lanes=False):
MotherMachinePlotter.__init__(self, lane_indcs, pos_indcs, tindcs, MotherMachineFile)
self.extract_conn_comp_func = extract_conn_comp_func
self.client = client
if has_two_lanes:
self.cmap = fire[1:]*2
else:
self.cmap = fire[1:]
basic_prop_dict = dict(width=self.img_cropped_width, height=self.img_cropped_height,
fontsize={'title':0, 'xlabel':0, 'ylabel':0, 'ticks':0})
img_prop_dict = basic_prop_dict.copy()
img_prop_dict['cmap'] = self.cmap
self.img_prop_dict = img_prop_dict
self.rgb_prop_dict = basic_prop_dict.copy()
self.layout_prop_dict = basic_prop_dict.copy()
def construct_image_arrays(self):
# need to make copies to make dask happy
extract_conn_comp_func = self.extract_conn_comp_func
lane_num = self.recC.lane_num.copy()
pos_num = self.recC.pos_num.copy()
MotherMachineFile = self.MotherMachineFile
kwargs = self.recC.kwargs.copy()
tindcs = self.tindcs.copy()
g_image = lambda t_frame: PlotConnCompSettings.extract_image(t_frame,lane_num, pos_num,
extract_conn_comp_func,
MotherMachineFile,**kwargs)
img0, conn_comp0 = g_image(self.recC.t_frame)
fut = self.client.map(g_image, tindcs)
return fut, img0, conn_comp0
def show_images(self,lane_num, pos_num, t_frame, *args):
keys = [pk.label for pk in PlotConnCompSettings.param_kdims]
kwargs = dict(zip(keys,args))
# only re-calculate if something other than t_frame is called
if self.recC.has_changed(lane_num, pos_num, **kwargs):
self.recC.just_changed = True
self.recC.update(lane_num, pos_num, t_frame, **kwargs)
fut, img0, conn_comp0 = self.construct_image_arrays()
self.recC.fut = fut
else:
self.recC.just_changed = False
# simply accelerates first value change
if self.recC.just_changed:
ri = img0
cc = conn_comp0
else:
ri, cc = self.recC.fut[np.where(self.tindcs == t_frame)[0][0]].result()
ims = (plot_raw_image(ri) + plot_conn_comp(cc,cmap=self.cmap)).cols(1)
ims = ims.options({'RGB':self.rgb_prop_dict,'Image':self.img_prop_dict,'Layout':self.layout_prop_dict})
return ims
def plot(self):
kdims = self.indcs_kdims + PlotConnCompSettings.param_kdims
dmap = hv.DynamicMap(self.show_images, kdims = kdims)
dmap = dmap.redim.range(y=(0.5-self.scale_factor_h,0.5),x=(-0.5,-0.5+self.scale_factor_w))
return dmap
def heatmap(adata,
genes,
groups=None,
compare='genes',
agg_fns=['mean', 'var'],
use_raw=False,
order_keys=[],
hover=True,
show_highlight=False,
show_scatter=False,
subsample=None,
keep_frac=0.2,
seed=None,
xrotation=90,
yrotation=0,
colorbar=True,
cont_cmap=None,
height=200,
width=600,
save=None,
**scatter_kwargs):
'''
Plot a heatmap with groups selected from a drop-down menu.
If `show_highlight=True` and `show_scatterplot=True`, additional
interaction occurrs when clicking on the highlighted heatmap.
Params
-------
adata: anndata.AnnData
adata object
genes: List[Str]
genes in `adata.var_names`
groups: List[Str], optional (default: `None`)
categorical observation in `adata.obs`,
if `None`, get all groups from `adata.obs`
compare: Str, optional (default: `'genes'`)
only used when `show_scatterplot=True`,
creates a drop-down:
if `'genes'`:
drow-down menu will contain values from `genes` and clicking
a gene in highlighted heatmap will plot scatterplot of the 2
genes with groups colored in
if `'basis'`:
drow-down menu will contain available bases and clicking
a gene in highlighted heatmap will plot the gene in the selected
embedding with its expression colored in
if `'order'`:
drop-down menu will contain values from `order_keys`,
and clicking on a gene in highlighted heatmap will plot its expression
in selected order
agg_fns: List[Str], optional (default: `['mean', 'var']`
names of pandas' aggregation functions, such `'min'`, ...
the first function specified is mapped to colors
use_raw: Bool, optional (default: `False`)
whether to use `.raw` for gene expression
order_keys: List[Str], optional (default: `None`)
keys in `adata.obs`, used when `compare='order'`
hover: Bool, optional (default: `True`)
whether to display hover information over the heatmap
show_highlight: Bool, optional (default: `False`)
whether to show when using boxselect
show_scatter: Bool, optional (default: `False`)
whether to show a scatterplot,
if `True`, overrides `show_highlight=False`
subsample: Str, optional (default: `'decimate'`)
subsampling strategy for large data
possible values are `None, 'none', 'decimate'`
keep_frac: Float, optional (default: `0.2`)
fraction of cells to keep, used when `subsample='decimate'`
seed: Union[Float, NoneType], optional (default: `None`)
random seed, used when `subsample='decimate'`
xrotation: Int, optional (default: `90`)
rotation of labels on x-axis
yrotation: Int, optional (default: `0`)
rotation of labels on y-axis
colorbar: Bool, optional (default: `True`)
whether to show colorbar
cont_cmap: Union[List[Str], NoneType], optional (default, `None`)
colormap of the heatmap,
if `None`, use `Viridis256`
height: Int, optional (default: `200`)
height of the heatmap
width: Int, optional (default: `600`)
width of the heatmap
save: Union[os.PathLike, Str, NoneType], optional (default: `None`)
path where to save the plot
**scatter_kwargs:
additional argument for `ipl.experimental.scatter`,
only used when `show_scatter=True`
Returns
-------
holoviews plot
'''
def _highlight(group, index):
original = hm[group]
if not index:
return original
return original.iloc[sorted(index)]
def _scatter(group, which, gwise, x, y):
indices = adata.obs[group] == y if gwise else np.isin(
adata.obs[group], highlight[group].data['y'])
# this is necessary
indices = np.where(indices)[0]
if is_ordered:
scatter_kwargs['order_key'] = which
x, y = None, x
elif f'X_{which}' in adata.obsm:
group = x
x, y = which, which
else:
x, y = x, which
return scatter2(adata,
x=x,
y=y,
color=group,
indices=indices,
**scatter_kwargs).opts(axiswise=True, framewise=True)
assert keep_frac >= 0 and keep_frac <= 1, f'`keep_perc` must be in interval `[0, 1]`, got `{keep_frac}`.'
assert subsample in (None, 'none','decimate'), f'Invalid subsampling strategy `{subsample}`. ' \
'Possible values are `None, \'none\', \'decimate\'`.'
assert compare in (
'genes', 'basis', 'order'
), f'`compare` must be one of `\'genes\', \'basis\', \'order\'`.'
if cont_cmap is None:
cont_cmap = Viridis256
is_ordered = False
scatter_kwargs['use_original_limits'] = True
scatter_kwargs['subsample'] = None
if 'plot_width' not in scatter_kwargs:
scatter_kwargs['plot_width'] = 300
if 'plot_height' not in scatter_kwargs:
scatter_kwargs['plot_height'] = 300
if groups is not None:
assert len(groups) > 0, f'Number of groups `> 1`.'
else:
groups = [k for k in adata.obs.keys() if is_categorical(adata.obs[k])]
kdims = [hv.Dimension('Group', values=groups, default=groups[0])]
hm = hv.DynamicMap(lambda g: _heatmap(adata,
genes,
agg_fns=agg_fns,
group=g,
hover=hover,
use_raw=use_raw,
cmap=cont_cmap,
xrotation=xrotation,
yrotation=yrotation,
colorbar=colorbar),
kdims=kdims).opts(frame_height=height,
frame_width=width)
if not show_highlight and not show_scatter:
return hm
highlight = hv.DynamicMap(_highlight,
kdims=kdims,
streams=[Selection1D(source=hm)])
if not show_scatter:
return (hm + highlight).cols(1)
if compare == 'basis':
basis = [b.lstrip('X_') for b in adata.obsm.keys()]
kdims += [hv.Dimension('Components', values=basis, default=basis[0])]
elif compare == 'genes':
kdims += [hv.Dimension('Genes', values=genes, default=genes[0])]
else:
is_ordered = True
k = scatter_kwargs.pop('order_key', None)
assert k is not None or order_keys != [], f'No order keys specified.'
if k is not None and k not in order_keys:
order_keys.append(k)
for k in order_keys:
assert k in adata.obs, f'Order key `{k}` not found in `adata.obs`.'
kdims += [hv.Dimension('Order', values=order_keys)]
kdims += [
hv.Dimension('Groupwise',
type=bool,
values=[True, False],
default=True)
]
scatter_stream = hv.streams.Tap(source=highlight,
x=genes[0],
y=adata.obs[groups[0]].values[0])
scatter = hv.DynamicMap(_scatter, kdims=kdims, streams=[scatter_stream])
if subsample == 'decimate':
scatter = decimate(scatter,
max_samples=int(adata.n_obs * keep_frac),
streams=[hv.streams.RangeXY(transient=True)],
random_seed=seed)
plot = (hm + highlight + scatter).cols(1)
if save is not None:
hv.rendered('bokeh').save(plot, save)
return plot
def show_time_range(st, run_id, t0, dt=10):
from functools import partial
import numpy as np
import pandas as pd
import holoviews as hv
from holoviews.operation.datashader import datashade, dynspread
hv.extension('bokeh')
import strax
import gc
# Somebody thought it was a good idea to call gc.collect explicitly somewhere in holoviews
# This makes dynamic PMT maps super slow
# Until I trace the offender:
gc.collect = lambda *args, **kwargs: None
# Custom wheel zoom tool that only zooms in time
from bokeh.models import WheelZoomTool
time_zoom = WheelZoomTool(dimensions='width')
# Get ADC->pe multiplicative conversion factor
from pax.configuration import load_configuration
from pax.dsputils import adc_to_pe
pax_config = load_configuration('XENON1T')["DEFAULT"]
to_pe = np.array(
[adc_to_pe(pax_config, ch) for ch in range(pax_config['n_channels'])])
tpc_r = pax_config['tpc_radius']
# Get locations of PMTs
r = []
for q in pax_config['pmts']:
r.append(
dict(x=q['position']['x'],
y=q['position']['y'],
i=q['pmt_position'],
array=q.get('array', 'other')))
f = 1.08
pmt_locs = pd.DataFrame(r)
records = st.get_array(run_id,
'raw_records',
time_range=(t0, t0 + int(1e10)))
# TOOD: don't reprocess, just load...
hits = strax.find_hits(records)
peaks = strax.find_peaks(hits,
to_pe,
gap_threshold=300,
min_hits=3,
result_dtype=strax.peak_dtype(n_channels=260))
strax.sum_waveform(peaks, records, to_pe)
# Integral in pe
areas = records['data'].sum(axis=1) * to_pe[records['channel']]
def normalize_time(t):
return (t - records[0]['time']) / 1e9
# Create dataframe with record metadata
df = pd.DataFrame(
dict(area=areas,
time=normalize_time(records['time']),
channel=records['channel']))
# Convert to holoviews Points
points = hv.Points(
df,
kdims=[
hv.Dimension('time', label='Time', unit='sec'),
hv.Dimension('channel', label='PMT number', range=(0, 260))
],
vdims=[
hv.Dimension(
'area',
label='Area',
unit='pe',
# range=(0, 1000)
)
])
def pmt_map(t_0, t_1, array='top', **kwargs):
# Compute the PMT pattern (fast)
ps = points[(t_0 <= points['time']) & (points['time'] < t_1)]
areas = np.bincount(ps['channel'],
weights=ps['area'],
minlength=len(pmt_locs))
# Which PMTs should we include?
pmt_mask = pmt_locs['array'] == array
d = pmt_locs[pmt_mask].copy()
d['area'] = areas[pmt_mask]
# Convert to holoviews points
d = hv.Dataset(d,
kdims=[
hv.Dimension('x',
unit='cm',
range=(-tpc_r * f, tpc_r * f)),
hv.Dimension('y',
unit='cm',
range=(-tpc_r * f, tpc_r * f)),
hv.Dimension('i', label='PMT number'),
hv.Dimension('area', label='Area', unit='PE')
])
return d.to(hv.Points,
vdims=['area', 'i'],
group='PMTPattern',
label=array.capitalize(),
**kwargs).opts(plot=dict(color_index=2,
tools=['hover'],
show_grid=False),
style=dict(size=17, cmap='magma'))
def pmt_map_range(x_range, array='top', **kwargs):
# For use in dynamicmap with streams
if x_range is None:
x_range = (0, 0)
return pmt_map(x_range[0], x_range[1], array=array, **kwargs)
xrange_stream = hv.streams.RangeX(source=points)
# TODO: weigh by area
def channel_map():
return dynspread(
datashade(
points, y_range=(0, 260),
streams=[xrange_stream])).opts(plot=dict(
width=600,
tools=[time_zoom, 'xpan'],
default_tools=['save', 'pan', 'box_zoom', 'save', 'reset'],
show_grid=False))
def plot_peak(p):
# It's better to plot amplitude /time than per bin, since
# sampling times are now variable
y = p['data'][:p['length']] / p['dt']
t_edges = np.arange(p['length'] + 1, dtype=np.int64)
t_edges = t_edges * p['dt'] + p['time']
t_edges = normalize_time(t_edges)
# Correct step plotting from Knut
t_ = np.zeros(2 * len(y))
y_ = np.zeros(2 * len(y))
t_[0::2] = t_edges[0:-1]
t_[1::2] = t_edges[1::]
y_[0::2] = y
y_[1::2] = y
c = hv.Curve(dict(time=t_, amplitude=y_),
kdims=points.kdims[0],
vdims=hv.Dimension('amplitude',
label='Amplitude',
unit='PE/ns'),
group='PeakSumWaveform')
return c.opts(
plot=dict( # interpolation='steps-mid',
# default_tools=['save', 'pan', 'box_zoom', 'save', 'reset'],
# tools=[time_zoom, 'xpan'],
width=600,
shared_axes=False,
show_grid=True),
style=dict(color='b')
# norm=dict(framewise=True)
)
def peaks_in(t_0, t_1):
return peaks[(normalize_time(peaks['time'] +
peaks['length'] * peaks['dt']) > t_0)
& (normalize_time(peaks['time']) < t_1)]
def plot_peaks(t_0, t_1, n_max=10):
# Find peaks in this range
ps = peaks_in(t_0, t_1)
# Show only the largest n_max peaks
if len(ps) > n_max:
areas = ps['area']
max_area = np.sort(areas)[-n_max]
ps = ps[areas >= max_area]
return hv.Overlay(items=[plot_peak(p) for p in ps])
def plot_peak_range(x_range, **kwargs):
# For use in dynamicmap with streams
if x_range is None:
x_range = (0, 10)
return plot_peaks(x_range[0], x_range[1], **kwargs)
top_map = hv.DynamicMap(partial(pmt_map_range, array='top'),
streams=[xrange_stream])
bot_map = hv.DynamicMap(partial(pmt_map_range, array='bottom'),
streams=[xrange_stream])
waveform = hv.DynamicMap(plot_peak_range, streams=[xrange_stream])
layout = waveform + top_map + channel_map() + bot_map
return layout.cols(2)
def GenMap(sonar_data, include_nearest_landmark=True):
gps_longitude_range = sonar_data.longitude.min().values.item(
), sonar_data.longitude.max().values.item()
gps_latitude_range = sonar_data.latitude.min().values.item(
), sonar_data.latitude.max().values.item()
lon_min = gps_longitude_range[0]
lon_max = gps_longitude_range[1]
lat_min = gps_latitude_range[0]
lat_max = gps_latitude_range[1]
nearest_landmark_debug = ''
if include_nearest_landmark:
lon_min, lon_max, lat_min, lat_max, nearest_landmark_debug = AdjustToIncludeNearestLandmark(
lon_min, lon_max, lat_min, lat_max)
if nearest_landmark_debug != '':
nearest_landmark_debug = '. With nearest landmark: ' + nearest_landmark_debug
logger.debug('Generating graph inside area: (%s, %s) - (%s, %s)%s',
lon_min, lat_min, lon_max, lat_max, nearest_landmark_debug)
tiles = holoviews.Tiles(
'https://maps.wikimedia.org/osm-intl/{Z}/{X}/{Y}@2x.png',
name="Wikipedia")
tiles = tiles.opts(width=600, height=600)
# Adjust the framing of the tiles to show the path area and not the entire world
# From: https://examples.pyviz.org/nyc_taxi/nyc_taxi.html
#left, bottom = datashader.geo.lnglat_to_meters(lon_min, lat_min)
#right, top = datashader.geo.lnglat_to_meters(lon_max, lat_max)
# @todo cleanup, we are now using web mercator metres everywhere
left, bottom = (lon_min, lat_min)
right, top = (lon_max, lat_max)
# http://holoviews.org/_modules/holoviews/core/dimension.html
tiles = tiles.redim(x=holoviews.Dimension('x', range=(left, right)),
y=holoviews.Dimension('y', range=(bottom, top)))
logger.debug('left:%s right:%s bottom:%s top:%s', left, right, bottom, top)
#import code
#code.interact(local=dict(globals(), **locals()))
channel = sonar_data.sel(channel=sonar.lowrance_log_parser.PRIMARY)
latitude_arr = channel.latitude.values
longitude_arr = channel.longitude.values
frame_index_arr = channel.frame_index.values
depth_arr = channel.water_depth.values
#logger.info('Creating vector field for the path')
#x_arr = []
#y_arr = []
#angle_arr = []
#magnitude_arr = []
#for i in range(0, len(frame_index_arr)):
# x,y = datashader.geo.lnglat_to_meters(longitude_arr[i], latitude_arr[i])
# angle = channel.heading.values[i]
# magnitude = channel.speed_gps.values[i]
# x_arr.append(x)
# y_arr.append(y)
# angle_arr.append(angle)
# magnitude_arr.append(magnitude)
## http://holoviews.org/reference/elements/bokeh/VectorField.html
#path = holoviews.VectorField((x_arr, y_arr, angle_arr, magnitude_arr))
#path = path.opts(magnitude='Magnitude', color='Magnitude')
#logger.info('Done')
path_points = []
for i in range(0, len(frame_index_arr)):
# @todo cleanup
#path_points.append(datashader.geo.lnglat_to_meters(longitude_arr[i], latitude_arr[i]))
path_points.append((longitude_arr[i], latitude_arr[i], depth_arr[i]))
path = holoviews.Path(path_points,
vdims='depth',
kdims=[
holoviews.Dimension('easting'),
holoviews.Dimension('northing')
])
path = path.opts(color='depth', width=600, height=600, cmap='viridis')
# See https://holoviz.org/tutorial/Composing_Plots.html for the easting/northing plot
# @todo Note: Sometimes the x/y axis in overlay changes to use the path values not the tiles values which is not what we want we want to displat lon/lat not northings eastings
overlay = tiles * path
return overlay
def _plot(self) -> pn.panel:
"""
Represent the initial and final state of the lattice.
Graphical representation of the lattice. The image shows the
initial and final state of the grid (in order to compare how
the network has evolved), as well as the number of speakers
as a function of time. self.track = True is needed to call this
method.
"""
grid_flat = self.memory.reshape(self.memory.shape[0], -1)
speakers_a = (grid_flat == 1).sum(1)
speakers_b = (grid_flat == -1).sum(1)
speakers_ab = (grid_flat == 0).sum(1)
total = speakers_a + speakers_b + speakers_ab == (self.width * self.height) * np.ones(
len(self.memory),
)
if not np.all(total):
raise ValueError(
"The total number of speakers does not correspond to the lattice size!",
)
# Plots
colors = ["navy", "white", "red"]
data_start = self.memory[0]
data_end = self.grid.data
grid_start = {
"xdata": np.arange(1, data_start.shape[1] + 1),
"ydata": np.arange(1, data_start.shape[0] + 1),
"zdata": data_start,
}
grid_end = {
"xdata": np.arange(1, data_end.shape[1] + 1),
"ydata": np.arange(1, data_end.shape[0] + 1),
"zdata": data_end,
}
plot_start = hv.Image(
grid_start,
kdims=["xdata", "ydata"],
vdims=hv.Dimension("zdata", range=(-1, 1)),
label="Initial grid",
)
plot_end = hv.Image(
grid_end,
kdims=["xdata", "ydata"],
vdims=hv.Dimension("zdata", range=(-1, 1)),
label="Final grid",
)
plot_curvea = hv.Curve(speakers_a, label="Speakers A").opts(color="red")
plot_curveb = hv.Curve(speakers_b, label="Speakers B").opts(color="navy")
plot_curveab = hv.Curve(speakers_ab, label="Speakers AB").opts(color="gray")
# Compositions
grids = plot_start + plot_end
lines = plot_curvea * plot_curveb * plot_curveab
layout = grids + lines
# Options
layout.opts(
opts.Image(
invert_yaxis=True,
cmap=colors,
colorbar=True,
width=350,
labelled=[],
colorbar_opts={
"title": "Languages",
"title_text_align": "left",
"major_label_overrides": {-1: "B", 0: "AB", 1: "A"},
"ticker": FixedTicker(ticks=[-1, 0, 1]),
"major_label_text_align": "right",
},
),
opts.Curve(xlabel="Iterations", ylabel="Number of speakers", width=700),
)
return display(pn.Column(pn.Row(plot_start, plot_end), lines))
def covar_plot(x_var, y_var, x_range, y_range):
cvs = ds.Canvas(plot_width=plot_width,
plot_height=plot_height,
x_range=x_range,
y_range=y_range)
agg = cvs.points(data, x_var, y_var, ds.count(y_var))
return hv.Image(agg).opts(colorbar=False,
cmap='Blues',
logz=True,
width=int(plot_width * zoom_multiplier),
height=int(plot_height * zoom_multiplier))
plot_dict = hv.OrderedDict([])
for yy in range(0, len(y_var)):
# y_range = (np.nanmin(data.loc[:, yy]),np.nanmax(data.loc[:, yy]))
y_range = (np.nanquantile(data.loc[:, y_var[yy]], .001),
np.nanquantile(data.loc[:, y_var[yy]], .999))
for xx in range(yy, len(x_var)):
# x_range = (np.nanmin(data.loc[:, x_var]),np.nanmax(data.loc[:, x_var]))
x_range = (np.nanquantile(data.loc[:, x_var[xx]], .001),
np.nanquantile(data.loc[:, x_var[xx]], .999))
plot_dict[(y_var[yy], x_var[xx])] = covar_plot(y_var[yy], x_var[xx],
y_range, x_range)
kdims = [hv.Dimension(('y_var', 'yy')), hv.Dimension(('x_var', 'xx'))]
holomap = hv.HoloMap(plot_dict, kdims=kdims)
# holomap.opts(opts.Curve(width=600))
grid = hv.GridSpace(holomap, sort=False)
grid
def view(self, *args, **kwargs):
return self.element(
self.amplitude * np.sin(np.linspace(0, np.pi * self.frequency)),
vdims=[hv.Dimension('y',
range=(-5, 5))])(style=dict(color=self.color))
def plot_grids(data,fields,plot_type):
if len(data) == 2:
ds,grid_ds = data
if fields == 'Sources':
print('Plotting Sources..')
event_times_seconds = xr.DataArray(ds.event_time.values.astype('<m8[s]')/86400,dims='number_of_events')
event_times_seconds.shape,ds.event_time.shape
dss = ds.copy()
dss['times'] = event_times_seconds.astype(int)
xr_dataset = gv.Dataset(hv.Points((dss.event_longitude.values, dss.event_latitude.values,dss.times.values),
kdims=[hv.Dimension('x'),hv.Dimension('y')],vdims=['time']))
fig = gv.tile_sources.Wikipedia.opts(width=900, height=900) * xr_dataset.to.points(['x','y']).opts(opts.Points(color='time'))
else:
print(plot_type)
z = grid_ds.flash_extent_density
x = grid_ds.grid_longitude.values
y = grid_ds.grid_latitude.values
# get xarray dataset, suited for handling raster data in pyviz
xr_dataset = gv.Dataset(hv.Image((x, y, np.log10(z.mean(axis=0))), bounds=(x.min(),y.min(),x.max(),y.max()),
kdims=[hv.Dimension('x'), hv.Dimension('y')], datatype=['grid']))
# create contours from image
gv.FilledContours(xr_dataset)
fig = gv.tile_sources.Wikipedia.opts(width=900, height=900) * xr_dataset.to.image(['x', 'y']).opts(cmap='cubehelix', alpha=0.8)#gv.FilledContours(xr_dataset).opts(cmap='viridis', alpha=0.5)
responsive = pn.pane.HoloViews(fig, config={'responsive': True})
return(responsive)
else:
grid_ds = data
print('No Data Processed')
if fields == 'Sources':
xr_dataset = gv.Dataset(hv.Points((np.repeat(-101.1,1), np.repeat(32,2),np.repeat(10,2)),
kdims=[hv.Dimension('x'),hv.Dimension('y')],vdims=['time']))
fig = gv.tile_sources.Wikipedia.opts(width=900, height=900) * xr_dataset.to.points(['x','y']).opts(opts.Points(color='time'))
elif plot_type == 'Grids':
z = np.random.randn(10,(100,100))
x = np.repeat(-101.1,100)
y = np.repeat(32,100)
# get xarray dataset, suited for handling raster data in pyviz
xr_dataset = gv.Dataset(hv.Image((x, y, np.log10(z)), bounds=(x.min(),y.min(),x.max(),y.max()),
kdims=[hv.Dimension('x'), hv.Dimension('y')], datatype=['grid']))
# create contours from image
gv.FilledContours(xr_dataset)
fig = gv.tile_sources.Wikipedia.opts(width=900, height=900) * xr_dataset.to.image(['x', 'y']).opts(cmap='cubehelix', alpha=0.8)#gv.FilledContours(xr_dataset).opts(cmap='viridis', alpha=0.5)
responsive = pn.pane.HoloViews(fig, config={'responsive': True})
return(responsive)
cmap='Wistia',
color_levels=3)
im6 = hv.operation.contours(hv.Image(data,
**hvargs,
group=appl,
label='No. of stages, 90% CCR',
vdims='nstages90'),
levels=[1.0, 2.0, 3.0],
filled=True).options(**opts,
cmap='Wistia',
color_levels=3)
image = im1 + im2 + im3 + im4 + im5 + im6
image = image.cols(2)
return image #im1+im2
# Define the dimension over which to make the HoloMap
kdim = hv.Dimension(('appl', 'Application'), default=appl_list[0])
# Make the HoloMap and collate it
hmap = hv.HoloMap({appl: get_baseplot(appl) for appl in appl_list}, kdims=kdim)
hmap = hmap.collate()
#hmap = hv.DynamicMap(get_baseplot, kdims=kdim).redim.values(appl=appl_list)
#hmap.options(framewise=True)
# Render this to a file
renderer = hv.renderer('bokeh')
renderer.save(hmap, 'test')
#plot = renderer.get_plot(hmap).state
#io.output_file("test.html",mode='inline')
#io.show(plot)
b_est_vector = np.linspace(b_est_min,b_est_max,n_est)
# create matrix with squared error depending on a_est and b_est in the goal to draw contour lines
matrix = np.zeros((n_est,n_est))
for i in range(n_est):
for j in range(n_est):
matrix[i,j] = predictions(a_est_vector[j], b_est_vector[-i-1])[2] # these are the sum of squares
bounds = (a_est_min,b_est_min,a_est_max,b_est_max) # bounds for the coming plot
a_est_grid, b_est_grid = np.meshgrid(a_est_vector, b_est_vector) # create input grid
hv_img = hv.Image(a_est_grid + b_est_grid, ['a estimate','b estimate'], bounds = bounds) # initialize plot; hv.Image content is added in next line
hv_img.data = matrix # overwrite data with data we want
contour_levels=50
hv_contours = hv_img * hv.operation.contours(hv_img, levels=contour_levels)
# set up dimensions for kdims for coming dynamic maps
dim_a_est = hv.Dimension('a_est', range=(a_est_min, a_est_max), default=1)
dim_b_est = hv.Dimension('b_est', range=(b_est_min, b_est_max), default=-3.0)
# dynamic maps
dmap_coords_a_b_est = hv.DynamicMap(hv_point_est, kdims=[dim_a_est,dim_b_est])
dmap_squares = hv.DynamicMap(hv_squares, kdims=[dim_a_est,dim_b_est])
dmap_line_est = hv.DynamicMap(hv_line_est, kdims=[dim_a_est,dim_b_est])
ls_layout = (dmap_squares * dmap_line_est * hv_points.options(marker='o', size=5)).redim.range(x=(-0.5,12.5), y=(-3,10))
# plot styling options
color_palette = 'PuRd'
col_1 = bp.all_palettes[color_palette][9][1]
col_2 = bp.all_palettes[color_palette][9][2]
col_3 = bp.all_palettes[color_palette][9][2]
cmap_custom = hv.plotting.util.polylinear_gradient(['#000000', '#000000'], 2) # color map in only black
def plot_policy_gantt_chart(
policies,
effects=False,
colors="categorical",
fig_kwargs=None,
):
"""Plot a Gantt chart of the policies."""
if fig_kwargs is None:
fig_kwargs = {}
fig_kwargs = {**DEFAULT_FIGURE_KWARGS, **fig_kwargs}
if isinstance(policies, dict):
df = (pd.DataFrame(policies).T.reset_index().rename(columns={
"index": "name"
}).astype({
"start": "datetime64",
"end": "datetime64"
}).drop(columns="policy"))
elif isinstance(policies, pd.DataFrame):
df = policies
else:
raise ValueError(
"'policies' should be either a dict or pandas.DataFrame.")
if effects:
effect_kwargs = effects if isinstance(effects, dict) else {}
effects = compute_pseudo_effect_sizes_of_policies(policies=policies,
**effect_kwargs)
effects_s = pd.DataFrame([{
"policy": name,
"effect": effects[name]["mean"]
} for name in effects]).set_index("policy")["effect"]
df = df.merge(effects_s, left_on="name", right_index=True)
df["alpha"] = (1 - df["effect"] + 0.1) / 1.1
else:
df["alpha"] = 1
df = df.reset_index()
df = _complete_dates(df)
df = _add_color_to_gantt_groups(df, colors)
df = _add_positions(df)
hv.extension("bokeh", logo=False)
segments = hv.Segments(
df,
[
hv.Dimension("start", label="Date"),
hv.Dimension("position", label="Affected contact model"),
"end",
"position",
],
)
y_ticks_and_labels = list(zip(*_create_y_ticks_and_labels(df)))
tooltips = [("Name", "@name")]
if effects:
tooltips.append(("Effect", "@effect"))
hover = HoverTool(tooltips=tooltips)
gantt = segments.opts(
color="color",
alpha="alpha",
tools=[hover],
yticks=y_ticks_and_labels,
**fig_kwargs,
)
return gantt
def rs_fMRI_plots(SBJ, RUN, WL_sec, corr_range):
# Load rs fMRI data
# -----------------
file_name = SBJ + '_fanaticor_Craddock_T2Level_0200_wl' + str(
WL_sec).zfill(
3) + 's_ws002s_' + RUN + '_PCA_vk97.5.swcorr.pkl' # Data file name
data_path = osp.join('/data/SFIM_Vigilance/PRJ_Vigilance_Smk02/PrcsData',
SBJ, 'D02_Preproc_fMRI', file_name) # Path to data
data_df = pd.read_pickle(data_path).T # Read data into pandas data frame
num_samp = data_df.shape[0] # Save number of samples as a varable
# Load sleep segmenting data
# --------------------------
seg_path = osp.join(PRJDIR, 'Data', 'Samika_DSet02', 'Sleep_Segments',
SBJ + '_' + RUN + '_WL_' + str(WL_sec) +
'sec_Sleep_Segments.pkl') # Path to segment data
seg_df = pd.read_pickle(seg_path) # Load segment data
# Compute correlation and distance matrix
# ---------------------------------------
data_corr = np.corrcoef(data_df) # Correlation matrix
data_dist = pairwise_distances(data_df,
metric='euclidean') # Distance matrix
# Compute distribution of correlation and distance matrix
# -------------------------------------------------------
triangle = np.mask_indices(num_samp, np.triu,
k=1) # Top triangle mask for matricies
corr_freq, corr_edges = np.histogram(
np.array(data_corr)[triangle], 100
) # Compute histogram of top triangle of correlation matrix (100 bars)
dist_freq, dist_edges = np.histogram(
np.array(data_dist)[triangle],
100) # Compute histogram of top triangle of distance matrix (100 bars)
# Create sleep segments plots
# ---------------------------
sleep_color_map = {
'Wake': 'orange',
'Stage 1': 'yellow',
'Stage 2': 'green',
'Stage 3': 'blue',
'Undetermined': 'gray'
} # Color key for sleep staging
seg_x = hv.Segments(seg_df, [
hv.Dimension('start', range=(-10, num_samp - 1.5)),
hv.Dimension('start_event', range=(-5, num_samp - 1.5)), 'end',
'end_event'
], 'stage').opts(color='stage',
cmap=sleep_color_map,
line_width=7,
show_legend=True) # x axis segments
seg_y = hv.Segments(seg_df, [
hv.Dimension('start_event', range=(-10, num_samp - 1.5)),
hv.Dimension('start', range=(-5, num_samp - 1.5)), 'end_event', 'end'
], 'stage').opts(color='stage',
cmap=sleep_color_map,
line_width=7,
show_legend=False) # y axis segments
seg_plot = (seg_x * seg_y).opts(xlabel=' ', ylabel=' ',
show_legend=False) # All segments
# Create matrix and histogram plots
# ---------------------------------
# raterize() fucntion used for big data set
corr_img = rasterize(
hv.Image(np.rot90(data_corr),
bounds=(-0.5, -0.5, num_samp - 1.5, num_samp - 1.5)).opts(
cmap='viridis', colorbar=True,
title='Correlation Matrix')).redim.range(z=corr_range)
dist_img = rasterize(
hv.Image(np.rot90(data_dist),
bounds=(-0.5, -0.5, num_samp - 1.5,
num_samp - 1.5)).opts(cmap='viridis',
colorbar=True,
title='Distance Matrix'))
corr_his = rasterize(
hv.Histogram(
(corr_edges, corr_freq)).opts(xlabel='Correlation',
height=300,
width=400,
title='Correlation Histogram'))
dist_his = rasterize(
hv.Histogram((dist_edges, dist_freq)).opts(xlabel='Distance',
height=300,
width=400,
title='Distance Histogram'))
corr_img_wseg = (corr_img * seg_plot).opts(
width=600, height=300, legend_position='right'
) # Overlay sleep segemnt plot with correlation matrix
dist_img_wseg = (dist_img * seg_plot).opts(
width=600, height=300, legend_position='right'
) # Overlay sleep segemnt plot with distance matrix
dash = (corr_img_wseg + corr_his + dist_img_wseg + dist_his).opts(
opts.Layout(shared_axes=False)).cols(2) # Dashboard of all plots
return dash
def create_doc(doc, data_dir):
def get_spectrogram(s0=None,
s1=None,
size=1024,
overlap=1. / 8,
zfill=1,
mode='psd'):
s_size = np.dtype(np.complex64).itemsize
if s1 is None and s0 is None:
ds = None
elif s1 is None:
ds = None
elif s0 is None:
ds = np.abs(s1)
else:
ds = np.abs(s1 - s0)
if ds is not None and s0 is not None:
flen = getsize(join(data_dir, file))
print("file size: {} bytes".format(flen))
if (ds + s0) * s_size > flen:
ds = flen - s0 * s_size
samples = np.memmap(join(data_dir, file),
dtype='complex64',
mode='r',
offset=s0 * s_size,
shape=(ds, ) if ds else None)
if ds is None:
ds = len(samples)
if ds / size > (res + 0.5) * height:
noverlap = -int(float(size) * float(ds) / size / height / res)
else:
noverlap = size // (1. / overlap)
f, t, S = signal.spectrogram(samples,
samp_rate,
nperseg=size,
nfft=int(
next_power_of_2(size) * int(zfill)),
noverlap=noverlap,
return_onesided=False,
scaling='density',
mode=mode) #
f = fftshift(f)
S = fftshift(S, axes=(0, ))
if mode == 'psd':
S = 10 * np.log10(S)
return f, t, S, samples
def get_spectrogram_img(z_min, z_max, tf_r, zfill, overlap, show_realfreq,
freq_unit, x_range, y_range):
lock.acquire()
show_realfreq = bool(show_realfreq)
hv_image = None
try:
print("y_range:", y_range, type(y_range))
if type(y_range[0]) != float:
if np.issubdtype(y_range[0],
np.datetime64) and time is not None:
y_range = [
(y - np.datetime64(time)
).astype('timedelta64[us]').astype('float') / 1e6
for y in y_range
]
#elif np.issubdtype(y0, np.timedelta64):
# y0, y1 = [y.astype('timedelta64[s]').astype('float') for y in [y0,y1]]
# tranform back to relative frequency if required
print(doc.session_context.show_realfreq, x_range)
last_freq_unit = doc.session_context.freq_unit
x_range = [x * freq_units_names[last_freq_unit] for x in x_range]
if doc.session_context.show_realfreq:
x_range = [(x - freq) for x in x_range]
print(doc.session_context.show_realfreq, "after transform",
x_range)
(x0, x1), (y0, y1) = x_range, y_range
#print("y0 dtype:", y0.dtype)
s0, s1 = sorted([
min(max(int(yr * samp_rate), 0), total_samples)
for yr in [y0, y1]
])
scale = samp_rate / np.abs(x_range[1] - x_range[0])
size = int(width *
scale) # required freq resolution to fulfill zoom level
ds = np.abs(
s1 - s0) # number of samples covered at the current zoom level
if ds / size < height:
size = int(np.sqrt(ds * scale * 10**tf_r))
f, t, S, samples = get_spectrogram(
s0,
s1,
size=size,
overlap=overlap,
mode=mode,
zfill=zfill if size < res * width else 1)
t += max(min(y0, y1), 0)
f_range = (x0 <= f) & (f <= x1)
image = S[f_range, :]
f = f[f_range]
#if ds / size > height:
# image = signal.resample(image, height*2, axis=0)
print(image.shape)
if intp_enabled:
ratio = np.array(image.shape, dtype=np.float) / np.array(
(height * res, width * res))
if np.min(ratio) < 1:
scale = np.max(
np.abs(image)
) # normalization factor for image because rescale needs that
image = rescale(image / scale, 1. / np.min(ratio),
order=1) * scale
f = signal.resample(f, image.shape[0])
t = signal.resample(t, image.shape[1])
print("after resampling: ", image.shape)
del samples
#image = hv.Image(image, bounds=(x0, y0, x1, y1)).redim.range(z=(z_min, z_max)) # TODO get exact values in bounds
if show_realtime and time is not None:
t = np.datetime64(time) + np.array(
t * 1e6).astype('timedelta64[us]')
#else:
# t = (t*1e6) #.astype('timedelta64[us]')
if show_realfreq:
f += freq
f /= freq_units_names[freq_unit]
#image = image.astype('float16') # trying to reduce network bandwidth...
print("image dtype:", image.dtype)
#hv_image = hv.Image(np.flip(image, axis=1), bounds=(min(f), max(t), max(f), min(t))) \
hv_image = hv.Image(xr.DataArray(image, coords=[f,t], dims=['f','t'], name='z'), ['f','t'], 'z') \
.options(
xlabel="Frequency [{}]".format(freq_unit),
ylabel="Time " + '[s]' if not show_realtime else '') \
.redim.range(z=(z_min, z_max))
if doc.session_context.show_realfreq != show_realfreq or doc.session_context.freq_unit != freq_unit:
hv_image = hv_image \
.redim.range(f=(min(f), max(f))) \
.redim.range(t=(min(t), max(t)))
print("redimming axis range")
doc.session_context.show_realfreq = show_realfreq
doc.session_context.freq_unit = freq_unit
except Exception as e:
print("Exception in image generation:", e)
print(traceback.format_exc())
lock.release()
return hv_image
def get_param(name, default, t=str):
try:
args = doc.session_context.request.arguments
if t == str:
p = str(args.get(name)[0], 'utf-8')
else:
p = t(args.get(name)[0])
except:
p = default
return p
time = None
freq = 0
file = basename(get_param('file', '', str))
skip = get_param('skip', 0, int)
keep = get_param('keep', None, int)
# low resolution for raspberry
width, height = get_param('width', 600, int), get_param('height', 500, int)
res = get_param('res', 1.5, float)
ds_enabled = get_param('ds', None, str) # datashade option (default: avg)
intp_enabled = get_param('intp', 0, int) # interpolation enable flap
show_realtime = bool(get_param('rt', 0,
int)) # show real time on vertical axis
mode = get_param('mode', 'psd', str)
t_range = get_param('t', None, str)
f_range = get_param('f', None, str)
if t_range is not None:
try:
parts = t_range.split(",")
if len(parts) == 2:
t_range = list(map(float, parts))
except:
pass
if f_range is not None:
try:
parts = f_range.split(",")
if len(parts) == 2:
f_range = list(map(float, parts))
elif len(parts) == 1:
_f = abs(float(parts[0]))
f_range = (-_f, _f)
except:
pass
if file.endswith(".meta"):
config = ConfigParser()
config.read(join(data_dir, file))
file = splitext(file)[0] + ".raw"
samp_rate = int(config['main']['samp_rate'])
freq = float(config['main']['freq'])
time = datetime.fromtimestamp(float(config['main']['time']))
else:
samp_rate = get_param('samp_rate', 128000, int)
f, t, S, samples = get_spectrogram(s0=skip * samp_rate,
s1=keep * samp_rate if keep else None,
size=width,
mode=mode)
total_samples = len(samples)
del samples
# default is True
if show_realtime and time is not None:
t = np.datetime64(time) + np.array(t).astype('timedelta64[s]')
doc.session_context.show_realfreq = False
doc.session_context.freq_unit = freq_units[1000]
range_stream = RangeXY(x_range=tuple(
x / freq_units_names[doc.session_context.freq_unit]
for x in ((min(f_range), max(f_range)) if f_range else (min(f),
max(f)))),
y_range=(max(t_range), min(t_range)) if t_range else
(max(t), min(t))) # transient=True
z_range = (np.min(S), np.max(S))
z_init = np.percentile(S, (50, 100))
dmap = hv.DynamicMap(
get_spectrogram_img,
streams=[range_stream],
kdims=[
hv.Dimension('z_min', range=z_range, default=z_init[0]),
hv.Dimension('z_max', range=z_range, default=z_init[1]),
hv.Dimension('tf_r',
label='Time-Frequency pixel ratio',
range=(-10., 10.),
default=0.),
hv.Dimension('zfill',
label='Zero-filling factor',
range=(1, 10),
default=2),
hv.Dimension('overlap',
label='Overlap factor',
range=(-1., 1.),
default=1. / 8),
#hv.Dimension('show_realtime', label='Show real time on vertical axis', range=(0,1), default=0),
hv.Dimension('show_realfreq',
label='Show real frequency',
range=(0, 1),
default=int(doc.session_context.show_realfreq)),
hv.Dimension('freq_unit',
label='Frequency unit',
values=list(
map(lambda x: freq_units[x],
sorted(freq_units.keys()))),
default=doc.session_context.freq_unit),
#hv.Dimension('mode', label='Spectrogram mode', values=['psd', 'angle', 'phase', 'magnitude'], default='psd')
]).options(
framewise=True, # ???
) #.redim.range(z=z_init)
#dmap = dmap.opts(opts.Image(height=height, width=width))
if ds_enabled != None:
print("datashade enabled: yes")
if ds_enabled == "" or ds_enabled == "mean":
ds_enabled = dsr.mean
elif ds_enabled == "max":
ds_enabled = dsr.max
else:
print(
"warning: invalid option for datashade. using default value: mean"
)
ds_enabled = dsr.mean
dmap = regrid(dmap,
aggregator=ds_enabled,
interpolation='linear',
upsample=True,
height=height * 2,
width=width * 2) # aggregation=dsr.max
#dmap = dmap.hist(num_bins=150, normed=False)
dmap = dmap.opts(
opts.Image(
cmap='viridis',
framewise=True,
colorbar=True,
height=height,
width=width,
tools=['hover'],
title='{}, {} {} sps'.format(
time.strftime('%Y-%m-%d %H:%M:%S') if time else 'Time unknown',
format_freq(freq), samp_rate)),
opts.Histogram(framewise=False, width=150))
#plot = renderer.get_plot(hist, doc).state
#widget = renderer.get_widget(hist, None, position='right').state
#hvobj = layout([plot, widget])
#plot = layout([renderer.get_plot(hist, doc).state])
#doc.add_root(plot)
doc = renderer.server_doc(dmap, doc=doc)
doc.title = 'Waterfall Viewer'
# ---
import numpy as np
import holoviews as hv
hv.extension('bokeh')
from holoviews import streams, opts, dim
# ## 2D plots with interactive slicing
# ls = np.linspace(0, 10, 200)
xx, yy = np.meshgrid(ls, ls)
bounds = (0, 0, 10, 10) # Coordinate system: (left, bottom, right, top)
energy = hv.Dimension('energy', label='E', unit='MeV')
distance = hv.Dimension('distance', label='d', unit='m')
charge = hv.Dimension('charge', label='Q', unit='pC')
# +
image = hv.Image(np.sin(xx) * np.cos(yy),
bounds=bounds,
kdims=[energy, distance],
vdims=charge)
pointer = streams.PointerXY(x=5, y=5, source=image)
dmap = hv.DynamicMap(lambda x, y: hv.VLine(x) * hv.HLine(y), streams=[pointer])
x_sample = hv.DynamicMap(
lambda x, y: image.sample(energy=x).opts(color='darkred'),
streams=[pointer])
y_sample = hv.DynamicMap(
