def mass_plotting(self, filename):
data_dict = self.data_generation(filename)
self.data_test(data_dict)
name_check = data_dict["gen_info"]["DATA FILES"]
attr_id = name_check[1][4][:-3] + "_" + name_check[2][2]
TOOLS="pan,wheel_zoom,box_zoom,reset, hover, previewsave"
figure_obj = figure(plot_width = 1000, plot_height = 800, y_axis_type = "log",
title = attr_id, tools = TOOLS)
figure_obj.yaxis.axis_label = data_dict["data"][0]["y_unit"]
figure_obj.xaxis.axis_label = data_dict["data"][0]["x_unit"]
hover = figure_obj.select(dict(type = HoverTool))
hover.tooltips=[("Value:", "$top")]
hist, edges = np.histogram(data_dict["data"][0]["y"], bins = data_dict["data"][0]["x"])
source = ColumnDataSource(data = dict(top = hist, left = edges[:-1], right = edges[1:],
x_unit = data_dict["data"][0]["x_unit"], y_unit = data_dict["data"][0]["y_unit"],
edges = edges))
#hist = figure_obj.Histogram(source )
figure_obj.quad(top = "top", bottom = 0, left = "left" , right = "right", source = source)
matplot_button = Button(label = "create matplotlib plot")
matplot_button.on_click(lambda source_list = source:
self.matplotlib_export_ms(source_list))
return Panel(child = hplot(figure_obj, matplot_button), title = attr_id)
def read_file_tab(self):
"""Lets the user choose a data file to read"""
# Drop down list
self.file_select = Select(name='Data files',
value='',
options=[],
title='Data files')
# Status text
self.file_status = Div(text='', width=self.page_width)
# Update the file_select and file_status controls with scan data
self.scan_folder()
# This line is here deliberately. The scan_folder would trigger
# the on-change function and we don't want that first time around.
self.file_select.on_change('value', self.file_changed)
# Re-scan button
file_rescan = Button(label="Rescan folder", button_type="success")
file_rescan.on_click(self.scan_folder)
# Layout
c = column(self.file_select,
self.file_status,
file_rescan)
return Panel(child=c, title="Read from file")
def run(self):
""" Start the CLI logic creating the input source, data conversions,
chart instances to show and all other niceties provided by CLI
"""
try:
self.limit_source(self.source)
children = []
if self.smart_filters:
copy_selection = Button(label="copy current selection")
copy_selection.on_click(self.on_copy)
children.append(copy_selection)
self.chart = create_chart(
self.series, self.source, self.index, self.factories,
self.map_options, children=children, **self.chart_args
)
self.chart.show()
self.has_ranged_x_axis = 'ranged_x_axis' in self.source.columns
self.columns = [c for c in self.source.columns if c != 'ranged_x_axis']
if self.smart_filters:
for chart in self.chart.charts:
chart.source.on_change('selected', self, 'on_selection_changed')
self.chart.session.poll_document(self.chart.doc)
except TypeError:
if not self.series:
series_list = ', '.join(self.chart.values.keys())
print(hm.ERR_MSG_TEMPL % series_list)
raise
if self.sync_with_source:
keep_source_input_sync(self.input, self.update_source, self.last_byte)
class Offsets_Panel(object):
def __init__(self, ut330):
self.ut330 = ut330
self.t_current = TextInput(title="Temperature current")
self.h_current = TextInput(title="Humidity current")
self.p_current = TextInput(title="Pressure current")
self.t_offset = TextInput(title="Temperature offset")
self.h_offset = TextInput(title="Humidity offset")
self.p_offset = TextInput(title="Pressure offset")
self.read_offsets = Button(label='Read offsets')
self.write_offsets = Button(label='Write offsets')
def _layout_(self):
return VBox(HBox(self.t_current, self.t_offset, width=500),
HBox(self.h_current, self.h_offset, width=500),
HBox(self.p_current, self.p_offset, width=500),
HBox(self.read_offsets, self.write_offsets, width=500))
def panel(self):
return Panel(child=self._layout_(), title="Offsets")
def _read_(self):
offsets = self.ut330.read_offsets()
self.t_current.value = str(offsets['temperature'])
self.h_current.value = str(offsets['humidity'])
self.p_current.value = str(offsets['pressure'])
self.t_offset.value = str(offsets['temperature offset'])
self.h_offset.value = str(offsets['humidity offset'])
self.p_offset.value = str(offsets['pressure offset'])
def _write_(self):
offsets = {'temperature offset': float(self.t_offset.value),
'humidity offset': float(self.h_offset.value),
'pressure offset': float(self.p_offset.value)}
self.ut330.write_offsets(offsets)
def callbacks(self):
self.read_offsets.on_click(self._read_)
self.write_offsets.on_click(self._write_)
def device_read(self):
self._read_()
def make_layout():
plot, source = make_plot()
columns = [
TableColumn(field="dates", title="Date"),
TableColumn(field="downloads", title="Downloads"),
]
data_table = DataTable(source=source, columns=columns, width=400, height=400)
button = Button(label="Randomize data", type="success")
button.on_click(click_handler)
buttons = VBox(children=[button])
vbox = VBox(children=[buttons, plot, data_table])
return vbox
def make_layout():
plot, source = make_plot()
columns = [
TableColumn(field="dates", title="Date", editor=DateEditor(), formatter=DateFormatter()),
TableColumn(field="downloads", title="Downloads", editor=IntEditor()),
]
data_table = DataTable(source=source, columns=columns, width=400, height=400, editable=True)
button = Button(label="Randomize data", button_type="success")
button.on_click(click_handler)
buttons = WidgetBox(children=[button],width=800)
column = Column(children=[buttons, plot, data_table])
return column
def time_tab(self):
"""The date and time setting and getting tab"""
self.time_status = Div(text="", width=self.page_width)
time_connect = Button(label='Connect to UT330',
button_type="success")
time_disconnect = Button(label='Disconnect from UT330',
button_type="success")
time_get = Button(label='Get UT330 date and time',
button_type="success")
self.time_compare = Div(text="", width=self.page_width)
time_set = Button(label='Set the UT330 date and time',
button_type="success")
time_connect.on_click(self.time_connect)
time_disconnect.on_click(self.time_disconnect)
time_get.on_click(self.time_get)
time_set.on_click(self.time_set)
l = layout([self.time_status],
[time_connect, time_disconnect],
[time_get, self.time_compare],
[time_set])
return Panel(child=l, title="Date and time setting")
def device_data_tab(self):
"""Reading device data"""
self.data_status = Div(text="", width=self.page_width)
data_connect = Button(label='Connect to UT330',
button_type="success")
data_read = Button(label='Read data',
button_type="success")
data_write = Button(label='Write data to disk',
button_type="success")
data_erase = Button(label='Erase data',
button_type="success")
data_disconnect = Button(label='Disconnect from UT330',
button_type="success")
data_connect.on_click(self.data_connect)
data_read.on_click(self.data_read)
data_write.on_click(self.data_write)
data_erase.on_click(self.data_erase)
data_disconnect.on_click(self.data_disconnect)
if self.device_connected:
self.data_status.text = ('UT330 device connected. The Read, '
'Write, Erase, and Disconnect buttons '
'will work.')
else:
self.data_status.text = ('UT330 device is <strong>NOT</strong> '
'connected. The '
'Read, Write, Erase, and Disconnect '
'buttons will <strong>not work</strong>. '
'Press the '
'Connect button if the UT330 is '
'connected on a USB port.')
# Layout
l = layout([[self.data_status],
[data_connect, data_disconnect],
[data_read, data_write, data_erase]],
width=self.page_width)
return Panel(child=l,
title="Read from device")
class SqlConfig:
def __init__(self):
self.config = {}
self.load(update_widgets=False)
self.input_types = ['host', 'port', 'dbname', 'user', 'password']
self.input_widget = {key: TextInput for key in self.input_types}
self.input_widget['password'] = PasswordInput
div_sql = Div(text="<b>SQL Settings</b>")
title = {
'host': 'Host',
'port': 'Port',
'dbname': 'Database Name',
'user': '******',
'password': '******'
}
self.input = {
key: self.input_widget[key](value=self.config[key],
title=title[key],
width=300)
for key in self.input_types
}
for key in self.input_types:
self.input[key].on_change('value', self.save_needed)
self.check_tables_button = Button(label='Check Tables',
button_type='primary',
width=100)
self.check_tables_button.on_click(self.check_tables)
self.create_tables_button = Button(label='Create Tables',
button_type='primary',
width=100)
self.create_tables_button.on_click(self.create_tables)
self.clear_tables_button = Button(label='Clear Tables',
button_type='primary',
width=100)
self.clear_tables_button.on_click(self.clear_tables)
self.reload_button = Button(label='Reload',
button_type='primary',
width=100)
self.reload_button.on_click(self.load)
self.save_button = Button(label='Save',
button_type='default',
width=100)
self.save_button.on_click(self.save)
self.echo_button = Button(label="Echo",
button_type='primary',
width=100)
self.echo_button.on_click(self.echo)
self.layout = column(
div_sql, row(self.input['host'], self.input['port']),
row(self.input['user'], self.input['password']),
self.input['dbname'],
row(self.reload_button, self.echo_button, self.save_button),
row(self.check_tables_button, self.create_tables_button,
self.clear_tables_button))
def load(self, update_widgets=True):
self.config = load_sql_settings()
if not is_sql_connection_defined():
write_sql_connection_settings(self.config)
if update_widgets:
for key in self.input_types:
self.input[key].value = self.config[key]
self.save_button.button_type = 'default'
self.save_button.label = 'Save'
def save(self):
self.update_config()
write_sql_connection_settings(self.config)
self.load()
self.save_button.button_type = 'default'
self.save_button.label = 'Save'
def update_config(self):
for key in self.input_types:
self.config[key] = self.input[key].value
def echo(self):
self.update_config()
initial_button_type = self.echo_button.button_type
initial_label = self.echo_button.label
if validate_sql_connection(config=self.config, verbose=False):
self.echo_button.button_type = 'success'
self.echo_button.label = 'Success'
else:
self.echo_button.button_type = 'warning'
self.echo_button.label = 'Fail'
time.sleep(1.5)
self.echo_button.button_type = initial_button_type
self.echo_button.label = initial_label
def check_tables(self):
initial_label = self.check_tables_button.label
initial_button_type = self.check_tables_button.button_type
try:
table_result = {
table: DVH_SQL().check_table_exists(table)
for table in ['dvhs', 'plans', 'beams', 'rxs']
}
if all(table_result.values()):
self.check_tables_button.button_type = 'success'
self.check_tables_button.label = 'Success'
else:
self.check_tables_button.button_type = 'warning'
self.check_tables_button.label = 'Fail'
except:
self.check_tables_button.button_type = 'warning'
self.check_tables_button.label = 'No Connection'
time.sleep(1.5)
self.check_tables_button.button_type = initial_button_type
self.check_tables_button.label = initial_label
def create_tables(self):
initial_label = self.create_tables_button.label
initial_button_type = self.create_tables_button.button_type
if initial_label == 'Cancel':
self.create_tables_button.button_type = 'primary'
self.create_tables_button.label = 'Create Tables'
self.clear_tables_button.button_type = 'primary'
self.clear_tables_button.label = 'Clear Tables'
else:
try:
DVH_SQL().initialize_database()
except:
self.create_tables_button.button_type = 'warning'
self.create_tables_button.label = 'No Connection'
time.sleep(1.5)
self.create_tables_button.button_type = initial_button_type
self.create_tables_button.label = initial_label
def clear_tables(self):
if self.clear_tables_button.button_type == 'danger':
try:
DVH_SQL().reinitialize_database()
except:
self.clear_tables_button.button_type = 'warning'
self.clear_tables_button.label = 'No Connection'
time.sleep(1.5)
self.clear_tables_button.button_type = 'primary'
self.clear_tables_button.label = 'Clear Tables'
self.create_tables_button.button_type = 'primary'
self.create_tables_button.label = 'Create Tables'
self.clear_tables_button.button_type = 'primary'
self.clear_tables_button.label = 'Clear Tables'
elif self.clear_tables_button.button_type == 'primary':
self.clear_tables_button.button_type = 'danger'
self.clear_tables_button.label = 'Are you sure?'
self.create_tables_button.button_type = 'success'
self.create_tables_button.label = 'Cancel'
def save_needed(self, attr, old, new):
self.save_button.label = 'Save Needed'
self.save_button.button_type = 'warning'
def create_interact_ui(doc):
# The data source includes metadata for hover-over tooltips
lc_source = prepare_lightcurve_datasource(lc)
tpf_source = prepare_tpf_datasource(tpf, aperture_mask)
# Create the lightcurve figure and its vertical marker
fig_lc, vertical_line = make_lightcurve_figure_elements(
lc, lc_source, ylim_func=ylim_func)
# Create the TPF figure and its stretch slider
pedestal = -np.nanmin(tpf.flux.value) + 1
if scale == "linear":
pedestal = 0
fig_tpf, stretch_slider = make_tpf_figure_elements(
tpf,
tpf_source,
pedestal=pedestal,
fiducial_frame=0,
vmin=vmin,
vmax=vmax,
scale=scale,
cmap=cmap,
tools=tools,
)
# Helper lookup table which maps cadence number onto flux array index.
tpf_index_lookup = {cad: idx for idx, cad in enumerate(tpf.cadenceno)}
# Interactive slider widgets and buttons to select the cadence number
cadence_slider = Slider(
start=np.min(tpf.cadenceno),
end=np.max(tpf.cadenceno),
value=np.min(tpf.cadenceno),
step=1,
title="Cadence Number",
width=490,
)
r_button = Button(label=">", button_type="default", width=30)
l_button = Button(label="<", button_type="default", width=30)
export_button = Button(label="Save Lightcurve",
button_type="success",
width=120)
message_on_save = Div(text=" ", width=600, height=15)
# Callbacks
def _create_lightcurve_from_pixels(tpf,
selected_pixel_indices,
transform_func=transform_func):
"""Create the lightcurve from the selected pixel index list"""
selected_indices = np.array(selected_pixel_indices)
selected_mask = np.isin(pixel_index_array, selected_indices)
lc_new = tpf.to_lightcurve(aperture_mask=selected_mask)
lc_new.meta["APERTURE_MASK"] = selected_mask
if transform_func is not None:
lc_transformed = transform_func(lc_new)
if len(lc_transformed) != len(lc_new):
warnings.warn(
"Dropping cadences in `transform_func` is not "
"yet supported due to fixed time coordinates."
"Skipping the transformation...",
LightkurveWarning,
)
else:
lc_new = lc_transformed
lc_new.meta["APERTURE_MASK"] = selected_mask
return lc_new
def update_upon_pixel_selection(attr, old, new):
"""Callback to take action when pixels are selected."""
# Check if a selection was "re-clicked", then de-select
if (sorted(old) == sorted(new)) & (new != []):
# Trigger recursion
tpf_source.selected.indices = new[1:]
if new != []:
lc_new = _create_lightcurve_from_pixels(
tpf, new, transform_func=transform_func)
lc_source.data["flux"] = lc_new.flux.value
if ylim_func is None:
ylims = get_lightcurve_y_limits(lc_source)
else:
ylims = _to_unitless(ylim_func(lc_new))
fig_lc.y_range.start = ylims[0]
fig_lc.y_range.end = ylims[1]
else:
lc_source.data["flux"] = lc.flux.value * 0.0
fig_lc.y_range.start = -1
fig_lc.y_range.end = 1
message_on_save.text = " "
export_button.button_type = "success"
def update_upon_cadence_change(attr, old, new):
"""Callback to take action when cadence slider changes"""
if new in tpf.cadenceno:
frameno = tpf_index_lookup[new]
fig_tpf.select("tpfimg")[0].data_source.data["image"] = [
tpf.flux.value[frameno, :, :] + pedestal
]
vertical_line.update(location=tpf.time.value[frameno])
else:
fig_tpf.select("tpfimg")[0].data_source.data["image"] = [
tpf.flux.value[0, :, :] * np.NaN
]
lc_source.selected.indices = []
def go_right_by_one():
"""Step forward in time by a single cadence"""
existing_value = cadence_slider.value
if existing_value < np.max(tpf.cadenceno):
cadence_slider.value = existing_value + 1
def go_left_by_one():
"""Step back in time by a single cadence"""
existing_value = cadence_slider.value
if existing_value > np.min(tpf.cadenceno):
cadence_slider.value = existing_value - 1
def save_lightcurve():
"""Save the lightcurve as a fits file with mask as HDU extension"""
if tpf_source.selected.indices != []:
lc_new = _create_lightcurve_from_pixels(
tpf,
tpf_source.selected.indices,
transform_func=transform_func)
lc_new.to_fits(
exported_filename,
overwrite=True,
flux_column_name="SAP_FLUX",
aperture_mask=lc_new.meta["APERTURE_MASK"].astype(np.int),
SOURCE="lightkurve interact",
NOTE="custom mask",
MASKNPIX=np.nansum(lc_new.meta["APERTURE_MASK"]),
)
if message_on_save.text == " ":
text = '<font color="black"><i>Saved file {} </i></font>'
message_on_save.text = text.format(exported_filename)
export_button.button_type = "success"
else:
text = '<font color="gray"><i>Saved file {} </i></font>'
message_on_save.text = text.format(exported_filename)
else:
text = (
'<font color="gray"><i>No pixels selected, no mask saved</i></font>'
)
export_button.button_type = "warning"
message_on_save.text = text
def jump_to_lightcurve_position(attr, old, new):
if new != []:
cadence_slider.value = lc.cadenceno[new[0]]
# Map changes to callbacks
r_button.on_click(go_right_by_one)
l_button.on_click(go_left_by_one)
tpf_source.selected.on_change("indices", update_upon_pixel_selection)
lc_source.selected.on_change("indices", jump_to_lightcurve_position)
export_button.on_click(save_lightcurve)
cadence_slider.on_change("value", update_upon_cadence_change)
# Layout all of the plots
sp1, sp2, sp3, sp4 = (
Spacer(width=15),
Spacer(width=30),
Spacer(width=80),
Spacer(width=60),
)
widgets_and_figures = layout(
[fig_lc, fig_tpf],
[
l_button, sp1, r_button, sp2, cadence_slider, sp3,
stretch_slider
],
[export_button, sp4, message_on_save],
)
doc.add_root(widgets_and_figures)
def start_program():
# Set up widgets
text = TextInput(title="HashTag", placeholder='Write a HashTag')
columns = [
'Today', 'From yesterday', 'Two days ago', 'Three days ago',
'Four days ago', 'Five days ago', 'Six days ago'
]
days = Select(title='Days', value='Today', options=columns)
button = Button(label="Fetch Data", button_type="success")
button1 = Button(label="Refresh Map", button_type="danger", disabled=True)
message_box = Div(text="Welcome")
# Callback function for running the program when hastag requested
def callback(event):
if (text.value == ""):
layout1.children[0] = display_message(
"HashTag should not be Empty")
return
# Disabled the properties for user utill data is fetched
button.disabled = True
button.button_type = "danger"
sentiment.disabled = True
continent.disabled = True
button1.disabled = True
button1.button_type = "danger"
old = 0
if (days.value == "Today"):
old = 0
elif (days.value == "From yesterday"):
old = 1
elif (days.value == "Two days ago"):
old = 2
elif (days.value == "Three days ago"):
old = 3
elif (days.value == "Four days ago"):
old = 4
elif (days.value == "Five days ago"):
old = 5
else:
old = 6
#Calling the main function for retriving the data
main(hashtag=text.value,
days=old,
layout=layout1,
button=button,
sentiment=sentiment,
continent=continent,
button1=button1)
# create_figure function for creating the google map onto the browser with data from output.csv
def create_figure(event):
conti = continent.value
senti = sentiment.value
if (conti == "North America"):
map_options = GMapOptions(lat=33.465777,
lng=-88.369025,
map_type="roadmap",
zoom=3)
elif (conti == "South America"):
map_options = GMapOptions(lat=-8.783195,
lng=-55.491477,
map_type="roadmap",
zoom=3)
elif (conti == "Africa"):
map_options = GMapOptions(lat=-8.783195,
lng=34.508523,
map_type="roadmap",
zoom=3)
elif (conti == "Asia"):
map_options = GMapOptions(lat=34.047863,
lng=100.619655,
map_type="roadmap",
zoom=3)
elif (conti == "Europe"):
map_options = GMapOptions(lat=54.525961,
lng=15.255119,
map_type="roadmap",
zoom=3)
elif (conti == "Antartica"):
map_options = GMapOptions(lat=-82.862752,
lng=135.000000,
map_type="roadmap",
zoom=3)
else:
map_options = GMapOptions(lat=-25.274398,
lng=133.775136,
map_type="roadmap",
zoom=3)
# Calling the Google API Key for displaying google maps
p = gmap("AIzaSyCJMprrXTtmUqciVaVgskmHxskLkVjrE6A",
map_options,
title="World Map",
width=950)
#Ploting the legends on the map
p.circle(None,
None,
size=5,
fill_color="green",
fill_alpha=0.7,
legend="Positive")
p.circle(None,
None,
size=5,
fill_color="yellow",
fill_alpha=0.7,
legend="Neutral")
p.circle(None,
None,
size=5,
fill_color="red",
fill_alpha=0.7,
legend="Negative")
# Load function for extracting data from "output.csv" file
def load(data):
lat = []
lon = []
with open('output.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
x = float(row["longitude"])
y = float(row["latitude"])
z = row["sentiment"]
if (z == data):
lat.append(y)
lon.append(x)
data1 = dict(lat=lat, lon=lon)
return data1
# pointing the data on Google Maps
if (senti == "All"):
source = ColumnDataSource(data=load(data="positive"))
p.circle(x="lon",
y="lat",
size=8,
fill_color="green",
fill_alpha=1.0,
source=source)
source = ColumnDataSource(data=load(data="negative"))
p.circle(x="lon",
y="lat",
size=8,
fill_color="red",
fill_alpha=1.0,
source=source)
source = ColumnDataSource(data=load(data="neutral"))
p.circle(x="lon",
y="lat",
size=8,
fill_color="yellow",
fill_alpha=1.0,
source=source)
elif (senti == "Positive"):
source = ColumnDataSource(data=load(data="positive"))
p.circle(x="lon",
y="lat",
size=8,
fill_color="green",
fill_alpha=1.0,
source=source)
elif (senti == "Negative"):
source = ColumnDataSource(data=load(data="negative"))
p.circle(x="lon",
y="lat",
size=8,
fill_color="red",
fill_alpha=1.0,
source=source)
else:
source = ColumnDataSource(data=load(data="neutral"))
p.circle(x="lon",
y="lat",
size=8,
fill_color="yellow",
fill_alpha=1.0,
source=source)
return p #End of function create_figure()
#update function for on_change event listener
def update(attr, old, new):
layout.children[1] = create_figure(event=None)
#update1 function for on_click event listener
def update1():
layout.children[1] = create_figure(event=None)
# Event listner for buttons
button.on_event(ButtonClick, callback)
button1.on_click(update1)
#Set all Parameters for dropdown menus
columns = ["All", "Positive", "Neutral", "Negative"]
sentiment = Select(title='Sentiment',
value='All',
options=columns,
disabled=True)
sentiment.on_change('value', update)
continents = [
"North America", "South America", "Africa", "Asia", "Antartica",
"Europe", "Australia"
]
continent = Select(title='Continents',
value='North America',
options=continents,
disabled=True)
continent.on_change('value', update)
# Set up layouts and add to document
controls = widgetbox([text, days, button, sentiment, continent, button1])
layout1 = column(display_message("Welcome"), controls)
layout = row(layout1, create_figure(event=None))
#Drawing widgets onto the browser
curdoc().add_root(layout)
curdoc().title = "World Map"
bokeh_legend = Legend(items=[("", [])],
orientation="vertical",
border_line_color="black",
label_text_font_size={'value': '9pt'},
click_policy='hide',
visible=False)
bokeh_legend.label_width = 100
plot.add_layout(bokeh_legend, "right")
plot.y_range = Range1d(0, 100)
plot.extra_y_ranges['secondary'] = Range1d(0, 100)
# select file
file_selection_button = Button(label="Select Files",
button_type="success",
width=120)
file_selection_button.on_click(load_files_group)
files_selector_spacer = Spacer(width=10)
group_selection_button = Button(label="Select Directory",
button_type="primary",
width=140)
group_selection_button.on_click(load_directory_group)
update_files_button = Button(label="Update Files",
button_type="default",
width=50)
update_files_button.on_click(reload_all_files)
auto_update_toggle_button = Toggle(label="Auto Update",
button_type="default",
global current_topic
if current_topic is None:
current_topic = min_topic
elif current_topic == max_topic:
current_topic = min_topic
else:
current_topic += 1
mdssource.selected = [current_topic-1]
session.store_objects(mdssource)
def on_btn_clear():
current_topic = None
mdssource.selected = []
session.store_objects(mdssource)
btn_prev.on_click(on_btn_prev)
btn_next.on_click(on_btn_next)
btn_clear.on_click(on_btn_clear)
slider = Slider(title="Slide to adjust relevance metric", value=0.25, start=0.0, end=1.0, step=0.01)
topics = HBox(height=50, children=[btn_prev, btn_next, btn_clear])
metric = HBox(height=50, children=[slider])
left_panel = VBox(children=[topics, intertopic_distance_map])
right_panel = VBox(children=[metric, top_R_terms])
layout = HBox(children=[left_panel, right_panel])
document.add(layout)
session.store_document(document)
def colorbar_slider(fig):
'''
Adds interactive sliders and text input boxes for the colorbar.
Returns a layout object to be put into a gridplot
'''
cb = get_colorbar_renderer(fig)
data = get_image_data(fig)
data = reject_outliers_quick(data)
datamin = nanmin(data)
datamax = nanmax(data)
im = get_glyph_renderer(fig) # Get image renderer
from bokeh.models import CustomJS, Slider, TextInput
from bokeh.models.widgets import Button
from bokeh.layouts import widgetbox
model = Slider() # trick it into letting datamin and datamax into CustomJS
model.tags.append(datamin) # Hide these in here
model.tags.append(datamax)
callback_u = CustomJS(args=dict(cb=cb, im=im, model=model), code="""
var cm = cb.color_mapper;
var upp = upper_slider.get('value');
upper_input.value = upp.toString()
lower_slider.end = upp
cm.high = upp;
im.glyph.color_mapper.high = upp;
if (cm.low >= cm.high){
cm.low = upp/1.1 // to prevent limits being the same
im.glyph.color_mapper.low = low/1.1;
}
if (upp > model.tags[1]){
upper_slider.end = upp
}
""")
callback_l = CustomJS(args=dict(cb=cb, im=im, model=model), code="""
var cm = cb.color_mapper;
var low = lower_slider.get('value');
lower_input.value = low.toString()
upper_slider.start = low
cm.low = low;
im.glyph.color_mapper.low = low;
if (cm.high <= cm.low){
cm.high = low*1.1 // to prevent limits being the same
im.glyph.color_mapper.high = low*1.1;
}
if (low < model.tags[0]){
lower_slider.start = low
}""")
callback_ut = CustomJS(args=dict(cb=cb, im=im, model=model), code="""
var cm = cb.color_mapper;
var upp = parseFloat(upper_input.get('value'));
upper_slider.value = upp
cm.high = upp;
im.glyph.color_mapper.high = upp;
if (cm.low >= cm.high){
cm.low = upp/1.1 // to prevent limits being the same
im.glyph.color_mapper.low = upp/1.1;
}
if (upp > model.tags[1]){
upper_slider.end = upp
}
""")
callback_lt = CustomJS(args=dict(cb=cb, im=im, model=model), code="""
var cm = cb.color_mapper;
var low = parseFloat(lower_input.get('value'));
lower_slider.value = low
cm.low = low;
im.glyph.color_mapper.low = low;
if (cm.high <= cm.low){
cm.high = low*1.1 // to prevent limits being the same
im.glyph.color_mapper.high = low*1.1;
}
if (low < model.tags[0]){
lower_slider.start = low
}
""")
callback_reset_js = CustomJS(args=dict(cb=cb, im=im, model=model), code="""
var cm = cb.color_mapper;
var low = model.tags[0];
var high = model.tags[1];
low = parseFloat(low.toPrecision(3)) // 3 sig figs
high = parseFloat(high.toPrecision(3)) // 3 sig figs
lower_slider.value = low;
lower_slider.set('step', (high-low)/50);
cm.low = low;
upper_slider.value = high;
upper_slider.set('step', (high-low)/50);
cm.high = high;
im.glyph.color_mapper.low = low;
im.glyph.color_mapper.high = high;
lower_input.value = low.toString();
upper_input.value = high.toString();
lower_slider.start = low;
lower_slider.end = high;
upper_slider.start = low;
upper_slider.end = high;
model.trigger('change')
cb_obj.trigger('change)')
""")
reset_button = Button(label='Reset', callback = callback_reset_js)
def callback_reset(*args, **kwargs):
from IPython.display import Javascript, display
# display(callback_reset_js)
# callback_reset_js.name = None
# callback_reset_js.name = 'test'
# display('Plot updated, press reset to rescale!')
# cb.color_mapper.low = datamin
# cb.color_mapper.high = datamax
# im.glyph.color_mapper.low = datamin
# im.glyph.color_mapper.high = datamax
# lower_slider.start = datamin
# lower_slider.end = datamax
# lower_slider.value = datamin
# upper_slider.start = datamin
# upper_slider.end = datamax
# lower_slider.value = datamax
# lower_input.value = str(datamin)
# upper_input.value = str(datamax)
# update()
# fig.text(x=0,y=0,text='Plot updated, press reset to rescale!')
# reset_button.label='Reset: Data changed! Press me!'
# reset_button.trigger('clicks',0,1)
reset_button.on_click(callback_reset)
# def callback_die(attr, old, new):
# from IPython.display import display
# display('yoooo')
# display(old)
# display(new)
# raise Exception()
# exception_button = Button(label='KILL ME')
# exception_button.on_click(callback_die)
lower_slider = Slider(start=datamin, end=datamax, value=datamin, step=(datamax-datamin)/50, # smallest step is 1e-5
title="Lower lim", callback=callback_l)
lower_slider.width=100
upper_slider = Slider(start=datamin, end=datamax, value=datamax, step=(datamax-datamin)/50,
title="Upper lim", callback=callback_u)
upper_slider.width=100
lower_input = TextInput(callback=callback_lt, value = str(datamin), width=50)
upper_input = TextInput(callback=callback_ut, value = str(datamax), width=50)
# add all of these widgets as arguments to the callback functions
for callback in ['l', 'u', 'lt', 'ut', 'reset_js']:
for widget in ['lower_slider', 'upper_slider','lower_input','upper_input', 'reset_button']:
exec('callback_%s.args["%s"] = %s' %(callback, widget, widget))
wb = widgetbox([upper_slider, upper_input, lower_slider, lower_input, reset_button], width=100, sizing_mode = 'stretch_both')
return wb
def plot():
# INPUT WIDGETS
collection_list = ['bugaboo']
#collection_list = util.CONN[util.DB].collection_names(include_system_collections=False)
gliders = sorted([platformID for platformID in collection_list if len(platformID)>2])
gliders = Select(title = 'PlatformID', value = gliders[0], options = gliders)
prev_glider = Button(label = '<')
next_glider = Button(label = '>')
glider_controlbox = HBox(children = [gliders, prev_glider, next_glider])
control_box = HBox(glider_controlbox)
# DATA VARS
heatdata = dict(x=[1.5,2],y=[2,1.5],values=[0,10])
t = 1000*util.time()
heatdata = dict(x=[t-4*60*1000,t-3*60*1000,t-2*60*1000,t-1*60*1000],
y=[1,1,2,2], values=[1,2,3,4])
# FIGURES AND AXIS
fig1 = HeatMap(data=heatdata, x='x', y='y', values='values', title = 'SUPERHOT')
# LEGEND
#TODO
"""
legend = Figure(tools=None)
legend.toolbar_location=None
legend.rect(x=0.5, y='value', fill_color='color', width=1, height=1, source=heatdata)
layout = hplot(main, legend)
"""
# CALLBACK FUNCS
def update_data(attrib,old,new):
g = gliders.value
#depth = load_sensor(g, 'm_depth')
#celcius = load_sensor(g, 'sci_water_temp')
t = 1000*util.time()
ego = dict(x=[t-4*60*1000,t-3*60*1000,t-2*60*1000,t-1*60*1000],
y=[1,1,2,2], values=[1,2,3,4])
fig1.pprint_props()
#GLIDER SELECTS
def glider_buttons(increment):
ops = gliders.options
new_index = ops.index(gliders.value) + increment
if new_index >= len(ops):
new_index = 0
elif new_index < 0:
new_index = len(ops)-1
gliders.value = ops[new_index]
def next_glider_func():
glider_buttons(1)
def prev_glider_func():
glider_buttons(-1)
gliders.on_change('value', update_data)
next_glider.on_click(next_glider_func)
prev_glider.on_click(prev_glider_func)
update_data(None,None,None)
return vplot(control_box, fig1)
# import libraries
from bokeh.io import output_file, show
from bokeh.models.widgets import TextInput, Button, Paragraph
from bokeh.layouts import layout
# prepare the bokeh output file
output_file('simple_bokeh.html')
# create widgets
text_input = TextInput(value='Bryan')
button = Button(label='Generate Text')
output = Paragraph()
# create a function to update
def update():
output.text = 'Hello, ' + text_input
button.on_click(update)
lo = layout([[button, text_input], [output]])
show(lo)
def offset_tab(self):
"""Reading/writing device offsets"""
# True if the offset device data has been read, false otherwise
self.offset_device_read = False
offset_status_h = Div(text="<strong>Status</strong>")
self.offset_status = Div(text="", width=self.page_width)
# Connect to device button
# ========================
offset_controls_h = Div(text="<strong>Device controls</strong>")
offset_connect = Button(label='Connect to UT330',
button_type="success")
offset_read = Button(label='Read offset', button_type="success")
offset_write = Button(label='Write offset', button_type="success")
offset_disconnect = Button(label='Disconnect from UT330',
button_type="success")
offset_connect.on_click(self.offset_connect)
offset_read.on_click(self.offset_read)
offset_write.on_click(self.offset_write)
offset_disconnect.on_click(self.offset_disconnect)
# Offsets
# =======
offset_offsets_h = Div(text="<strong>Offsets</strong>")
self.offset_t_current = TextInput(title="Temperature current")
self.offset_h_current = TextInput(title="Humidity current")
self.offset_p_current = TextInput(title="Pressure current")
self.offset_t = TextInput(title="Temperature offset")
self.offset_h = TextInput(title="Humidity offset")
self.offset_p = TextInput(title="Pressure offset")
# Values to widgets
# =================
if self.device_connected:
self.offset_connected()
else:
self.offset_not_connected()
if self.device_connected:
self.offset_status.text = ('UT330 device connected. The Read, '
'Write, and Disconnect buttons '
'will work.')
else:
self.offset_status.text = ('UT330 device is <strong>NOT</strong> '
'connected. The '
'Read, Write, and Disconnect buttons '
'will <strong>not work</strong>. '
'Click the '
'Connect button if the UT330 is '
'connected on a USB port.')
# Layout
# ======
l = layout([[offset_status_h],
[self.offset_status],
[offset_controls_h],
[offset_connect,
offset_read,
offset_write,
offset_disconnect],
[offset_offsets_h],
[self.offset_t_current,
self.offset_h_current,
self.offset_p_current],
[self.offset_t,
self.offset_h,
self.offset_p]],
width=self.page_width)
return Panel(child=l,
title="Read/write offset")
def config_tab(self):
"""Reading/writing device configuration"""
# True if the config device data has been read, false otherwise
self.config_device_read = False
# Device connectivity
# ===================
config_conn_head = Div(text="<strong>Connectivity</strong>")
self.config_status = Div(text="", width=self.page_width)
config_connect = Button(label='Connect to UT330',
button_type="success")
config_read = Button(label='Read config', button_type="success")
config_write = Button(label='Write config', button_type="success")
config_disconnect = Button(label='Disconnect from UT330',
button_type="success")
config_connect.on_click(self.config_connect)
config_read.on_click(self.config_read)
config_write.on_click(self.config_write)
config_disconnect.on_click(self.config_disconnect)
# Show the configuration data
# ===========================
# Set up the widgets
config_device_head = Div(text="<strong>Configuration</strong>")
self.config_device_name = TextInput(title="Device name")
self.config_device_time = TextInput(title="Device time")
self.config_computer_time = TextInput(title="Computer time")
self.config_t_high = TextInput(title="High temperature alarm (C)")
self.config_t_low = TextInput(title="Low temperature alarm (C)")
self.config_h_high = TextInput(title="High humidity alarm (%RH)")
self.config_h_low = TextInput(title="Low humidity alarm (%RH)")
self.config_p_high = TextInput(title="High pressure alarm")
self.config_p_low = TextInput(title="Low pressure alarm")
self.config_sampling = TextInput(title="Sampling interval (s)")
self.config_overwrite_records = Select(title="Overwrite records",
options=['False', 'True'])
self.config_delay_start = Select(title="Delay start",
options=['No delay', 'Delay'])
self.config_delay = TextInput(title="Delay (s)")
# Status data
# ===========
config_status_head = Div(text="<strong>Status</strong>")
self.config_power = TextInput(title="Battery power (%)")
self.config_readings = TextInput(title="Readings")
# Disable user input for these widgets
self.config_power.disabled = True
self.config_readings.disabled = True
# Values to widgets
# =================
if self.device_connected:
self.config_connected()
else:
self.config_not_connected()
# Set up the display
layout = column(row(config_conn_head),
row(self.config_status),
row(config_connect,
config_read,
config_write,
config_disconnect),
row(config_device_head),
row(self.config_device_name,
self.config_device_time,
self.config_computer_time),
row(self.config_t_low,
self.config_h_low,
self.config_p_low),
row(self.config_t_high,
self.config_h_high,
self.config_p_high),
row(self.config_sampling),
row(self.config_overwrite_records,
self.config_delay_start,
self.config_delay),
row(config_status_head),
row(self.config_power, self.config_readings))
return Panel(child=layout,
title="Read/write configuration")
date_slider.on_change("value", update)
time_slider.on_change("value", update)
type_button.on_change("active", update)
def on_selection_change(attr, old, new):
print("on_selection_change")
indices = new["1d"]["indices"]
if indices:
sector_id = geo_sectors.loc[indices[0]]["sector_id"]
sector_select.value = str(sector_id) # Calls update in chain
#update_sources(sector_id=sector_id)
map_plot.get_data_source().on_change("selected", on_selection_change)
def close_session():
session.close()
button.on_click(close_session)
# Page build
inputs = VBoxForm(children=[sector_select, date_slider, time_slider,
type_button])
map_box = HBox(children=[map_colorbar.get_plot(), map_plot.get_plot()])
plots = VBox(children=[map_box, conso_plot.get_plot()])
root_element = HBox(children=[inputs, plots])
update(None, None, None) # Init data
# Session
print("Starting server")
curdoc().add_root(root_element)
from __future__ import print_function
from bokeh.client import push_session
from bokeh.io import curdoc
from bokeh.models import CustomJS
from bokeh.models.layouts import WidgetBox
from bokeh.models.widgets import (
Button, Toggle, Dropdown, CheckboxGroup, RadioGroup,
CheckboxButtonGroup, RadioButtonGroup,
)
button = Button(label="Button (enabled) - has click event", button_type="primary")
button.on_click(lambda: print('button: click'))
button.js_on_click(CustomJS(code="console.log('button: click', this.toString())"))
button_disabled = Button(label="Button (disabled) - no click event", button_type="primary", disabled=True)
button_disabled.on_click(lambda: print('button_disabled: click'))
button_disabled.js_on_click(CustomJS(code="console.log('button_disabled: click', this.toString())"))
toggle_inactive = Toggle(label="Toggle button (initially inactive)", button_type="success")
toggle_inactive.on_click(lambda value: print('toggle_inactive: %s' % value))
toggle_inactive.js_on_click(CustomJS(code="console.log('toggle_inactive: ' + this.active, this.toString())"))
toggle_active = Toggle(label="Toggle button (initially active)", button_type="success", active=True)
toggle_active.on_click(lambda value: print('toggle_active: %s' % value))
toggle_active.js_on_click(CustomJS(code="console.log('toggle_active: ' + this.active, this.toString())"))
menu = [("Item 1", "item_1_value"), ("Item 2", "item_2_value"), None, ("Item 3", "item_3_value")]
dropdown = Dropdown(label="Dropdown button", button_type="warning", menu=menu)
dropdown.on_click(lambda value: print('dropdown: %s' % value))
auth_button.button_type = 'danger'
time.sleep(3)
auth_button.label = 'Authenticate'
auth_button.button_type = 'warning'
######################################################
# Layout objects
######################################################
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Custom authorization
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
auth_user = TextInput(value='', title='User Name:', width=150)
auth_pass = PasswordInput(value='', title='Password:'******'inbox'], title="Inbox", width=600)
input_inbox.on_change('value', update_inbox)
input_imported = TextInput(value=directories['imported'],
title="Imported",
width=600)
input_imported.on_change('value', update_imported)
i = i + 1
source.callback = CustomJS(args=dict(colors=colors, clusters=i),
code="""
var inds = cb_obj.getv('selected')['1d'].indices;
var d1 = cb_obj.data;
if (inds.length == 0) { return; }
for (i = 0; i < inds.length; i++) {
d1['cluster'][inds[i]] = clusters;
d1['color'][inds[i]] = colors[clusters];
}
cb_obj.change.emit();
""")
source.selected.indices = []
cluster_button.on_click(callback)
run_button.on_click(run_task)
source.callback = CustomJS(args=dict(colors=colors, clusters=i),
code="""
var inds = cb_obj.getv('selected')['1d'].indices;
var d1 = cb_obj.data;
if (inds.length == 0) { return; }
for (i = 0; i < inds.length; i++) {
d1['cluster'][inds[i]] = clusters;
d1['color'][inds[i]] = colors[clusters];
}
cb_obj.change.emit();
""")
xdr = Range1d(start=0, end=300)
v0_value = v0.value
v1_value = v1.value
v2_value = v2.value
v3_value = v3.value
# Generate the new curve
m_projection = [v0_value, v1_value, v2_value, v3_value]
rec = get_reconstruction(m_eigs_t, m_projection, m_mean_vector)
x = [i for i in xrange(np.shape(rec)[0])]
y = [v for v in rec]
source.data = dict(x=x, y=y)
def on_click(*args):
m_projection = [0., 0., 0., 0.]
rec = get_reconstruction(m_eigs_t, m_projection, m_mean_vector)
x = [i for i in xrange(np.shape(rec)[0])]
y = [v for v in rec]
source.data = dict(x=x, y=y)
for w in [v0, v1, v2, v3]:
w.on_change('value', update_data)
button.on_click(on_click)
# Set up layouts and add to document
inputs = HBox(children=[v0, v1, v2, v3, button])
curdoc().add_root(VBox(children=[inputs, plot], width=1200))
class Config_Panel(object):
def __init__(self, ut330):
self.ut330 = ut330
self.device_name = TextInput(title="Device name")
self.device_time = TextInput(title="Device time")
self.computer_time = TextInput(title="Computer time")
self.t_high = TextInput(title="High temperature alarm (C)")
self.t_low = TextInput(title="Low temperature alarm (C)")
self.h_high = TextInput(title="High humidity alarm (%RH)")
self.h_low = TextInput(title="Low humidity alarm (%RH)")
self.p_high = TextInput(title="High pressure alarm")
self.p_low = TextInput(title="Low pressure alarm")
self.sampling = TextInput(title="Sampling interval (s)")
self.overwrite_records = Select(title="Overwrite records",
options=['False', 'True'])
self.delay_start = Select(title="Delay start",
options=['No delay', 'Delay'])
self.delay = TextInput(title="Delay (s)")
self.power = TextInput(title="Battery power (%)")
self.readings = TextInput(title="Readings")
self.read_config = Button(label='Read config')
self.write_config = Button(label='Write config')
def _layout_(self):
return VBox(HBox(self.device_name,
self.device_time,
self.computer_time,
width=700),
HBox(self.t_low,
self.h_low,
self.p_low,
width=700),
HBox(self.t_high,
self.h_high,
self.p_high,
width=700),
HBox(self.sampling, width=700),
HBox(self.overwrite_records,
self.delay_start,
self.delay,
width=700),
HBox(self.power,
self.readings,
width=700),
HBox(self.read_config,
self.write_config,
width=700))
def panel(self):
return Panel(child=self._layout_(), title="Configuration")
def callbacks(self):
self.read_config.on_click(self._read_)
self.write_config.on_click(self._write_)
def device_read(self):
self._read_()
def _read_(self):
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
config = self.ut330.read_config()
self.device_name.value = config['device name']
self.device_time.value = config['timestamp'].strftime("%Y-%m-%d "
"%H:%M:%S")
self.computer_time.value = now
self.t_high.value = str(config['high temperature alarm'])
self.t_low.value = str(config['low temperature alarm'])
self.h_high.value = str(config['high humidity alarm'])
self.h_low.value = str(config['low humidity alarm'])
self.p_high.value = 'None'
self.p_low.value = 'None'
self.sampling.value = str(config['sampling interval'])
self.overwrite_records.value = 'True' if config['overwrite records'] \
else 'False'
self.delay_start.value = 'Delay' if config['delay start'] \
else 'No delay'
self.delay.value = str(config['delay timing'])
self.power.value = str(config['battery power'])
self.readings.value = "{0}/{1}".format(config['readings count'],
config['readings limit'])
def _write_(self):
config = {'device name': self.device_name.value,
'sampling interval': int(self.sampling.value),
'overwrite records': self.overwrite_records.value == 'True',
'delay timing': int(self.delay.value),
'delay start': self.delay_start.value == 'Delay',
'high temperature alarm': int(self.t_high.value),
'low temperature alarm': int(self.t_low.value),
'high humidity alarm': int(self.h_high.value),
'low humidity alarm': int(self.h_low.value)}
self.ut330.write_config(config)
title=u'业绩时间窗口',
width=200,
options=time_selections)
time_select.on_change('value', lambda attr, old, new: update_data())
comp_stock_hold_select = Select(value=u'全部',
title=u'公募基金重仓持股',
width=200,
options=comp_holds)
comp_stock_hold_select.on_change('value',
lambda attr, old, new: update_comp_hold())
# today = datetime.datetime.today()
# time_start = TextInput(label=(today - datetime.timedelta(365)).strftime('%Y-%m-%d'), title=u'开始日期', width=200)
# time_end = TextInput(label=today.strftime('%Y-%m-%d'), title=u'结束日期', width=200)
# time_end.on_change('value', lambda attr, old, new: update_data())
fund_button = Button(label=u"筛选基金", button_type="success", width=200)
fund_button.on_click(select_fund)
update_button = Button(label=u'更新数据', button_type='success', width=200)
update_button.on_click(update_new_data)
fund_text = TextInput(value='000088.OF', title=u'基金Wind代码', width=200)
fund_text.on_change('value', lambda attr, old, new: update_data())
comp_select = Select(value=u'嘉实基金管理有限公司',
title=u'基金公司',
width=200,
options=comps)
comp_select.on_change('value', lambda attr, old, new: update_comp())
columns = [
TableColumn(field='sec_name', title=u'基金简称'),
TableColumn(field='wind_code', title=u'万得代码'),
TableColumn(field='fundmanager', title=u'基金经理'),
TableColumn(field='netasset',
print 'After: ' + str(undefinedSource.data) + '\n' ########################################################################### # Setup figure, glyphs, and a datasource glyphDict = create_glyphs() undefinedSource = bkM.ColumnDataSource(data=dict(x=[0,4,8], y=[0,0,0], width=[0.25,0.25,0.25], height=[2,2,2])) fig = create_fig() ########################################################################### # Add widgets and their callbacks from bokeh.models.widgets import Button, TextInput changeA_button = Button(label='UpdateFig A') changeA_button.on_click(update_figure_A) changeB_button = Button(label='UpdateFig B') changeB_button.on_click(update_figure_B) widgets = [changeA_button, changeB_button] from bokeh.layouts import widgetbox widgetBox = widgetbox(*widgets, sizing_mode='fixed') # Add figure and widgets to a layout ########################################################################### from bokeh.layouts import layout layout = layout( [[fig], [widgetBox]], sizing_mode='fixed') ###########################################################################
def index():
# CODA_export_file = '//longbasp0005/D/Documents/Qlik/Sense/Content/Text Analytics/Data/CODA_Export.csv'
CODA_export_file = 'data/CODA_Export.csv'
issues_df = load_issues_fast(CODA_export_file)
dend_threshold = 3.2
number_of_clusters = 14
# universe_issues_list = issues_df['CASE_NUMBER'].head(100).tolist()
# issues_list = ['572', '204', '3191', '202', '709', '626', '858', '98']
issues_dict = dict(issues_list=[""], issues_membership=[""])
# creat a list for the all issues interesterd called universe issues
# removing rejected and remediated issues
universe_issues_list = \
issues_df.CASE_NUMBER[~issues_df.DETAILED_DESCRIPTION.isin(["Rejected", "Remediated"])].tolist()
df_tfidf = pd.DataFrame()
# set the color palette to be used for assigning denrogram colors
set_link_color_palette(Category20_20)
# =============================================================================
# Create a new figure with a needed tools
# =============================================================================
p = figure(
tools="pan,wheel_zoom,box_zoom,box_select,save,reset",
plot_width=1000,
plot_height=800,
)
# =============================================================================
# Create source data containing and making the plot
# =============================================================================
# adding the threshold line as an exrta line in the last entry
dend_source = ColumnDataSource(data=dict(x=[], y=[], leg_color=[]))
leaves_source = ColumnDataSource(data=dict(leaves_x=[],
leaves_y=[],
leaves_names=[],
leaves_color=[],
leaves_size=[],
clusters=[]))
image_source = ColumnDataSource(
data=dict(url=["static/static_dend.png", "", ""],
text=["", "Enter your issues IDs", ""],
text_x=[0, 40, 100],
text_y=[0, 40, 80]))
# making fake source data for dend slider
source_for_slider = ColumnDataSource(data=dict(value=[]))
# add the static dend
p.image_url(url="url",
x=0,
y=0,
w=100,
h=80,
anchor="bottom_left",
source=image_source)
p.text(x="text_x", y="text_y", text="text", source=image_source)
# Plot dendrogram U shapes using multi_line function
p.multi_line("x", "y", color="leg_color", source=dend_source)
# remove the x axis tickers by setting them to empty
p.xaxis.ticker = []
# Add label to axex
p.yaxis.axis_label = "Distance"
p.xaxis.axis_label = "Issue ID"
# creating corcles at the end of each leaf
p.circle(x="leaves_x",
y="leaves_y",
fill_alpha=0.7,
size="leaves_size",
fill_color="leaves_color",
source=leaves_source)
# Adding lables to plot
labels = LabelSet(x="leaves_x",
y="leaves_y",
text="leaves_names",
angle=-60,
angle_units="deg",
y_offset=-10,
x_offset=-5,
source=leaves_source,
render_mode='canvas')
p.add_layout(labels)
# =============================================================================
# Set up widgets
# =============================================================================
# using the fake data source method for CustumJS callback
# look at https://stackoverflow.com/questions/38375961/throttling-in-bokeh-application/38379136
dend_threshold_slider = Slider(title="Dendrogram Threshold over distance",
value=2.0,
start=0.0,
end=10,
step=0.05,
callback_policy='mouseup',
callback=CustomJS(
args=dict(source=source_for_slider),
code="""
source.data = { value: [cb_obj.value] }
"""))
issues_list_input = TextInput(title="Enter issues IDs")
clipboard_text = TextInput(title="Selected issues IDs")
columns = [
TableColumn(field="leaves_names", title="Issue ID"),
TableColumn(field="clusters", title="Cluster"),
]
data_table = DataTable(source=leaves_source, columns=columns, width=280)
submit_button = Button(label="Submit", button_type="success")
universe_checkbox = CheckboxGroup(
labels=["Universe issues dendrogram (please allow 30s to complete)"])
# =============================================================================
# Set up callbacks
# =============================================================================
def update_issue_list():
"""
Callback function that updates issues_list dict, tfidf and Z
"""
global issues_dict
global Z
global dend_threshold
change_dend = False
# get new input issues list
issues_list = issues_list_input.value.strip()
issues_list = issues_list.replace('\n', ',').replace('|', ',').replace(
'\t', ',').replace(' ', ',').replace(',,', ',').split(",")
# universe is selected
if (0 in universe_checkbox.active):
# set issues membership to work for node color
# all universe nodes turns to light blue first
issues_dict["issues_membership"] = \
["lightblue"] * len(universe_issues_list)
# if there is any user list, turn their membership to black
for x in match(issues_list, universe_issues_list):
issues_dict["issues_membership"][x] = "black"
issues_dict["issues_list"] = universe_issues_list
Z = create_universe_Z(CODA_export_file, issues_df,
universe_issues_list, number_of_clusters)
# the epsilon (1e-10) was added for correct collering of dendrogram
# The exact Z[-k,2] value results in an extra cluster colored
dend_threshold = Z[-number_of_clusters, 2] + 1e-10
change_dend = True
elif (issues_list != [""]):
issues_dict["issues_list"] = issues_list
# if there is only user list, all of them have black memebership
issues_dict["issues_membership"] = ["black"] * len(issues_list)
# use match function to find the input issue list in the universe list
# avoid using pandas isin as it keep the reference df order not the
# input issues list order
df_tfidf = make_tfidf(issues_df.iloc[match(
issues_dict['issues_list'], issues_df.CASE_NUMBER.tolist()), ])
Z = make_Z(df_tfidf)
# arbitrary dend threshold for non-unverse dednrigrams
dend_threshold = max(Z[:, 2]) / 1.7
change_dend = True
if change_dend:
# clearing the static image
image_source.data = dict(url=[], text=[], text_x=[], text_y=[])
# change slider end point
dend_threshold_slider.update(
end=max(Z[:, 2]),
value=dend_threshold,
)
# update the dnedrogram
update_dend()
def update_dend():
"""
Callback function that change the dendrogram by calling make_R
Updates whole dendrogram, including legs colors
Updates the leaves, including leaves colors, clusters and membership
"""
# Use the unpdated dend_threshold
# Creat new dendrogram
R = make_R(Z, dend_threshold, issues_dict["issues_list"])
# Generate the new curve
dend_source.data = dict(x=R['icoord'] + [[0, max(max(R['icoord']))]],
y=R['dcoord'] +
[[dend_threshold, dend_threshold]],
leg_color=R['color_list'] + ["#FF0000"])
# find the correct order of nodes for the membership assignment
membership_index = match(R['ivl'], issues_dict["issues_list"])
# use the membership_index to find correct color and size for nodes
# base on their universe or user list membership
leaves_color = \
[issues_dict["issues_membership"][x] for x in membership_index]
# if the node color is black set size to 9 otherwise set it to 5
leaves_size = \
np.where(np.array(leaves_color) == "black", 9, 5).tolist()
leaves_source.data = dict(
leaves_x=list(range(5,
len(R["ivl"]) * 10, 10)),
leaves_y=[0] * len(R["ivl"]),
leaves_names=R['ivl'],
leaves_color=leaves_color,
leaves_size=leaves_size,
clusters=R['Clusters'],
)
def update_clipboard_text(attr, old, new):
"""
Callback function that put the selected issues in the clipboard text box
for exporting.
"""
clipboard_df = pd.DataFrame({
"Issues": [
leaves_source.data['leaves_names'][i]
for i in new['1d']['indices']
],
"Clusters":
[leaves_source.data['clusters'][i] for i in new['1d']['indices']]
})
# [leaves_source.data['leaves_names'][i] for i in new['1d']['indices']]
clipboard_text.value = " ".join(clipboard_df['Issues'])
clipboard_df.to_csv("clipboard_df.csv", sep=',', encoding='utf-8')
# use an intermidiate function to pass new threshold to dend update
def slider_delay(attr, old, new):
global dend_threshold
dend_threshold = source_for_slider.data["value"][0]
update_dend()
source_for_slider.on_change("data", slider_delay)
# calling call back function upon changes in interactive widgets
issues_list_input.on_change('value',
lambda attr, old, new: update_issue_list())
submit_button.on_click(update_issue_list)
#dend_threshold_slider.on_change('value', lambda attr, old, new: update_dend())
leaves_source.on_change('selected', update_clipboard_text)
universe_checkbox.on_change('active',
lambda attr, old, new: update_issue_list())
# =============================================================================
# Set up layouts and add to document
# =============================================================================
# making the descrption for the page
page_header = Div(text="""<h1>Data Quality Issue Clustering</h1>
Start creating your issues dendrogram by entering your issues
IDs. Fine tune your clusters by changing the <b>Dendrogram
Threshold</b>. (Click on the link for more information about
<a href="https://en.wikipedia.org/wiki/Dendrogram" target="_blank">dendrograms</a>).
The following DQ Direct fields were used for
this clustering: <b>Source System, Issue Detection Point,
Issue Summary, Detailed Description</b> and <b>Attribute Document
Header</b>. Check the tick box for <b>Universe issues
dendrogram</b> if you want to compare your issues list with
all other non-rejected and non-remediated issues.
""",
width=1300)
# have to move to directory file system to use static images
link_to_qlik = Div(text="""
<a href="https://www.google.com" target="_blank">
<img float:right src="static/qlik_link.png" alt="DQ Direct MI Dashboard" height="50" width="70", align="right">
<h3>DQ Direct MI Dashboard</h3>
</a>
""")
page_footer = Div(text="""Developed by
<a href="https://www.google.com">DQ Analytics</a>.
Any questions please contact \
<a href="https://www.google.com" target="_blank">Andrew McGeough</a> or
""",
width=1200)
# laying out widgets and figure
page_layout = column(
page_header,
row(
column(
widgetbox(
[issues_list_input, submit_button, universe_checkbox],
width=300), dend_threshold_slider,
Spacer(width=300, height=20), data_table,
Spacer(width=300, height=20), clipboard_text, link_to_qlik),
p), page_footer)
# Determine the selected feature
# Embed plot into HTML via Flask Render
script, div = components(page_layout)
return render_template("bokeh.html", script=script, div=div)
)
cluster_stats.width = 500
cluster_stats.height = 100
cluster_commonality = Div(
render_as_text=False,
text=generate_display_string("")
)
cluster_commonality.width = 500
cluster_commonality.height = 300
split_button = Button(label="Split", button_type="success", width=150)
no_split_button = Button(label="Keep", button_type="success", width=150)
start_button = Button(label="Start", button_type="success", width=150)
split_button.on_click(lambda: _fit_util(1))
no_split_button.on_click(lambda: _fit_util(0))
start_button.on_click(lambda: _fit_util(2))
inputs = widgetbox(text_input, cluster_stats, cluster_commonality)
clusters = _fixed_clusters
l = layout(
[
[
column(p_title, row(start_button, no_split_button, split_button), inputs),
p
]
]
)
curdoc().add_root(l)
def plot():
# FIGURES AND X-AXIS
fig1 = Figure(title = 'Energy', plot_width = WIDTH, plot_height = HEIGHT, tools = TOOLS)
timeticks = DatetimeTickFormatter(formats=dict(seconds =["%b%d %H:%M:%S"],
minutes =["%b%d %H:%M"],
hours =["%b%d %H:%M"],
days =["%b%d %H:%M"],
months=["%b%d %H:%M"],
years =["%b%d %H:%M %Y"]))
fig1.xaxis.formatter = timeticks
# INPUT WIDGETS
collection_list = CONN[DB].collection_names(include_system_collections=False)
gliders = sorted([platformID for platformID in collection_list if len(platformID)>2])
gliders = Select(title = 'PlatformID', value = gliders[0], options = gliders)
prev_glider = Button(label = '<')
next_glider = Button(label = '>')
glider_controlbox = HBox(children = [gliders, prev_glider, next_glider])
max_amphr = TextInput(title='Max AmpHrs', value='1040')
deadby_date = TextInput(title='Deadby Date', value='')
data_controlbox = HBox(max_amphr, deadby_date, width = 300)
control_box = HBox(glider_controlbox,
data_controlbox)
# DATA VARS
coulombs_raw = ColumnDataSource(dict(x=[],y=[]))
coulombs_ext = ColumnDataSource(dict(x=[],y=[]))
coulombs_per = ColumnDataSource(dict(x=[],y=[]))
# AXIS setup
fig1.yaxis.axis_label = 'Coulombs (AmpHr)'
fig1.extra_y_ranges = {'usage': Range1d(start=0, end=1200)}
# PLOT OBJECTS
fig1.line( 'x', 'y', source = coulombs_raw, legend = 'm_coulombs_amphr_total', color = 'blue')
fig1.circle('x', 'y', source = coulombs_raw, legend = 'm_coulombs_amphr_total', color = 'blue')
fig1.line( 'x', 'y', source = coulombs_ext, legend = 'projected', color = 'red')
#fig1.cross('x', 'y', source = coulombs_ext, legend = 'projected', size=10, color = 'red')
fig1.renderers.append(Span(name = 'maxamp_span', location = int(max_amphr.value), dimension = 'width', line_color= 'green', line_dash='dashed', line_width=2))
fig1.renderers.append(Span(name = 'maxamp_intersect', location = 1000*time.time(), dimension = 'height', line_color= 'green', line_dash='dashed', line_width=2))
fig1.legend[0].location = 'top_left'
fig1.legend[0].legend_padding = 30
# CALLBACK FUNCS
def update_coulombs(attrib,old,new):
g = gliders.value
coulombs_raw.data = load_sensor(g, 'm_coulomb_amphr_total')
#coulombs_per.data = moving_usage(coulombs_raw.data)
update_projection(None,None,None)
def update_projection(attrib,old,new):
g = gliders.value
try:
fig1.select('maxamp_span')[0].location = int(max_amphr.value)
coulombs_ext.data, deadby_date.value = calc_deadby_date(g, int(max_amphr.value))
fig1.select('maxamp_intersect')[0].location = coulombs_ext.data['x'][-1]
except Exception as e:
print('update_projection error',type(e),e)
#GLIDER SELECTS
def glider_buttons(increment):
ops = gliders.options
new_index = ops.index(gliders.value) + increment
if new_index >= len(ops):
new_index = 0
elif new_index < 0:
new_index = len(ops)-1
gliders.value = ops[new_index]
def next_glider_func():
glider_buttons(1)
def prev_glider_func():
glider_buttons(-1)
gliders.on_change('value', update_coulombs)
next_glider.on_click(next_glider_func)
prev_glider.on_click(prev_glider_func)
max_amphr.on_change('value', update_projection)
update_coulombs(None,None,None)
return vplot(control_box, fig1)
def plotting(self):
if self.debug:
self.debug_file = open("debug_output.txt", "w")
self.debug_file.write("Initialized plotting subroutine \n")
TOOLS="pan,wheel_zoom,box_zoom,reset,hover,previewsave"
tab_plots = []
self.all_elements = []
self.elements_comparison = []
for filename in self.filenames:
if "ITO" in filename:
tab_plots.append(self.mass_plotting(filename))
continue
data_dict = self.data_generation(filename)
self.data_test(data_dict)
name_check = data_dict["gen_info"]["DATA FILES"]
attr_id = name_check[1][4][:-3] + "_" + name_check[2][2]
self.attribute_ids.append(attr_id)
attr_extra_y_ranges = False
attr_extra_x_ranges = False
local_source_line = []
"""
create plots for each datafile and put them in a tab.
"""
y_axis_units = [x["y_unit"] for x in data_dict["data"]]
x_axis_units = [x["x_unit"] for x in data_dict["data"]]
figure_obj = figure(plot_width = 1000, plot_height = 800, y_axis_type = "log",
title = attr_id, tools = TOOLS)
#figure_obj.axes.major_label_text_font_size("12pt")
#figure_obj.major_label_text_font_size("12pt")
hover = figure_obj.select(dict(type = HoverTool))
hover.tooltips = [
("Element:", "@element"),
("(x, y):", "($x, $y)")]
self.figure_data.append((figure_obj, data_dict))
figure_obj.yaxis.axis_label = y_axis_units[0]
figure_obj.xaxis.axis_label = x_axis_units[0]
if not all(x == y_axis_units[0] for x in y_axis_units):
for unit, dataset in zip(y_axis_units, data_dict["data"]):
if not unit == y_axis_units[0]:
extra_y_ranges_exists = attr_extra_y_ranges
extra_y_ranges_exists = True
if self.debug:
self.debug_file.write("Added extra y-axis for file_id: %s, element: %s | New length %g \n"
%(attr_id, dataset["sample_element"], len(figure_obj.yaxis)))
figure_obj.extra_y_ranges = {"foo": Range1d(start = np.amin(dataset["y"]),
end = np.amax(dataset["y"]))}
figure_obj.add_layout(LogAxis(y_range_name = "foo", axis_label = unit), "right")
break
if not all(x == x_axis_units[0] for x in x_axis_units):
for unit, dataset in zip(x_axis_units, data_dict["data"]):
if not unit == x_axis_units[0]:
extra_x_ranges_exists = attr_extra_x_ranges
extra_x_ranges_exists = True
if self.debug:
self.debug_file.write("Added extra x-axis for file_id: %s, element: %s. | New length %g \n"
%(attr_id, dataset["sample_element"], len(figure_obj.yaxis)))
figure_obj.extra_x_ranges = {"bar": Range1d(start = np.amin(dataset["x"]),
end = np.amax(dataset["x"]))}
figure_obj.add_layout(LinearAxis(x_range_name = "bar", axis_label = unit), "above")
break
figure_obj.xaxis.axis_label = x_axis_units[0]
colour_list = Spectral11 + RdPu9 + Oranges9
colour_indices = [0, 2, 8, 10, 12, 14, 20, 22, 1, 3, 9, 11, 13, 15]
list_of_elements = []
source_list = []
line_list = []
for dataset, color_index in zip(data_dict["data"], colour_indices):
self.all_elements.append(dataset["sample_element"]) #strip isotope number
color = colour_list[color_index]
source = ColumnDataSource(data = dataset) #Datastructure for source of plotting
self.source_test(source)
list_of_elements.append(dataset["sample_element"])
line_glyph = figure_obj.line("x", "y", source = source,
line_width = 2,
line_color = color,
legend = dataset["sample_element"])
if self.debug:
self.debug_file.write("Create line object on figure %s at %s \n" %(id(figure_obj), id(line_glyph)))
line_list.append(line_glyph)
source_list.append(source)
local_source_line.append([[source, line] for source, line in zip(source_list, line_list)])
self.source_line.append(local_source_line)
#Calculations on the dataset
text_input_rsf = TextInput(value = "default", title = "RSF or SF (at/cm^3): ")
do_integral_button = Button(label = "Calibration integral")
smoothing_button = Button(label = "smth selct elem")
matplot_button = Button(label = "Create matplotlib fig")
text_input_sputter = TextInput(value = "default", title = "Sputter speed: number unit")
text_input_crater_depth = TextInput(value = "default", title = "Depth of crater in: number unit")
radio_group = RadioGroup(labels = list_of_elements, active=0)
text_input_xval_integral = TextInput(value = "0", title = "x-delimiter ")
text_input_dose = TextInput(value = "0", title = "Dose[cm^-2] ")
#Save files for later use
save_flexDPE_button = Button(label = "Save element for FlexPDE")
save_all_flexDPE_button = Button(label = "Save all elements for FlexPDE")
save_textfile_button = Button(label = "Sava Data in textfile")
#Pointers to methods on click / change handlers
radio_group.on_change("active", lambda attr, old, new: None)
matplot_button.on_click(lambda source_list = source_list:
self.matplotlib_export(source_list))
do_integral_button.on_click(lambda
source_list = source_list,
line_list = line_list,
source_line = self.source_line,
figure_data = self.figure_data,
data_dict = data_dict,
radio = radio_group,
x_box = text_input_xval_integral,
dose = text_input_dose,
extra_y_ranges = attr_extra_y_ranges:
self.integrate(data_dict, source_list, line_list, source_line, figure_data, radio, x_box, dose, extra_y_ranges))
smoothing_button.on_click(lambda
source_list = source_list,
radio = radio_group,
data_dict = data_dict,
x_box = text_input_xval_integral:
self.smoothing(source_list, data_dict, radio, x_box) )
save_flexDPE_button.on_click(lambda
source_list = source_list,
attrname = attr_id,
radio = radio_group:
self.write_to_flexPDE(source_list, attrname, radio))
save_all_flexDPE_button.on_click(lambda
source_list = source_list,
attrname = attr_id:
self.write_all_to_flexPDE(source_list, attrname))
save_textfile_button.on_click(lambda
data_dict = data_dict,
source_list = source_list,
attrname = attr_id,
radio = radio_group:
self.write_new_datafile(data_dict, source_list, attrname,radio))
text_input_rsf.on_change("value", lambda attr, old, new,
radio = radio_group,
data_dict = data_dict,
figure = figure_obj,
source_list = source_list,
text_input = text_input_rsf,
line_list = line_list,
which = "rsf":
self.update_data(line_list, data_dict, source_list, figure, radio, text_input, new, which))
text_input_sputter.on_change("value", lambda attr, old, new,
radio = radio_group,
data_dict = data_dict,
figure = figure_obj,
source_list = source_list,
text_input = text_input_sputter,
which = "sputter":
self.update_data(data_dict, source_list, figure, radio, text_input, new, which))
text_input_crater_depth.on_change("value", lambda attr, old, new,
radio = radio_group,
data_dict = data_dict,
source_list = source_list,
figure = figure_obj,
text_input = text_input_crater_depth,
which = "crater_depth":
self.update_data(data_dict, source_list, figure, radio, text_input, new, which))
#Initialization of actual plotting.
tab_plots.append(Panel(child = hplot(figure_obj,
vform(
vform(radio_group, save_flexDPE_button, save_all_flexDPE_button, save_textfile_button, matplot_button),
vform(text_input_rsf, smoothing_button, text_input_sputter, text_input_crater_depth)
),
vform(text_input_xval_integral, text_input_dose, do_integral_button)),
title = attr_id))
"""
Check to see if one or more element exists in the samples and creat a comparison plot for each
of those elements.
"""
for element in self.all_elements:
checkers = list(self.all_elements)
checkers.remove(element)
if element in checkers and not element in self.elements_comparison:
self.elements_comparison.append(element)
"""create plots for each element that is to be compared """
for comparison_element in self.elements_comparison:
figure_obj = figure(plot_width = 1000, plot_height = 800, y_axis_type = "log", title = comparison_element, tools = TOOLS)
#figure_obj.xaxis.major_label_text_font_size("12pt")
#figure_obj.yaxis.major_label_text_font_size("12pt")
y_axis_units = []
x_axis_units = []
comparison_datasets = []
for data_dict_iter in self.column(self.figure_data, 1):
for dataset in data_dict_iter["data"]:
if dataset["sample_element"] == comparison_element:
comparison_datasets.append(dataset)
y_axis_units.append(dataset["y_unit"])
x_axis_units.append(dataset["x_unit"])
figure_obj.xaxis.axis_label = comparison_datasets[-1]["x_unit"]
figure_obj.yaxis.axis_label = comparison_datasets[-1]["y_unit"]
if not all(x == y_axis_units[-1] for x in y_axis_units):
for unit, data in zip(y_axis_units, comparison_datasets):
if not unit == y_axis_units[-1]:
figure_obj.extra_y_ranges = {"foo": Range1d(start = np.amin(data["y"]),
end = np.amax(data["y"]))}
figure_obj.add_layout(LogAxis(y_range_name = "foo", axis_label = unit), "right")
break
if not all(x == x_axis_units[-1] for x in x_axis_units):
for unit, data in zip(x_axis_units, comparison_datasets):
if not unit == x_axis_units[-1]:
figure_obj.extra_x_ranges = {"bar": Range1d(start = np.amin(data["x"]),
end = np.amax(data["x"]))}
figure_obj.add_layout(LinearAxis(x_range_name = "bar", axis_label = unit), "above")
break
active_sources = []
for data_dict, source_line_nested, attr_id, color_index in zip(self.column(self.figure_data, 1), self.source_line, self.attribute_ids, colour_indices):
for dataset, source_lis_coup, in zip(data_dict["data"], source_line_nested[0]):
source_local = source_lis_coup[0]
active_sources.append(source_local)
self.source_test(source_local)
self.source_dataset_test(source_local, dataset)
if dataset["sample_element"] == comparison_element:
color = colour_list[color_index]
"""
Logic that ensures that plots get put with correspoinding axes.
"""
if dataset["x_unit"] != x_axis_units[-1] or dataset["y_unit"] != y_axis_units[-1]:
if dataset["x_unit"] != x_axis_units[-1] and dataset["y_unit"] != y_axis_units[-1]:
name_check = data_dict["gen_info"]["DATA FILES"]
attr_id = name_check[1][4][:-3] + "_" + name_check[2][2]
figure_obj.line("x", "y", source = source_local,
line_width = 2,
line_color = color,
legend = attr_id,
x_range_name = "bar",
y_range_name = "foo")
elif dataset["x_unit"] != x_axis_units[-1]:
figure_obj.line("x", "y", source = source_local,
line_width = 2,
line_color = color,
legend = attr_id,
x_range_name = "bar")
else:
figure_obj.line("x", "y", source = source_local,
line_width = 2,
line_color = color,
legend = attr_id,
y_range_name = "foo")
else:
figure_obj.line("x", "y", source = source_local,
line_width = 2,
line_color = color,
legend = attr_id)
matplot_button = Button(label = "Create matplotlib fig")
save_all_flexDPE_button = Button(label = "Save all elements for FlexPDE")
matplot_button.on_click(lambda source_list = active_sources:
self.matplotlib_export(source_list))
save_all_flexDPE_button.on_click(lambda
source_list = active_sources,
attrname = comparison_element:
self.write_all_to_flexPDE(source_list, attrname))
tab_plots.append(Panel(child = hplot(figure_obj, vform(save_all_flexDPE_button, matplot_button)),
title = comparison_element))
tabs = Tabs(tabs = tab_plots)
#curdoc().add_root(tabs)
session = push_session(curdoc())
session.show()
session.loop_until_closed()
def make_tab():
# Slider to select width of bin
pastdays_select = RangeSlider(start=0,
end=999,
value=(0, 999),
step=1,
title='Past Days',
sizing_mode="stretch_both")
# Slider to select buffer size
bufferdays_select = Slider(start=.01,
end=9,
value=0.01,
step=.1,
title='Buffer Size (days)',
sizing_mode="stretch_both")
# Re-read
refresh_button = Button(label="Time window and buffer are up to date",
button_type="success",
sizing_mode="stretch_both")
refresh_button.disabled = True
# read data
flights, available_carriers = read(pastdays_select, bufferdays_select)
# CheckboxGroup to select carrier to display
locationcodes = np.unique(
['.'.join(seedid.split('.')[:3]) for seedid in available_carriers])
selections = []
for locationcode in locationcodes[:maxnstation]:
matching = [s for s in available_carriers if locationcode in s]
active = [i for i, m in enumerate(matching)] # if "Z" == m[-1]]
selections += [
CheckboxButtonGroup(labels=matching,
active=active,
sizing_mode="stretch_both")
]
#selections += [MultiSelect(#title="Option:",
# value=matching,
# active=active)
# Find the initially selected carrieres
initial_carriers = [s.labels[i] for s in selections for i in s.active]
# Slider to select width of bin
binwidth_select = Slider(start=16,
end=160,
step=16,
value=80,
title='Bin number',
sizing_mode="stretch_both")
# RangeSlider control to select start and end of plotted delays
range_select = RangeSlider(start=-1,
end=999,
value=(-.2, 99),
step=.1,
title='Range (sec)',
sizing_mode="stretch_both")
# Switch from lines to hists
type_switch = RadioButtonGroup(labels=["Histogram", "Cumulated dist."],
active=0,
sizing_mode="stretch_both")
# Find the initially selected carrieres
plottype = type_switch.labels[type_switch.active]
src = {}
for output in ['Latencies', 'Delays', 'PSD']:
src[output] = make_dataset(flights,
initial_carriers,
range_start=range_select.value[0],
range_end=range_select.value[1],
bin_width=binwidth_select.value,
output=output,
plottype=plottype)
callback = partial(update,
output='Delays',
type_switch=type_switch,
flights=flights,
src=src,
selections=selections,
range_select=range_select,
binwidth_select=binwidth_select)
callbacklat = partial(update,
output='Latencies',
type_switch=type_switch,
flights=flights,
src=src,
range_select=range_select,
selections=selections,
binwidth_select=binwidth_select)
callbackpsd = partial(update,
output='PSD',
type_switch=type_switch,
flights=flights,
src=src,
range_select=range_select,
selections=selections,
binwidth_select=binwidth_select)
callbackneedsread = partial(needsreadupdate, refresh_button=refresh_button)
callbackread = partial(readupdate,
src=src,
selections=selections,
range_select=range_select,
type_switch=type_switch,
binwidth_select=binwidth_select,
pastdays_select=pastdays_select,
bufferdays_select=bufferdays_select,
refresh_button=refresh_button)
[
s.on_change('active', callback, callbacklat, callbackpsd)
for s in selections
]
type_switch.on_change('active', callback, callbacklat, callbackpsd)
binwidth_select.on_change('value', callback, callbacklat, callbackpsd)
range_select.on_change('value', callback, callbacklat, callbackpsd)
pastdays_select.on_change('value', callbackneedsread)
bufferdays_select.on_change('value', callbackneedsread)
refresh_button.on_click(callbackread)
p = {}
for output in ['PSD', 'Latencies', 'Delays']:
p[output] = make_plot(src[output], output=output)
# Create a row layout
graphs = [p[k] for k in p]
controls = [
type_switch, binwidth_select, range_select, refresh_button,
pastdays_select, bufferdays_select, *selections[:maxnstation]
]
graphslayout = column(children=graphs, sizing_mode="stretch_both")
controlslayout = column(
children=controls,
sizing_mode='fixed', #stretch_width',
width=400,
)
layout = row(children=[graphslayout, controlslayout],
sizing_mode='stretch_both')
# Make a tab with the layout
return Panel(child=layout, title='Channels')
session = push_session(curdoc())
def start_handler():
global playing
if not playing:
curdoc().add_periodic_callback(update, 50)
playing = True
def stop_handler():
global playing
if playing:
curdoc().remove_periodic_callback(update)
playing = False
button_start = Button(label="Start", button_type="success")
button_start.on_click(start_handler)
button_stop = Button(label="Stop", button_type="danger")
button_stop.on_click(stop_handler)
controls = hplot(button_start, button_stop)
layout = vplot(controls, p)
@cosine(w=0.03)
def update(step):
if playing:
r2.data_source.data["y"] = y * step
r2.glyph.line_alpha = 1 - 0.8 * abs(step)
playing = True
curdoc().add_periodic_callback(update, 50)
columns = [
TableColumn(field="time", title="Time"),
TableColumn(field="msg", title="Msg", width = 600)]
data_table = DataTable(source=textsource, columns=columns, width=600)
button_save = Button(label="Save Run", button_type = 'warning')
def button_save_handler():
global read_data
print('button save worked')
run_modes= {'bat_type':settings['bat_type'], 'chrg_type':settings['chrg_type'], 'nominal_mah':settings['nominal_mah'],
'DC_or_CD':settings['DC_or_CD'], 'cycles':settings['cycles'], 'cells':settings['cells'], 'run_text':settings['run_text']}
excel_out = imax_0.write_excel_file(run_modes, settings['final_read'], read_data, settings['settings_dict'])
msg = 'Data saved to: ' + excel_out
print(msg)
text_update(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"), msg)
button_save.on_click(button_save_handler)
def text_update(t, msg):
global data_table
print('time and msg: ', t, msg)
new_data = dict(time=[t], msg=[msg],)
textsource.stream(new_data, 20) #adding the value is a scrolloff lines
data_table.update()
import imax_0
def check_device_status():
global device_dict
#used by startstop btn handler to determine if imax start btn was pressed, or imax running.
#global device_dict
EndPt_out = device_dict['EndPt_out']
value=(1, 24),
step=1,
show_value=False,
tooltips=False)
date_widget.on_change('value', date_function)
se_select = Select(title="Start/End Locations",
value="Start",
options=['Start', 'End'])
span_radio = RadioButtonGroup(labels=["Lock Map Area", "Auto Adjust Map Area"],
active=0)
bt = Button(label='Update Plot')
bt.on_click(update_click)
alpha_range_slider = Slider(start=0,
end=1,
value=0.4,
step=.1,
title="Spot Transparency")
size_range_slider = Slider(start=4, end=50, value=4, step=1, title="Spot Size")
def alpha_size(attr, old, new):
circle_plot.glyph.size = size_range_slider.value
circle_plot.glyph.fill_alpha = alpha_range_slider.value
TableColumn(field="v", title="Value"),
]
dataTable = DataTable(source=source, columns=columns, width=800, height=600)
tab2 = Panel(child=dataTable, title="Table")
tabs = Tabs(tabs=[tab1, tab2 ])
# tabs.css_classes = ["hide"]
autoUpdateCheckbox = CheckboxGroup(
labels=["Auto Update Data Source (every 15s)"], active=[])
autoUpdateCheckbox.disabled = True
gatewayControl.on_change('value', lambda attr, old, new: update_device())
deviceControl.on_change('value', lambda attr, old, new: update_indicator())
submitButton.on_click(lambda: callback())
autoUpdateCheckbox.on_click(lambda attr: auto_update(attr))
sizing_mode = 'fixed' # 'scale_width' also looks nice with this example
inputs = widgetbox(*controls, sizing_mode=sizing_mode, name="widgets")
plotwidget = widgetbox([autoUpdateCheckbox, tabs], sizing_mode=sizing_mode, name="plotwidget")
mainLayout = layout(children=[
[inputs, plotwidget]
], sizing_mode=sizing_mode, name="mainLayout")
doc.add_root(mainLayout)
doc.title = "ACME IoT Analytics"
def epoch_to_datetime(epoch):
"""
def modify_doc(doc):
reset_output(state=None)
print("Entering modify_doc ...")
global ecg_idx, ecg_filtered, crr_idx
df = pd.read_csv(os.path.join(
app.config['UPLOAD_FOLDER'], app.config['CUR_FILE']))
ecg_clean = df['P02S ECG']
ecg_idx = [i for i in range(len(ecg_clean))]
ecg_filtered = bandpass(ecg_clean, lf, hf, 250, 2, False)
ecg_filtered = np.interp(
ecg_filtered, (ecg_filtered.min(), ecg_filtered.max()), (-1, +1))
# ecg_filtered = scale(ecg_filtered, axis=0, with_mean=True,
# with_std=True, copy=True)
min_g = min(ecg_filtered)
max_g = max(ecg_filtered)
file_name_div = Div(text='Processing on file: ' +
app.config['CUR_FILE'].split('/')[-1])
div = Div(text="Quality")
radio_group = RadioGroup(
labels=["Good", "Boderline", "Poor"], active=2)
button_nxt_unit = Button(label="Next Unit", button_type="success")
skip_unit = RadioGroup(
labels=["Save This Unit", "Skip This Unit"], active=0)
marker_line_st = ColumnDataSource(data=dict(x=[0, 0], y=[min_g, max_g]))
marker_line_en = ColumnDataSource(data=dict(x=[0, 0], y=[min_g, max_g]))
s2 = ColumnDataSource(data=dict(x=[], y=[]))
sx = figure(width=1300, height=300, title="ecg_filtered "+str(lf) +
"-"+str(hf)+"Hz", x_axis_label='time', y_axis_label='acc')
sx.line(ecg_idx, ecg_filtered, legend="ecg_filtered "+str(lf) +
"-"+str(hf)+"Hz", line_color="red", line_width=1)
sx.line('x', 'y', source=marker_line_st,
legend="current unit ", line_color="blue", line_width=1)
sx.line('x', 'y', source=marker_line_en,
legend="current unit ", line_color="blue", line_width=1)
sx1 = figure(width=800, height=300, title="ecg_filtered "+str(lf) +
"-"+str(hf)+"Hz", x_axis_label='time', y_axis_label='acc')
sx1.line('x', 'y', source=s2, legend="ecg_filtered unit "+str(lf) +
"-"+str(hf)+"Hz", line_color="red", line_width=1)
peaks, _ = find_peaks(ecg_filtered, distance=150)
crr_idx = 0
marked_file = open(os.path.join(
app.config['MARKED_FOLDER'], app.config['CUR_FILE'][:-4] + '_' +
str(datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")) + '.txt'), 'w')
marked_file.write(app.config['CUR_FILE'] + '\n')
marked_file.write('P02S ECG' + '\n')
marked_file.write(str(len(peaks) - 1) + '\n')
res_df = {}
def my_nxt_unit_handler():
global crr_idx, ecg_filtered
if (crr_idx < len(peaks) - 1):
#save unit data
if(skip_unit.active == 0 and crr_idx > 0):
out_fn = os.path.join(
app.config['MARKED_FOLDER'], radio_group.labels[radio_group.active],
app.config['CUR_FILE'][:-4]+'_P2SECG_'+str(crr_idx)+'.json')
print(out_fn)
with open(out_fn, "w") as write_file:
json.dump(res_df, write_file)
st = int(max(0, peaks[crr_idx] -
(peaks[crr_idx + 1] - peaks[crr_idx]) / 2))
en = int(
min(peaks[crr_idx] + (peaks[crr_idx + 1] - peaks[crr_idx]) / 2, ecg_idx[-1]))
unit_idx = ecg_idx[st:en]
unit_data = ecg_filtered[st:en]
s2.data['x'] = unit_idx
s2.data['y'] = unit_data
marker_line_st.data['x'] = [st, st]
marker_line_en.data['x'] = [en, en]
print("crr marked quality: ", radio_group.labels[radio_group.active])
marked_file.write(str(st) + '\t' + str(en) + '\t' + str(en - st) + '\t' +
str(radio_group.labels[radio_group.active]) + '\n')
res_df['file_name'] = app.config['CUR_FILE']
res_df['unit_number'] = crr_idx
res_df['size'] = en - st +1
res_df['data'] = list(unit_data)
res_df['signal_name'] = 'P02S ECG'
res_df['quality'] = radio_group.labels[radio_group.active]
crr_idx += 1
if crr_idx == len(peaks)-1:
marked_file.close()
button_nxt_unit.on_click(my_nxt_unit_handler)
graphs = column(
list(itertools.chain.from_iterable([[sx, sx1, widgetbox(div)]])))
doc.add_root(widgetbox(file_name_div))
doc.add_root(graphs)
doc.add_root(row(widgetbox(radio_group), widgetbox(
button_nxt_unit), widgetbox(skip_unit)))
doc.theme = Theme(filename=os.path.join(THIS_FILE_PATH, "theme.yaml"))
auth_pass.value, 'generic')
if ACCESS_GRANTED:
auth_button.label = 'Access Granted'
auth_button.button_type = 'success'
curdoc().clear()
curdoc().add_root(tabs)
else:
auth_button.label = 'Failed'
auth_button.button_type = 'danger'
time.sleep(3)
auth_button.label = 'Authenticate'
auth_button.button_type = 'warning'
auth_user = TextInput(value='', title='User Name:', width=150)
auth_pass = PasswordInput(value='', title='Password:', width=150)
auth_button = Button(label="Authenticate", button_type="warning", width=100)
auth_button.on_click(auth_button_click)
layout_login = row(auth_user, Spacer(width=50), auth_pass, Spacer(width=50),
auth_button)
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Create the document Bokeh server will use to generate the web page
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if ACCESS_GRANTED:
curdoc().add_root(tabs)
else:
curdoc().add_root(layout_login)
curdoc().title = "DVH Analytics"
def x_change(attr, old, new):
params["x"] = new
update_data()
return None
def y_change(attr, old, new):
params["y"] = new
update_data()
return None
file_selector.on_change("value", file_change)
x_selector.on_change("value", x_change)
y_selector.on_change("value", y_change)
record_button.on_click(print_data)
r.data_source.on_change('selected', update_lasso)
widgets = widgetbox(file_selector,
x_selector,
y_selector,
record_button,
stats,
width=300)
layout = column(row(p, pv, widgets), row(ph, Spacer(width=500, height=200)))
curdoc().add_root(layout)
curdoc().title = "COUNTATRON 5000"
print("BOKEH GO!")
def generate_plots(ulog, px4_ulog, db_data, vehicle_data, link_to_3d_page):
""" create a list of bokeh plots (and widgets) to show """
plots = []
data = ulog.data_list
# COMPATIBILITY support for old logs
if any(elem.name == 'vehicle_air_data'
or elem.name == 'vehicle_magnetometer' for elem in data):
baro_alt_meter_topic = 'vehicle_air_data'
magnetometer_ga_topic = 'vehicle_magnetometer'
else: # old
baro_alt_meter_topic = 'sensor_combined'
magnetometer_ga_topic = 'sensor_combined'
for topic in data:
if topic.name == 'system_power':
# COMPATIBILITY: rename fields to new format
if 'voltage5V_v' in topic.data: # old (prior to PX4/Firmware:213aa93)
topic.data['voltage5v_v'] = topic.data.pop('voltage5V_v')
if 'voltage3V3_v' in topic.data: # old (prior to PX4/Firmware:213aa93)
topic.data['voltage3v3_v'] = topic.data.pop('voltage3V3_v')
# initialize flight mode changes
flight_mode_changes = get_flight_mode_changes(ulog)
# VTOL state changes
vtol_states = None
try:
cur_dataset = ulog.get_dataset('vehicle_status')
if np.amax(cur_dataset.data['is_vtol']) == 1:
vtol_states = cur_dataset.list_value_changes('in_transition_mode')
# find mode after transitions (states: 1=transition, 2=FW, 3=MC)
for i in range(len(vtol_states)):
if vtol_states[i][1] == 0:
t = vtol_states[i][0]
idx = np.argmax(cur_dataset.data['timestamp'] >= t) + 1
vtol_states[i] = (t, 2 +
cur_dataset.data['is_rotary_wing'][idx])
vtol_states.append((ulog.last_timestamp, -1))
except (KeyError, IndexError) as error:
vtol_states = None
# Heading
curdoc().template_variables['title_html'] = get_heading_html(
ulog, px4_ulog, db_data, link_to_3d_page)
# info text on top (logging duration, max speed, ...)
curdoc().template_variables['info_table_html'] = \
get_info_table_html(ulog, px4_ulog, db_data, vehicle_data, vtol_states)
curdoc().template_variables['error_labels_html'] = get_error_labels_html()
hardfault_html = get_hardfault_html(ulog)
if hardfault_html is not None:
curdoc().template_variables['hardfault_html'] = hardfault_html
# FIXME: for now, we use Google maps directly without bokeh, because it's not working reliably
# GPS map
# gps_plots = []
# gps_titles = []
# plot = plot_map(ulog, plot_config, map_type='google', api_key =
# get_google_maps_api_key(), setpoints=False)
# if plot is not None:
# gps_plots.append(plot)
# gps_titles.append('GPS Map: Satellite')
#
# plot = plot_map(ulog, plot_config, map_type='plain', setpoints=True)
# if plot is not None:
# gps_plots.append(plot)
# gps_titles.append('GPS Map: Plain')
#
# data_plot = DataPlot2D(data, plot_config, 'vehicle_local_position',
# x_axis_label = '[m]', y_axis_label='[m]', plot_height='large')
# data_plot.add_graph('y', 'x', colors2[0], 'Estimated')
# data_plot.change_dataset('vehicle_local_position_setpoint')
# data_plot.add_graph('y', 'x', colors2[1], 'Setpoint')
# if data_plot.finalize() is not None:
# gps_plots.append(data_plot.bokeh_plot)
# gps_titles.append('Local Position')
#
# if len(gps_plots) >= 2:
# tabs = []
# for i in range(len(gps_plots)):
# tabs.append(Panel(child=gps_plots[i], title=gps_titles[i]))
# gps_plot_height=plot_config['plot_height']['large'] + 30
# plots.append(Tabs(tabs=tabs, width=plot_width, height=gps_plot_height))
# elif len(gps_plots) == 1:
# plots.extend(gps_plots)
if is_running_locally():
# show the google maps plot via Bokeh, since the one in the html
# template does not work locally (we disable it further down)
map_plot = plot_map(ulog,
plot_config,
map_type='google',
api_key=get_google_maps_api_key(),
setpoints=False)
if map_plot is not None:
plots.append(map_plot)
# Position plot
data_plot = DataPlot2D(data,
plot_config,
'vehicle_local_position',
x_axis_label='[m]',
y_axis_label='[m]',
plot_height='large')
data_plot.add_graph('y',
'x',
colors2[0],
'Estimated',
check_if_all_zero=True)
if not data_plot.had_error: # vehicle_local_position is required
data_plot.change_dataset('vehicle_local_position_setpoint')
data_plot.add_graph('y', 'x', colors2[1], 'Setpoint')
# groundtruth (SITL only)
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph('y', 'x', color_gray, 'Groundtruth')
# GPS + position setpoints
plot_map(ulog,
plot_config,
map_type='plain',
setpoints=True,
bokeh_plot=data_plot.bokeh_plot)
if data_plot.finalize() is not None:
plots.append(data_plot.bokeh_plot)
if not is_running_locally(
): # do not enable Google Map if running locally
curdoc().template_variables['has_position_data'] = True
# initialize parameter changes
changed_params = None
if not 'replay' in ulog.msg_info_dict: # replay can have many param changes
if len(ulog.changed_parameters) > 0:
changed_params = ulog.changed_parameters
plots.append(None) # save space for the param change button
### Add all data plots ###
x_range_offset = (ulog.last_timestamp - ulog.start_timestamp) * 0.05
x_range = Range1d(ulog.start_timestamp - x_range_offset,
ulog.last_timestamp + x_range_offset)
# Altitude estimate
data_plot = DataPlot(data,
plot_config,
'vehicle_gps_position',
y_axis_label='[m]',
title='Altitude Estimate',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([lambda data: ('alt', data['alt'] * 0.001)],
colors8[0:1], ['GPS Altitude'])
data_plot.change_dataset(baro_alt_meter_topic)
data_plot.add_graph(['baro_alt_meter'], colors8[1:2],
['Barometer Altitude'])
data_plot.change_dataset('vehicle_global_position')
data_plot.add_graph(['alt'], colors8[2:3], ['Fused Altitude Estimation'])
data_plot.change_dataset('position_setpoint_triplet')
data_plot.add_circle(['current.alt'],
[plot_config['mission_setpoint_color']],
['Altitude Setpoint'])
data_plot.change_dataset('actuator_controls_0')
data_plot.add_graph([lambda data: ('thrust', data['control[3]'] * 100)],
colors8[6:7], ['Thrust [0, 100]'])
plot_flight_modes_background(data_plot, flight_mode_changes, vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# Roll/Pitch/Yaw angle & angular rate
for axis in ['roll', 'pitch', 'yaw']:
# angle
axis_name = axis.capitalize()
data_plot = DataPlot(data,
plot_config,
'vehicle_attitude',
y_axis_label='[deg]',
title=axis_name + ' Angle',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([lambda data: (axis, np.rad2deg(data[axis]))],
colors2[0:1], [axis_name + ' Estimated'],
mark_nan=True)
data_plot.change_dataset('vehicle_attitude_setpoint')
data_plot.add_graph(
[lambda data: (axis + '_d', np.rad2deg(data[axis + '_d']))],
colors2[1:2], [axis_name + ' Setpoint'],
mark_nan=True,
use_step_lines=True)
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph([lambda data: (axis, np.rad2deg(data[axis]))],
[color_gray], [axis_name + ' Groundtruth'])
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# rate
data_plot = DataPlot(data,
plot_config,
'vehicle_attitude',
y_axis_label='[deg/s]',
title=axis_name + ' Angular Rate',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(
[lambda data: (axis + 'speed', np.rad2deg(data[axis + 'speed']))],
colors3[0:1], [axis_name + ' Rate Estimated'],
mark_nan=True)
data_plot.change_dataset('vehicle_rates_setpoint')
data_plot.add_graph([lambda data: (axis, np.rad2deg(data[axis]))],
colors3[1:2], [axis_name + ' Rate Setpoint'],
mark_nan=True,
use_step_lines=True)
axis_letter = axis[0].upper()
rate_int_limit = '(*100)'
# this param is MC/VTOL only (it will not exist on FW)
rate_int_limit_param = 'MC_' + axis_letter + 'R_INT_LIM'
if rate_int_limit_param in ulog.initial_parameters:
rate_int_limit = '[-{0:.0f}, {0:.0f}]'.format(
ulog.initial_parameters[rate_int_limit_param] * 100)
data_plot.change_dataset('rate_ctrl_status')
data_plot.add_graph(
[lambda data: (axis, data[axis + 'speed_integ'] * 100)],
colors3[2:3], [axis_name + ' Rate Integral ' + rate_int_limit])
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph(
[lambda data: (axis + 'speed', np.rad2deg(data[axis + 'speed']))],
[color_gray], [axis_name + ' Rate Groundtruth'])
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# Local position
for axis in ['x', 'y', 'z']:
data_plot = DataPlot(data,
plot_config,
'vehicle_local_position',
y_axis_label='[m]',
title='Local Position ' + axis.upper(),
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([axis],
colors2[0:1], [axis.upper() + ' Estimated'],
mark_nan=True)
data_plot.change_dataset('vehicle_local_position_setpoint')
data_plot.add_graph([axis],
colors2[1:2], [axis.upper() + ' Setpoint'],
mark_nan=True,
use_step_lines=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# Velocity
data_plot = DataPlot(data,
plot_config,
'vehicle_local_position',
y_axis_label='[m/s]',
title='Velocity',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(['vx', 'vy', 'vz'], colors8[0:3], ['X', 'Y', 'Z'])
data_plot.change_dataset('vehicle_local_position_setpoint')
data_plot.add_graph(['vx', 'vy', 'vz'],
[colors8[5], colors8[4], colors8[6]],
['X Setpoint', 'Y Setpoint', 'Z Setpoint'],
use_step_lines=True)
plot_flight_modes_background(data_plot, flight_mode_changes, vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# Visual Odometry (only if topic found)
if any(elem.name == 'vehicle_visual_odometry' for elem in data):
# Vision position
data_plot = DataPlot(data,
plot_config,
'vehicle_visual_odometry',
y_axis_label='[m]',
title='Visual Odometry Position',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(['x', 'y', 'z'],
colors3, ['X', 'Y', 'Z'],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph(
['x', 'y', 'z'], colors8[2:5],
['Groundtruth X', 'Groundtruth Y', 'Groundtruth Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# Vision velocity
data_plot = DataPlot(data,
plot_config,
'vehicle_visual_odometry',
y_axis_label='[m]',
title='Visual Odometry Velocity',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(['vx', 'vy', 'vz'],
colors3, ['X', 'Y', 'Z'],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph(
['vx', 'vy', 'vz'], colors8[2:5],
['Groundtruth VX', 'Groundtruth VY', 'Groundtruth VZ'])
if data_plot.finalize() is not None: plots.append(data_plot)
# Vision attitude
data_plot = DataPlot(data,
plot_config,
'vehicle_visual_odometry',
y_axis_label='[deg]',
title='Visual Odometry Attitude',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
lambda data: ('roll', np.rad2deg(data['roll'])), lambda data:
('pitch', np.rad2deg(data['pitch'])), lambda data:
('yaw', np.rad2deg(data['yaw']))
],
colors3, ['Roll', 'Pitch', 'Yaw'],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph(
[
lambda data: ('roll', np.rad2deg(data['roll'])), lambda data:
('pitch', np.rad2deg(data['pitch'])), lambda data:
('yaw', np.rad2deg(data['yaw']))
], colors8[2:5],
['Roll Groundtruth', 'Pitch Groundtruth', 'Yaw Groundtruth'])
# Vision attitude rate
data_plot = DataPlot(data,
plot_config,
'vehicle_visual_odometry',
y_axis_label='[deg]',
title='Visual Odometry Attitude Rate',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
lambda data:
('rollspeed', np.rad2deg(data['rollspeed'])), lambda data:
('pitchspeed', np.rad2deg(data['pitchspeed'])), lambda data:
('yawspeed', np.rad2deg(data['yawspeed']))
],
colors3, ['Roll Rate', 'Pitch Rate', 'Yaw Rate'],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
data_plot.change_dataset('vehicle_groundtruth')
data_plot.add_graph([
lambda data:
('rollspeed', np.rad2deg(data['rollspeed'])), lambda data:
('pitchspeed', np.rad2deg(data['pitchspeed'])), lambda data:
('yawspeed', np.rad2deg(data['yawspeed']))
], colors8[2:5], [
'Roll Rate Groundtruth', 'Pitch Rate Groundtruth',
'Yaw Rate Groundtruth'
])
if data_plot.finalize() is not None: plots.append(data_plot)
# Airspeed vs Ground speed: but only if there's valid airspeed data
try:
cur_dataset = ulog.get_dataset('airspeed')
if np.amax(cur_dataset.data['indicated_airspeed_m_s']) > 0.1:
data_plot = DataPlot(data,
plot_config,
'vehicle_global_position',
y_axis_label='[m/s]',
title='Airspeed',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
lambda data: ('groundspeed_estimated',
np.sqrt(data['vel_n']**2 + data['vel_e']**2))
], colors3[2:3], ['Ground Speed Estimated'])
data_plot.change_dataset('airspeed')
data_plot.add_graph(['indicated_airspeed_m_s'], colors2[0:1],
['Airspeed Indicated'])
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
except (KeyError, IndexError) as error:
pass
# manual control inputs
# prefer the manual_control_setpoint topic. Old logs do not contain it
if any(elem.name == 'manual_control_setpoint' for elem in data):
data_plot = DataPlot(data,
plot_config,
'manual_control_setpoint',
title='Manual Control Inputs (Radio or Joystick)',
plot_height='small',
y_range=Range1d(-1.1, 1.1),
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
'y', 'x', 'r', 'z', lambda data:
('mode_slot', data['mode_slot'] / 6), 'aux1', 'aux2', lambda data:
('kill_switch', data['kill_switch'] == 1)
], colors8, [
'Y / Roll', 'X / Pitch', 'Yaw', 'Throttle [0, 1]', 'Flight Mode',
'Aux1', 'Aux2', 'Kill Switch'
])
# TODO: add RTL switch and others? Look at params which functions are mapped?
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
else: # it's an old log (COMPATIBILITY)
data_plot = DataPlot(data,
plot_config,
'rc_channels',
title='Raw Radio Control Inputs',
plot_height='small',
y_range=Range1d(-1.1, 1.1),
changed_params=changed_params,
x_range=x_range)
num_rc_channels = 8
if data_plot.dataset:
max_channels = np.amax(data_plot.dataset.data['channel_count'])
if max_channels < num_rc_channels: num_rc_channels = max_channels
legends = []
for i in range(num_rc_channels):
channel_names = px4_ulog.get_configured_rc_input_names(i)
if channel_names is None:
legends.append('Channel ' + str(i))
else:
legends.append('Channel ' + str(i) + ' (' +
', '.join(channel_names) + ')')
data_plot.add_graph(
['channels[' + str(i) + ']' for i in range(num_rc_channels)],
colors8[0:num_rc_channels],
legends,
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator controls 0
data_plot = DataPlot(data,
plot_config,
'actuator_controls_0',
y_start=0,
title='Actuator Controls 0',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(
['control[0]', 'control[1]', 'control[2]', 'control[3]'],
colors8[0:4], ['Roll', 'Pitch', 'Yaw', 'Thrust'],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes, vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator controls (Main) FFT (for filter & output noise analysis)
data_plot = DataPlotFFT(data,
plot_config,
'actuator_controls_0',
title='Actuator Controls FFT')
data_plot.add_graph(['control[0]', 'control[1]', 'control[2]'], colors3,
['Roll', 'Pitch', 'Yaw'])
if not data_plot.had_error:
if 'MC_DTERM_CUTOFF' in ulog.initial_parameters:
data_plot.mark_frequency(
ulog.initial_parameters['MC_DTERM_CUTOFF'], 'MC_DTERM_CUTOFF')
if 'IMU_GYRO_CUTOFF' in ulog.initial_parameters:
data_plot.mark_frequency(
ulog.initial_parameters['IMU_GYRO_CUTOFF'], 'IMU_GYRO_CUTOFF',
20)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator controls 1
# (only present on VTOL, Fixed-wing config)
data_plot = DataPlot(data,
plot_config,
'actuator_controls_1',
y_start=0,
title='Actuator Controls 1 (VTOL in Fixed-Wing mode)',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(
['control[0]', 'control[1]', 'control[2]', 'control[3]'],
colors8[0:4], ['Roll', 'Pitch', 'Yaw', 'Thrust'],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes, vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator outputs 0: Main
data_plot = DataPlot(data,
plot_config,
'actuator_outputs',
y_start=0,
title='Actuator Outputs (Main)',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
num_actuator_outputs = 8
if data_plot.dataset:
max_outputs = np.amax(data_plot.dataset.data['noutputs'])
if max_outputs < num_actuator_outputs:
num_actuator_outputs = max_outputs
data_plot.add_graph(
['output[' + str(i) + ']' for i in range(num_actuator_outputs)],
colors8[0:num_actuator_outputs],
['Output ' + str(i) for i in range(num_actuator_outputs)],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes, vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator outputs 1: AUX
data_plot = DataPlot(data,
plot_config,
'actuator_outputs',
y_start=0,
title='Actuator Outputs (AUX)',
plot_height='small',
changed_params=changed_params,
topic_instance=1,
x_range=x_range)
num_actuator_outputs = 8
# only plot if at least one of the outputs is not constant
all_constant = True
if data_plot.dataset:
max_outputs = np.amax(data_plot.dataset.data['noutputs'])
if max_outputs < num_actuator_outputs:
num_actuator_outputs = max_outputs
for i in range(num_actuator_outputs):
output_data = data_plot.dataset.data['output[' + str(i) + ']']
if not np.all(output_data == output_data[0]):
all_constant = False
if not all_constant:
data_plot.add_graph(
['output[' + str(i) + ']' for i in range(num_actuator_outputs)],
colors8[0:num_actuator_outputs],
['Output ' + str(i) for i in range(num_actuator_outputs)],
mark_nan=True)
plot_flight_modes_background(data_plot, flight_mode_changes,
vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# raw acceleration
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_axis_label='[m/s^2]',
title='Raw Acceleration',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
'accelerometer_m_s2[0]', 'accelerometer_m_s2[1]',
'accelerometer_m_s2[2]'
], colors3, ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# raw angular speed
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_axis_label='[deg/s]',
title='Raw Angular Speed (Gyroscope)',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
lambda data:
('gyro_rad[0]', np.rad2deg(data['gyro_rad[0]'])), lambda data:
('gyro_rad[1]', np.rad2deg(data['gyro_rad[1]'])), lambda data:
('gyro_rad[2]', np.rad2deg(data['gyro_rad[2]']))
], colors3, ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# magnetic field strength
data_plot = DataPlot(data,
plot_config,
magnetometer_ga_topic,
y_axis_label='[gauss]',
title='Raw Magnetic Field Strength',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(
['magnetometer_ga[0]', 'magnetometer_ga[1]', 'magnetometer_ga[2]'],
colors3, ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# distance sensor
data_plot = DataPlot(data,
plot_config,
'distance_sensor',
y_start=0,
y_axis_label='[m]',
title='Distance Sensor',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(['current_distance', 'covariance'], colors3[0:2],
['Distance', 'Covariance'])
if data_plot.finalize() is not None: plots.append(data_plot)
# gps uncertainty
# the accuracy values can be really large if there is no fix, so we limit the
# y axis range to some sane values
data_plot = DataPlot(data,
plot_config,
'vehicle_gps_position',
title='GPS Uncertainty',
y_range=Range1d(0, 40),
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(
['eph', 'epv', 'satellites_used', 'fix_type'], colors8[::2], [
'Horizontal position accuracy [m]',
'Vertical position accuracy [m]', 'Num Satellites used', 'GPS Fix'
])
if data_plot.finalize() is not None: plots.append(data_plot)
# gps noise & jamming
data_plot = DataPlot(data,
plot_config,
'vehicle_gps_position',
y_start=0,
title='GPS Noise & Jamming',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(['noise_per_ms', 'jamming_indicator'], colors3[0:2],
['Noise per ms', 'Jamming Indicator'])
if data_plot.finalize() is not None: plots.append(data_plot)
# thrust and magnetic field
data_plot = DataPlot(data,
plot_config,
magnetometer_ga_topic,
y_start=0,
title='Thrust and Magnetic Field',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
lambda data:
('len_mag',
np.sqrt(data['magnetometer_ga[0]']**2 + data['magnetometer_ga[1]']**2
+ data['magnetometer_ga[2]']**2))
], colors2[0:1], ['Norm of Magnetic Field'])
data_plot.change_dataset('actuator_controls_0')
data_plot.add_graph([lambda data: ('thrust', data['control[3]'])],
colors2[1:2], ['Thrust'])
if data_plot.finalize() is not None: plots.append(data_plot)
# Acceleration Spectrogram
data_plot = DataPlotSpec(data,
plot_config,
'sensor_combined',
y_axis_label='[Hz]',
title='Acceleration Power Spectral Density',
plot_height='small',
x_range=x_range)
data_plot.add_graph([
'accelerometer_m_s2[0]', 'accelerometer_m_s2[1]',
'accelerometer_m_s2[2]'
], ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# power
data_plot = DataPlot(data,
plot_config,
'battery_status',
y_start=0,
title='Power',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([
'voltage_v', 'voltage_filtered_v', 'current_a', lambda data:
('discharged_mah', data['discharged_mah'] / 100), lambda data:
('remaining', data['remaining'] * 10)
], colors8[::2] + colors8[1:2], [
'Battery Voltage [V]', 'Battery Voltage filtered [V]',
'Battery Current [A]', 'Discharged Amount [mAh / 100]',
'Battery remaining [0=empty, 10=full]'
])
data_plot.change_dataset('system_power')
if data_plot.dataset:
if 'voltage5v_v' in data_plot.dataset.data and \
np.amax(data_plot.dataset.data['voltage5v_v']) > 0.0001:
data_plot.add_graph(['voltage5v_v'], colors8[7:8], ['5 V'])
if 'voltage3v3_v' in data_plot.dataset.data and \
np.amax(data_plot.dataset.data['voltage3v3_v']) > 0.0001:
data_plot.add_graph(['voltage3v3_v'], colors8[5:6], ['3.3 V'])
if data_plot.finalize() is not None: plots.append(data_plot)
# estimator watchdog
try:
data_plot = DataPlot(data,
plot_config,
'estimator_status',
y_start=0,
title='Estimator Watchdog',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
estimator_status = ulog.get_dataset('estimator_status').data
plot_data = []
plot_labels = []
input_data = [
('NaN Flags', estimator_status['nan_flags']),
('Health Flags (vel, pos, hgt)', estimator_status['health_flags']),
('Timeout Flags (vel, pos, hgt)',
estimator_status['timeout_flags']),
('Velocity Check Bit',
(estimator_status['innovation_check_flags']) & 0x1),
('Horizontal Position Check Bit',
(estimator_status['innovation_check_flags'] >> 1) & 1),
('Vertical Position Check Bit',
(estimator_status['innovation_check_flags'] >> 2) & 1),
('Mag X, Y, Z Check Bits',
(estimator_status['innovation_check_flags'] >> 3) & 0x7),
('Yaw Check Bit',
(estimator_status['innovation_check_flags'] >> 6) & 1),
('Airspeed Check Bit',
(estimator_status['innovation_check_flags'] >> 7) & 1),
('Synthetic Sideslip Check Bit',
(estimator_status['innovation_check_flags'] >> 8) & 1),
('Height to Ground Check Bit',
(estimator_status['innovation_check_flags'] >> 9) & 1),
('Optical Flow X, Y Check Bits',
(estimator_status['innovation_check_flags'] >> 10) & 0x3),
]
# filter: show only the flags that have non-zero samples
for cur_label, cur_data in input_data:
if np.amax(cur_data) > 0.1:
data_label = 'flags_' + str(
len(plot_data)) # just some unique string
plot_data.append(lambda d, data=cur_data, label=data_label:
(label, data))
plot_labels.append(cur_label)
if len(plot_data) >= 8: # cannot add more than that
break
if len(plot_data) == 0:
# add the plot even in the absence of any problem, so that the user
# can validate that (otherwise it's ambiguous: it could be that the
# estimator_status topic is not logged)
plot_data = [lambda d: ('flags', input_data[0][1])]
plot_labels = [input_data[0][0]]
data_plot.add_graph(plot_data, colors8[0:len(plot_data)], plot_labels)
if data_plot.finalize() is not None: plots.append(data_plot)
except (KeyError, IndexError) as error:
print('Error in estimator plot: ' + str(error))
# RC Quality
data_plot = DataPlot(data,
plot_config,
'input_rc',
title='RC Quality',
plot_height='small',
y_range=Range1d(0, 1),
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph([lambda data: ('rssi', data['rssi'] / 100), 'rc_lost'],
colors3[0:2], ['RSSI [0, 1]', 'RC Lost (Indicator)'])
data_plot.change_dataset('vehicle_status')
data_plot.add_graph(['rc_signal_lost'], colors3[2:3],
['RC Lost (Detected)'])
if data_plot.finalize() is not None: plots.append(data_plot)
# cpu load
data_plot = DataPlot(data,
plot_config,
'cpuload',
title='CPU & RAM',
plot_height='small',
y_range=Range1d(0, 1),
changed_params=changed_params,
x_range=x_range)
data_plot.add_graph(['ram_usage', 'load'], [colors3[1], colors3[2]],
['RAM Usage', 'CPU Load'])
data_plot.add_span('load', line_color=colors3[2])
data_plot.add_span('ram_usage', line_color=colors3[1])
plot_flight_modes_background(data_plot, flight_mode_changes, vtol_states)
if data_plot.finalize() is not None: plots.append(data_plot)
# sampling: time difference
try:
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_range=Range1d(0, 25e3),
y_axis_label='[us]',
title='Sampling Regularity of Sensor Data',
plot_height='small',
changed_params=changed_params,
x_range=x_range)
sensor_combined = ulog.get_dataset('sensor_combined').data
sampling_diff = np.diff(sensor_combined['timestamp'])
min_sampling_diff = np.amin(sampling_diff)
plot_dropouts(data_plot.bokeh_plot, ulog.dropouts, min_sampling_diff)
data_plot.add_graph(
[lambda data: ('timediff', np.append(sampling_diff, 0))],
[colors3[2]], ['delta t (between 2 logged samples)'])
data_plot.change_dataset('estimator_status')
data_plot.add_graph(
[lambda data: ('time_slip', data['time_slip'] * 1e6)],
[colors3[1]], ['Estimator time slip (cumulative)'])
if data_plot.finalize() is not None: plots.append(data_plot)
except:
pass
# exchange all DataPlot's with the bokeh_plot and handle parameter changes
param_changes_button = Button(label="Hide Parameter Changes", width=170)
param_change_labels = []
# FIXME: this should be a CustomJS callback, not on the server. However this
# did not work for me.
def param_changes_button_clicked():
""" callback to show/hide parameter changes """
for label in param_change_labels:
if label.visible:
param_changes_button.label = 'Show Parameter Changes'
label.visible = False
label.text_alpha = 0 # label.visible does not work, so we use this instead
else:
param_changes_button.label = 'Hide Parameter Changes'
label.visible = True
label.text_alpha = 1
param_changes_button.on_click(param_changes_button_clicked)
jinja_plot_data = []
for i in range(len(plots)):
if plots[i] is None:
plots[i] = widgetbox(param_changes_button,
width=int(plot_width * 0.99))
if isinstance(plots[i], DataPlot):
if plots[i].param_change_label is not None:
param_change_labels.append(plots[i].param_change_label)
plot_title = plots[i].title
plots[i] = plots[i].bokeh_plot
fragment = 'Nav-'+plot_title.replace(' ', '-') \
.replace('&', '_').replace('(', '').replace(')', '')
jinja_plot_data.append({
'model_id': plots[i].ref['id'],
'fragment': fragment,
'title': plot_title
})
# changed parameters
plots.append(get_changed_parameters(ulog.initial_parameters, plot_width))
# information about which messages are contained in the log
# TODO: need to load all topics for this (-> log loading will take longer)
# but if we load all topics and the log contains some (external) topics
# with buggy timestamps, it will affect the plotting.
# data_list_sorted = sorted(ulog.data_list, key=lambda d: d.name + str(d.multi_id))
# table_text = []
# for d in data_list_sorted:
# message_size = sum([ULog.get_field_size(f.type_str) for f in d.field_data])
# num_data_points = len(d.data['timestamp'])
# table_text.append((d.name, str(d.multi_id), str(message_size), str(num_data_points),
# str(message_size * num_data_points)))
# topics_info = '<table><tr><th>Name</th><th>Topic instance</th><th>Message Size</th>' \
# '<th>Number of data points</th><th>Total bytes</th></tr>' + ''.join(
# ['<tr><td>'+'</td><td>'.join(list(x))+'</td></tr>' for x in table_text]) + '</table>'
# topics_div = Div(text=topics_info, width=int(plot_width*0.9))
# plots.append(widgetbox(topics_div, width=int(plot_width*0.9)))
# log messages
plots.append(get_logged_messages(ulog.logged_messages, plot_width))
# perf & top output
top_data = ''
perf_data = ''
for state in ['pre', 'post']:
if 'perf_top_' + state + 'flight' in ulog.msg_info_multiple_dict:
current_top_data = ulog.msg_info_multiple_dict['perf_top_' +
state + 'flight'][0]
flight_data = escape('\n'.join(current_top_data))
top_data += '<p>' + state.capitalize(
) + ' Flight:<br/><pre>' + flight_data + '</pre></p>'
if 'perf_counter_' + state + 'flight' in ulog.msg_info_multiple_dict:
current_perf_data = ulog.msg_info_multiple_dict['perf_counter_' +
state +
'flight'][0]
flight_data = escape('\n'.join(current_perf_data))
perf_data += '<p>' + state.capitalize(
) + ' Flight:<br/><pre>' + flight_data + '</pre></p>'
additional_data_html = ''
if len(top_data) > 0:
additional_data_html += '<h5>Processes</h5>' + top_data
if len(perf_data) > 0:
additional_data_html += '<h5>Performance Counters</h5>' + perf_data
if len(additional_data_html) > 0:
# hide by default & use a button to expand
additional_data_html = '''
<button class="btn btn-secondary" data-toggle="collapse" style="min-width:0;"
data-target="#show-additional-data">Show additional Data</button>
<div id="show-additional-data" class="collapse">
{:}
</div>
'''.format(additional_data_html)
additional_data_div = Div(text=additional_data_html,
width=int(plot_width * 0.9))
plots.append(
widgetbox(additional_data_div, width=int(plot_width * 0.9)))
curdoc().template_variables['plots'] = jinja_plot_data
return plots
value='constant',
options=['constant'] + columns)
size_column.on_change('value', update_columns)
slider = RangeSlider(start=years[0],
end=years[-1],
range=(years[0], years[-1]),
step=1,
title="Years")
slider.on_change('range', update_year)
region_column = CheckboxGroup(labels=regions[:], active=range(len(regions)))
region_column.on_change('active', update_columns)
reset_button = Button(label='Reset')
reset_button.on_click(reset)
#Create initial plot
p, sc_source, line_source = make_plot()
desc_box = Div(text=update_desc_box())
controls = widgetbox(
[y_column, size_column, region_column, slider, reset_button, desc_box],
width=430)
lout = row([p, controls])
curdoc().add_root(lout)
curdoc().title = "FH Scatterplot"
with open('time_plot_embed.html', 'w') as f:
script = autoload_server(url="https://fh-vis.herokuapp.com/time_plot")
def re_analyze_callback():
re_analyze()
# set callbacks
top_n_dropdown.on_change('value', top_n_phrases_changed_callback)
top_n_clusters_dropdown.on_change('value', top_n_clusters_changed_callback)
radio_group_area.on_change('active', selected_graph_changed_callback)
# pylint: disable=no-member
filter_topics_table_source.selected.on_change('indices',
filter_topic_selected_callback)
filter_custom_table_source.selected.on_change('indices',
filter_custom_selected_callback)
search_button.on_click(search_topic_callback)
clear_button.on_click(clear_search_callback)
filter_tabs.on_change('active', tab_changed_callback)
analyse_button.on_click(re_analyze_callback)
# start app
draw_ui(top_n_dropdown.value, top_n_clusters_dropdown.value,
radio_group_area.active)
doc = curdoc()
main_title = "Trend Analysis"
doc.title = main_title
doc.add_root(grid)
toggle = Toggle(label="Start", type="success")
toggle.on_click(toggle_handler)
toggle.active = False
# Set up reset button
def reset_handler():
global ii, current_time
ii = 0
current_time = 0
l_forward.data_source.data["y"] = forward_wave()
l_reverse.data_source.data["y"] = reverse_wave()
l_sum.data_source.data["y"] = l_forward.data_source.data["y"] + \
l_reverse.data_source.data["y"]
#t2.data_source.data["text"] = ['t = {:.3f} s'.format(current_time)]
button_reset = Button(label="Reset", type="success")
button_reset.on_click(reset_handler)
# Set up checkboxes to show/hide forward & reverse propagating waves
def checkbox_group_handler(active):
global alpha, alpha_slider
if 0 in active:
#l_forward.glyph.visible = True
l_forward.glyph.line_alpha = alpha_forward_reverse_waves
else:
#l_forward.glyph.visible = False
l_forward.glyph.line_alpha = 0.0
if 1 in active:
#l_reverse.glyph.visible = True
l_reverse.glyph.line_alpha = alpha_forward_reverse_waves
else:
#l_reverse.glyph.visible = False
value=1,
step=1,
title="Liczba wentylatorów")
cpuPowerSlider = Slider(start=0,
end=100,
value=50,
step=1,
title="Moc procesora [%]")
resetButton = Button(label="Reset")
widgetsTitle = Div(text="""<center><b>Panel kontrolny</b></center>""")
outsideTempSlider.on_change('value_throttled', update_outsideTemp)
targetTempSlider.on_change('value_throttled', update_targetTemp)
numberOfFansSlider.on_change('value_throttled', update_numberOfFans)
cpuPowerSlider.on_change('value_throttled', update_cpuPower)
resetButton.on_click(simulation_reset)
inputs = column([
widgetsTitle, targetTempSlider, outsideTempSlider, numberOfFansSlider,
cpuPowerSlider, resetButton
],
width=300)
#curdoc().theme = 'dark_minimal'
grid = gridplot([[temp_plot, err_plot], [temp2_plot, reg_plot]],
toolbar_location=None)
#grid.toolbar_location = None
curdoc().add_root(row(inputs, grid))
curdoc().title = "CPU Cooler Simulator"
curdoc().add_periodic_callback(callback, 1)
]
lower_bounds = [
float(lower_bound_1.value),
float(lower_bound_X.value),
float(lower_bound_2.value)
]
bets = ArbitrageOptimizer.get_optimal_bets(ratios, upper_bounds,
lower_bounds)
bet_1.value = str(round(bets[0], 0))
bet_X.value = str(round(bets[1], 0))
bet_2.value = str(round(bets[2], 0))
calculate_button.on_click(calculate_bets)
# create layout to be plotted
dataTable = get_data_table_from_games(games)
arbitrageTabs = Tabs(tabs=tabs, width=900)
calculator = column(row(upper_bound_1, upper_bound_X, upper_bound_2),
row(ratio_1, ratio_X, ratio_2),
row(lower_bound_1, lower_bound_X, lower_bound_2),
row(calculate_button), row(bet_1, bet_X, bet_2))
mainTabs = [
Panel(child=dataTable, title="Current profitable games"),
Panel(child=column(arbitrageTabs, calculator),
title="Current arbitrage opportunities")
]
layout = column(Tabs(tabs=mainTabs), sizing_mode='stretch_both')
if redraw_plot:
# if True: # pylint: disable=using-constant-test
figure = create_plot()
#TO DO: for some reason this destroys the coupling to source.data
# to figure out why (and then restrict this to actual redrawing scenarios)
l.children[0].children[1] = figure
redraw_plot = False
update_legends(l)
plot_info.text += " done!"
btn_plot.button_type = 'success'
return
btn_plot.on_click(update)
# pylint: disable=unused-argument
def on_change_clr(attr, old, new):
"""Remember to redraw plot next time, when necessary.
When switching between bond_type color and something else,
the plot needs to be redrawn.
"""
global redraw_plot
if (new == 'bond_type' or old == 'bond_type') and new != old:
redraw_plot = True
check_uniqueness(attr, old, new)
def generate_plots(ulog, px4_ulog, flight_mode_changes, db_data, vehicle_data):
""" create a list of bokeh plots (and widgets) to show """
plots = []
data = ulog.data_list
# Heading
sys_name = ''
if 'sys_name' in ulog.msg_info_dict:
sys_name = cgi.escape(ulog.msg_info_dict['sys_name']) + ' '
div = Div(text="<h1>" + sys_name + px4_ulog.get_mav_type() + "</h1>",
width=int(plot_width * 0.9))
header_divs = [div]
if db_data.description != '':
div_descr = Div(text="<h4>" + db_data.description + "</h4>",
width=int(plot_width * 0.9))
header_divs.append(div_descr)
# airframe
table_text = []
if 'SYS_AUTOSTART' in ulog.initial_parameters:
sys_autostart = ulog.initial_parameters['SYS_AUTOSTART']
airframe_data = get_airframe_data(sys_autostart)
if airframe_data is None:
table_text.append(('Airframe', str(sys_autostart)))
else:
airframe_type = ''
if 'type' in airframe_data:
airframe_type = ', ' + airframe_data['type']
table_text.append(
('Airframe', airframe_data.get('name') + airframe_type +
' <small>(' + str(sys_autostart) + ')</small>'))
# HW & SW
sys_hardware = ''
if 'ver_hw' in ulog.msg_info_dict:
sys_hardware = cgi.escape(ulog.msg_info_dict['ver_hw'])
table_text.append(('Hardware', sys_hardware))
release_str = ulog.get_version_info_str()
if release_str is None:
release_str = ''
release_str_suffix = ''
else:
release_str += ' <small>('
release_str_suffix = ')</small>'
if 'ver_sw' in ulog.msg_info_dict:
ver_sw = cgi.escape(ulog.msg_info_dict['ver_sw'])
ver_sw_link = 'https://github.com/PX4/Firmware/commit/' + ver_sw
table_text.append(
('Software Version', release_str + '<a href="' + ver_sw_link +
'" target="_blank">' + ver_sw[:8] + '</a>' + release_str_suffix))
if 'sys_os_name' in ulog.msg_info_dict and 'sys_os_ver_release' in ulog.msg_info_dict:
os_name = cgi.escape(ulog.msg_info_dict['sys_os_name'])
os_ver = ulog.get_version_info_str('sys_os_ver_release')
if os_ver is not None:
table_text.append(('OS Version', os_name + ', ' + os_ver))
table_text.append(('Estimator', px4_ulog.get_estimator()))
# dropouts
dropout_durations = [dropout.duration for dropout in ulog.dropouts]
if len(dropout_durations) > 0:
total_duration = sum(dropout_durations) / 1000
if total_duration > 5:
total_duration_str = '{:.0f}'.format(total_duration)
else:
total_duration_str = '{:.2f}'.format(total_duration)
table_text.append(
('Dropouts', '{:} ({:} s)'.format(len(dropout_durations),
total_duration_str)))
# logging duration
m, s = divmod(int((ulog.last_timestamp - ulog.start_timestamp) / 1e6), 60)
h, m = divmod(m, 60)
table_text.append(
('Logging duration', '{:d}:{:02d}:{:02d}'.format(h, m, s)))
# total vehicle flight time
flight_time_s = get_total_flight_time(ulog)
if flight_time_s is not None:
m, s = divmod(int(flight_time_s), 60)
h, m = divmod(m, 60)
days, h = divmod(h, 24)
flight_time_str = ''
if days > 0: flight_time_str += '{:d} days '.format(days)
if h > 0: flight_time_str += '{:d} hours '.format(h)
if m > 0: flight_time_str += '{:d} minutes '.format(m)
flight_time_str += '{:d} seconds '.format(s)
table_text.append(('Vehicle flight time', flight_time_str))
# vehicle UUID (and name if provided). SITL does not have a UUID
if 'sys_uuid' in ulog.msg_info_dict and sys_hardware != 'SITL':
sys_uuid = cgi.escape(ulog.msg_info_dict['sys_uuid'])
if vehicle_data is not None and vehicle_data.name != '':
sys_uuid = sys_uuid + ' (' + vehicle_data.name + ')'
if len(sys_uuid) > 0:
table_text.append(('Vehicle UUID', sys_uuid))
# Wind speed, rating, feedback
if db_data.wind_speed >= 0:
table_text.append(('Wind Speed', db_data.wind_speed_str()))
if len(db_data.rating) > 0:
table_text.append(('Flight Rating', db_data.rating_str()))
if len(db_data.feedback) > 0:
table_text.append(('Feedback', db_data.feedback.replace('\n',
'<br/>')))
if len(db_data.video_url) > 0:
table_text.append(('Video', '<a href="' + db_data.video_url +
'" target="_blank">' + db_data.video_url + '</a>'))
# generate the table
divs_text = '<table>' + ''.join([
'<tr><td class="left">' + a + ':</td><td>' + b + '</td></tr>'
for a, b in table_text
]) + '</table>'
header_divs.append(Div(text=divs_text, width=int(plot_width * 0.9)))
plots.append(widgetbox(header_divs, width=int(plot_width * 0.9)))
# FIXME: for now, we use Google maps directly without bokeh, because it's not working reliably
# GPS map
# gps_plots = []
# gps_titles = []
# plot = plot_map(ulog, plot_config, map_type='google', api_key =
# get_google_maps_api_key(), setpoints=False)
# plot = None
# if plot is not None:
# gps_plots.append(plot)
# gps_titles.append('GPS Map: Satellite')
#
# plot = plot_map(ulog, plot_config, map_type='plain', setpoints=True)
# if plot is not None:
# gps_plots.append(plot)
# gps_titles.append('GPS Map: Plain')
#
# data_plot = DataPlot2D(data, plot_config, 'vehicle_local_position',
# x_axis_label = '[m]', y_axis_label='[m]', plot_height='gps_map')
# data_plot.add_graph('y', 'x', colors2[0], 'Estimated')
# data_plot.change_dataset('vehicle_local_position_setpoint')
# data_plot.add_graph('y', 'x', colors2[1], 'Setpoint')
# if data_plot.finalize() is not None:
# gps_plots.append(data_plot.bokeh_plot)
# gps_titles.append('Local Position')
#
#
# if len(gps_plots) >= 2:
# tabs = []
# for i in range(len(gps_plots)):
# tabs.append(Panel(child=gps_plots[i], title=gps_titles[i]))
# gps_plot_height=plot_config['plot_height']['gps_map'] + 30
# plots.append(Tabs(tabs=tabs, width=plot_width, height=gps_plot_height))
# elif len(gps_plots) == 1:
# plots.extend(gps_plots)
# Position plot
data_plot = DataPlot2D(data,
plot_config,
'vehicle_local_position',
x_axis_label='[m]',
y_axis_label='[m]',
plot_height='gps_map')
data_plot.add_graph('y',
'x',
colors2[0],
'Estimated',
check_if_all_zero=True)
data_plot.change_dataset('vehicle_local_position_setpoint')
data_plot.add_graph('y', 'x', colors2[1], 'Setpoint')
# groundtruth (SITL only)
data_plot.change_dataset('vehicle_local_position_groundtruth')
data_plot.add_graph('y', 'x', color_gray, 'Groundtruth')
# GPS + position setpoints
plot_map(ulog,
plot_config,
map_type='plain',
setpoints=True,
bokeh_plot=data_plot.bokeh_plot)
if data_plot.finalize() is not None:
plots.append(data_plot.bokeh_plot)
curdoc().template_variables['has_position_data'] = True
# initialize parameter changes
changed_params = None
if not 'replay' in ulog.msg_info_dict: # replay can have many param changes
if len(ulog.changed_parameters) > 0:
changed_params = ulog.changed_parameters
plots.append(None) # save space for the param change button
### Add all data plots ###
# Altitude estimate
data_plot = DataPlot(data,
plot_config,
'vehicle_gps_position',
y_axis_label='[m]',
title='Altitude Estimate',
changed_params=changed_params)
data_plot.add_graph([lambda data: ('alt', data['alt'] * 0.001)],
colors8[0:1], ['GPS Altitude'])
data_plot.change_dataset('sensor_combined')
data_plot.add_graph(['baro_alt_meter'], colors8[1:2],
['Barometer Altitude'])
data_plot.change_dataset('vehicle_global_position')
data_plot.add_graph(['alt'], colors8[2:3], ['Fused Altitude Estimation'])
data_plot.change_dataset('position_setpoint_triplet')
data_plot.add_circle(['current.alt'],
[plot_config['mission_setpoint_color']],
['Altitude Setpoint'])
data_plot.change_dataset('actuator_controls_0')
data_plot.add_graph([lambda data: ('thrust', data['control[3]'] * 100)],
colors8[6:7], ['Thrust [0, 100]'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# Roll/Pitch/Yaw angle & angular rate
for axis in ['roll', 'pitch', 'yaw']:
# angle
axis_name = axis.capitalize()
data_plot = DataPlot(data,
plot_config,
'vehicle_attitude',
y_axis_label='[deg]',
title=axis_name + ' Angle',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([lambda data: (axis, np.rad2deg(data[axis]))],
colors2[0:1], [axis_name + ' Estimated'])
data_plot.change_dataset('vehicle_attitude_setpoint')
data_plot.add_graph(
[lambda data: (axis + '_d', np.rad2deg(data[axis + '_d']))],
colors2[1:2], [axis_name + ' Setpoint'])
data_plot.change_dataset('vehicle_attitude_groundtruth')
data_plot.add_graph([lambda data: (axis, np.rad2deg(data[axis]))],
[color_gray], [axis_name + ' Groundtruth'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# rate
data_plot = DataPlot(data,
plot_config,
'vehicle_attitude',
y_axis_label='[deg/s]',
title=axis_name + ' Angular Rate',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(
[lambda data: (axis + 'speed', np.rad2deg(data[axis + 'speed']))],
colors2[0:1], [axis_name + ' Rate Estimated'])
data_plot.change_dataset('vehicle_rates_setpoint')
data_plot.add_graph([lambda data: (axis, np.rad2deg(data[axis]))],
colors2[1:2], [axis_name + ' Rate Setpoint'])
data_plot.change_dataset('vehicle_attitude_groundtruth')
data_plot.add_graph(
[lambda data: (axis + 'speed', np.rad2deg(data[axis + 'speed']))],
[color_gray], [axis_name + ' Rate Groundtruth'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# Local position
for axis in ['x', 'y', 'z']:
data_plot = DataPlot(data,
plot_config,
'vehicle_local_position',
y_axis_label='[m]',
title='Local Position ' + axis.upper(),
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([axis], colors2[0:1],
[axis.upper() + ' Estimated'])
data_plot.change_dataset('vehicle_local_position_setpoint')
data_plot.add_graph([axis], colors2[1:2], [axis.upper() + ' Setpoint'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# Velocity
data_plot = DataPlot(data,
plot_config,
'vehicle_local_position',
y_axis_label='[m/s]',
title='Velocity',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(['vx', 'vy', 'vz'], colors3, ['X', 'Y', 'Z'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# Vision position (only if topic found)
if any(elem.name == 'vehicle_vision_position' for elem in data):
data_plot = DataPlot(data,
plot_config,
'vehicle_vision_position',
y_axis_label='[m]',
title='Vision Position',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(['x', 'y', 'z'], colors3, ['X', 'Y', 'Z'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
data_plot.change_dataset('vehicle_local_position_groundtruth')
data_plot.add_graph(
['x', 'y', 'z'], colors8[2:5],
['Groundtruth X', 'Groundtruth Y', 'Groundtruth Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# Vision velocity
data_plot = DataPlot(data,
plot_config,
'vehicle_vision_position',
y_axis_label='[m]',
title='Vision Velocity',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(['vx', 'vy', 'vz'], colors3, ['X', 'Y', 'Z'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
data_plot.change_dataset('vehicle_local_position_groundtruth')
data_plot.add_graph(
['vx', 'vy', 'vz'], colors8[2:5],
['Groundtruth X', 'Groundtruth Y', 'Groundtruth Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# Vision attitude
if any(elem.name == 'vehicle_vision_attitude' for elem in data):
data_plot = DataPlot(data,
plot_config,
'vehicle_vision_attitude',
y_axis_label='[deg]',
title='Vision Attitude',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([
lambda data: ('roll', np.rad2deg(data['roll'])), lambda data:
('pitch', np.rad2deg(data['pitch'])), lambda data:
('yaw', np.rad2deg(data['yaw']))
], colors3, ['Roll', 'Pitch', 'Yaw'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
data_plot.change_dataset('vehicle_attitude_groundtruth')
data_plot.add_graph(
[
lambda data: ('roll', np.rad2deg(data['roll'])), lambda data:
('pitch', np.rad2deg(data['pitch'])), lambda data:
('yaw', np.rad2deg(data['yaw']))
], colors8[2:5],
['Roll Groundtruth', 'Pitch Groundtruth', 'Yaw Groundtruth'])
if data_plot.finalize() is not None: plots.append(data_plot)
# Airspeed vs Ground speed
try:
control_state = ulog.get_dataset('control_state').data
# only plot if valid airspeed
if np.amax(control_state['airspeed_valid']) == 1:
data_plot = DataPlot(data,
plot_config,
'vehicle_global_position',
y_axis_label='[m/s]',
title='Airspeed',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([
lambda data: ('groundspeed_estimated',
np.sqrt(data['vel_n']**2 + data['vel_e']**2))
], colors3[2:3], ['Ground Speed Estimated'])
data_plot.change_dataset('control_state')
data_plot.add_graph(['airspeed'], colors2[0:1],
['Airspeed Estimated'])
plot_flight_modes_background(data_plot.bokeh_plot,
flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
except:
pass
# raw radio control inputs
data_plot = DataPlot(data,
plot_config,
'rc_channels',
title='Raw Radio Control Inputs',
plot_height='small',
y_range=Range1d(-1.1, 1.1),
changed_params=changed_params)
num_rc_channels = 8
if data_plot.dataset:
max_channels = np.amax(data_plot.dataset.data['channel_count'])
if max_channels < num_rc_channels: num_rc_channels = max_channels
legends = []
for i in range(num_rc_channels):
channel_names = px4_ulog.get_configured_rc_input_names(i)
if channel_names is None:
legends.append('Channel ' + str(i))
else:
legends.append('Channel ' + str(i) + ' (' +
', '.join(channel_names) + ')')
data_plot.add_graph(
['channels[' + str(i) + ']' for i in range(num_rc_channels)],
colors8[0:num_rc_channels], legends)
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator controls 0
data_plot = DataPlot(data,
plot_config,
'actuator_controls_0',
y_start=0,
title='Actuator Controls 0',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(
['control[0]', 'control[1]', 'control[2]', 'control[3]'], colors8[0:4],
['Roll', 'Pitch', 'Yaw', 'Thrust'])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# actuator outputs
data_plot = DataPlot(data,
plot_config,
'actuator_outputs',
y_start=0,
title='Actuator Outputs',
plot_height='small',
changed_params=changed_params)
num_actuator_outputs = 8
if data_plot.dataset:
max_outputs = np.amax(data_plot.dataset.data['noutputs'])
if max_outputs < num_actuator_outputs:
num_actuator_outputs = max_outputs
data_plot.add_graph(
['output[' + str(i) + ']' for i in range(num_actuator_outputs)],
colors8[0:num_actuator_outputs],
['Output ' + str(i) for i in range(num_actuator_outputs)])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# raw acceleration
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_axis_label='[m/s^2]',
title='Raw Acceleration',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([
'accelerometer_m_s2[0]', 'accelerometer_m_s2[1]',
'accelerometer_m_s2[2]'
], colors3, ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# raw angular speed
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_axis_label='[deg/s]',
title='Raw Angular Speed (Gyroscope)',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([
lambda data:
('gyro_rad[0]', np.rad2deg(data['gyro_rad[0]'])), lambda data:
('gyro_rad[1]', np.rad2deg(data['gyro_rad[1]'])), lambda data:
('gyro_rad[2]', np.rad2deg(data['gyro_rad[2]']))
], colors3, ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# magnetic field strength
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_axis_label='[gauss]',
title='Raw Magnetic Field Strength',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(
['magnetometer_ga[0]', 'magnetometer_ga[1]', 'magnetometer_ga[2]'],
colors3, ['X', 'Y', 'Z'])
if data_plot.finalize() is not None: plots.append(data_plot)
# distance sensor
data_plot = DataPlot(data,
plot_config,
'distance_sensor',
y_start=0,
y_axis_label='[m]',
title='Distance Sensor',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(['current_distance', 'covariance'], colors3[0:2],
['Distance', 'Covariance'])
if data_plot.finalize() is not None: plots.append(data_plot)
# gps uncertainty
# the accuracy values can be really large if there is no fix, so we limit the
# y axis range to some sane values
data_plot = DataPlot(data,
plot_config,
'vehicle_gps_position',
title='GPS Uncertainty',
y_range=Range1d(0, 40),
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(
['eph', 'epv', 'satellites_used', 'fix_type'], colors8[::2], [
'Horizontal position accuracy [m]',
'Vertical position accuracy [m]', 'Num Satellites used', 'GPS Fix'
])
if data_plot.finalize() is not None: plots.append(data_plot)
# gps noise & jamming
data_plot = DataPlot(data,
plot_config,
'vehicle_gps_position',
y_start=0,
title='GPS Noise & Jamming',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(['noise_per_ms', 'jamming_indicator'], colors3[0:2],
['Noise per ms', 'Jamming Indicator'])
if data_plot.finalize() is not None: plots.append(data_plot)
# thrust and magnetic field
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_start=0,
title='Thrust and Magnetic Field',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([
lambda data:
('len_mag',
np.sqrt(data['magnetometer_ga[0]']**2 + data['magnetometer_ga[1]']**2
+ data['magnetometer_ga[2]']**2))
], colors2[0:1], ['Norm of Magnetic Field'])
data_plot.change_dataset('actuator_controls_0')
data_plot.add_graph([lambda data: ('thrust', data['control[3]'])],
colors2[1:2], ['Thrust'])
if data_plot.finalize() is not None: plots.append(data_plot)
# power
# TODO: dischared in Ah?
data_plot = DataPlot(data,
plot_config,
'battery_status',
y_start=0,
title='Power',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph([
'voltage_v', 'voltage_filtered_v', 'current_a', lambda data:
('discharged_mah', data['discharged_mah'] / 100)
], colors8[::2], [
'Voltage [V]', 'Voltage filtered [V]', 'Current [A]',
'Discharged Amount [mAh / 100]'
])
if data_plot.finalize() is not None: plots.append(data_plot)
# estimator watchdog
data_plot = DataPlot(data,
plot_config,
'estimator_status',
y_start=0,
title='Estimator Watchdog',
plot_height='small',
changed_params=changed_params)
data_plot.add_graph(['nan_flags', 'health_flags', 'timeout_flags'],
colors3, [
'NaN Flags', 'Health Flags (vel, pos, hgt)',
'Timeout Flags (vel, pos, hgt)'
])
if data_plot.finalize() is not None: plots.append(data_plot)
# RC Quality
data_plot = DataPlot(data,
plot_config,
'input_rc',
title='RC Quality',
plot_height='small',
y_range=Range1d(0, 1),
changed_params=changed_params)
data_plot.add_graph(['rc_lost', lambda data: ('rssi', data['rssi'] / 100)],
colors3[0:2], ['RC Lost', 'RSSI [0, 1]'])
if data_plot.finalize() is not None: plots.append(data_plot)
# cpu load
data_plot = DataPlot(data,
plot_config,
'cpuload',
title='CPU & RAM',
plot_height='small',
y_range=Range1d(0, 1),
changed_params=changed_params)
data_plot.add_graph(['ram_usage', 'load'], [colors3[1], colors3[2]],
['RAM Usage', 'CPU Load'])
data_plot.add_span('load', line_color=colors3[2])
data_plot.add_span('ram_usage', line_color=colors3[1])
plot_flight_modes_background(data_plot.bokeh_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
# sampling: time difference
try:
data_plot = DataPlot(data,
plot_config,
'sensor_combined',
y_start=0,
y_axis_label='[us]',
title='Sampling Regularity of Sensor Data',
plot_height='small',
changed_params=changed_params)
sensor_combined = ulog.get_dataset('sensor_combined').data
sampling_diff = np.diff(sensor_combined['timestamp'])
min_sampling_diff = np.amin(sampling_diff)
plot_dropouts(data_plot.bokeh_plot, ulog.dropouts, min_sampling_diff)
data_plot.add_graph(
[lambda data: ('timediff', np.append(sampling_diff, 0))],
[colors3[2]], ['delta t (between 2 samples)'])
if data_plot.finalize() is not None: plots.append(data_plot)
except:
pass
# exchange all DataPlot's with the bokeh_plot and handle parameter changes
param_changes_button = Button(label="Hide Parameter Changes", width=170)
param_change_labels = []
# FIXME: this should be a CustomJS callback, not on the server. However this
# did not work for me.
def param_changes_button_clicked():
""" callback to show/hide parameter changes """
for label in param_change_labels:
if label.visible:
param_changes_button.label = 'Show Parameter Changes'
label.visible = False
label.text_alpha = 0 # label.visible does not work, so we use this instead
else:
param_changes_button.label = 'Hide Parameter Changes'
label.visible = True
label.text_alpha = 1
param_changes_button.on_click(param_changes_button_clicked)
jinja_plot_data = []
for i in range(len(plots)):
if plots[i] is None:
plots[i] = widgetbox(param_changes_button,
width=int(plot_width * 0.99))
if isinstance(plots[i], DataPlot):
if plots[i].param_change_label is not None:
param_change_labels.append(plots[i].param_change_label)
plots[i] = plots[i].bokeh_plot
plot_title = plots[i].title.text
fragment = 'Nav-'+plot_title.replace(' ', '-') \
.replace('&', '_').replace('(', '').replace(')', '')
jinja_plot_data.append({
'model_id': plots[i].ref['id'],
'fragment': fragment,
'title': plot_title
})
# changed parameters
param_names = []
param_values = []
param_defaults = []
param_mins = []
param_maxs = []
param_descriptions = []
default_params = get_default_parameters()
for param_name in sorted(ulog.initial_parameters):
param_value = ulog.initial_parameters[param_name]
if param_name.startswith('RC') or param_name.startswith('CAL_'):
continue
try:
if param_name in default_params:
default_param = default_params[param_name]
if default_param['type'] == 'FLOAT':
is_default = abs(
float(default_param['default']) -
float(param_value)) < 0.00001
if 'decimal' in default_param:
param_value = round(param_value,
int(default_param['decimal']))
else:
is_default = int(
default_param['default']) == int(param_value)
if not is_default:
param_names.append(param_name)
param_values.append(param_value)
param_defaults.append(default_param['default'])
param_mins.append(default_param.get('min', ''))
param_maxs.append(default_param.get('max', ''))
param_descriptions.append(
default_param.get('short_desc', ''))
else:
# not found: add it as if it were changed
param_names.append(param_name)
param_values.append(param_value)
param_defaults.append('')
param_mins.append('')
param_maxs.append('')
param_descriptions.append('(unknown)')
except Exception as error:
print(type(error), error)
param_data = dict(names=param_names,
values=param_values,
defaults=param_defaults,
mins=param_mins,
maxs=param_maxs,
descriptions=param_descriptions)
source = ColumnDataSource(param_data)
columns = [
TableColumn(field="names",
title="Name",
width=int(plot_width * 0.2),
sortable=False),
TableColumn(field="values",
title="Value",
width=int(plot_width * 0.15),
sortable=False),
TableColumn(field="defaults",
title="Default",
width=int(plot_width * 0.1),
sortable=False),
TableColumn(field="mins",
title="Min",
width=int(plot_width * 0.075),
sortable=False),
TableColumn(field="maxs",
title="Max",
width=int(plot_width * 0.075),
sortable=False),
TableColumn(field="descriptions",
title="Description",
width=int(plot_width * 0.40),
sortable=False),
]
data_table = DataTable(source=source,
columns=columns,
width=plot_width,
height=300,
sortable=False,
selectable=False)
div = Div(
text=
"""<b>Non-default Parameters</b> (except RC and sensor calibration)""",
width=int(plot_width / 2))
plots.append(widgetbox(div, data_table, width=plot_width))
# log messages
log_times = []
log_levels = []
log_messages = []
for m in ulog.logged_messages:
m1, s1 = divmod(int(m.timestamp / 1e6), 60)
h1, m1 = divmod(m1, 60)
log_times.append("{:d}:{:02d}:{:02d}".format(h1, m1, s1))
log_levels.append(m.log_level_str())
log_messages.append(m.message)
log_data = dict(times=log_times, levels=log_levels, messages=log_messages)
source = ColumnDataSource(log_data)
columns = [
TableColumn(field="times",
title="Time",
width=int(plot_width * 0.15),
sortable=False),
TableColumn(field="levels",
title="Level",
width=int(plot_width * 0.1),
sortable=False),
TableColumn(field="messages",
title="Message",
width=int(plot_width * 0.75),
sortable=False),
]
data_table = DataTable(source=source,
columns=columns,
width=plot_width,
height=300,
sortable=False,
selectable=False)
div = Div(text="""<b>Logged Messages</b>""", width=int(plot_width / 2))
plots.append(widgetbox(div, data_table, width=plot_width))
curdoc().template_variables['plots'] = jinja_plot_data
return plots
def scattergraph_tab(floorplans):
df = floorplans
# column_names
# df = df.rename(columns={'project_name':'Neighborhood','builder_name':'Builder','city':'Market','master_plan':'Community','typical_lot_size':'Lot Size','total_units_planned':'Total Units','total_units_sold':'Units Sold','total_remaining':'Unsold Homes','hoa1':'HOA','assessments':'Tax Rate','plan_name_1_field':'Plan Name','field_of_beds_1_field':'Beds','field_of_floors_1_field':'Floors','garage_1_field':'Garages','square_footage_1_field':'Square Footage','price_1_field':'Base Price'})
# colors
# movies["color"] = np.where(movies["Oscars"] > 0, "orange", "grey")
# movies["alpha"] = np.where(movies["Oscars"] > 0, 0.9, 0.25)
# movies.fillna(0, inplace=True) # just replace missing values with zero
# movies["revenue"] = movies.BoxOffice.apply(lambda x: '{:,d}'.format(int(x)))
axis_map = {
"Home Size": "Square Footage (1)",
"Base Price": "Price (1)",
"Lot Size (sf)": "Typical Lot Size",
"Latitude": "Lat",
"Longitude": "Long",
"Sales Rate": "Sales Rate"
}
# desc =
# Div(text=open(join(dirname(__file__), "description.html")).read(), sizing_mode="stretch_width")
# Max and Min Values
sf_max = math.ceil(df['Square Footage (1)'].max() / 500) * 500
sf_min = math.floor(df['Square Footage (1)'].min() / 500) * 500
price_max = math.ceil(df['Price (1)'].max() / 50000) * 50000
price_min = math.floor(df['Price (1)'].min() / 50000) * 50000
lot_max = math.ceil(df['Typical Lot Size'].max() / 500) * 500
lot_min = math.floor(df['Typical Lot Size'].min() / 500) * 500
# Input controls
max_sf_widget = TextInput(title="Maximum Square Footage",
value=str(sf_max))
min_sf_widget = TextInput(title="Minimum Square Footage",
value=str(sf_min))
max_price_widget = TextInput(title="Maximum Base Price",
value=str(price_max))
min_price_widget = TextInput(title="Minimum Base Price",
value=str(price_min))
max_lot_size_widget = TextInput(title="Maximum Lot Size (sf)",
value=str(lot_max))
min_lot_size_widget = TextInput(title="Minimum Lot Size (sf)",
value=str(lot_min))
# builder_select = MultiSelect(title="Builder Select", value=sorted(list(df['Builder Name'].unique())),options=sorted(list(df['Builder Name'].unique())))
# market_select = MultiSelect(title="Market Select", value=sorted(list(df['City'].unique())),options=sorted(list(df['City'].unique())))
x_axis = Select(title="X Axis",
options=sorted(axis_map.keys()),
value="Home Size")
y_axis = Select(title="Y Axis",
options=sorted(axis_map.keys()),
value="Base Price")
fha_span = Span(location=403000,
dimension='width',
line_color='blue',
line_dash='dashed',
line_width=.1)
cll_span = Span(location=453100,
dimension='width',
line_color='green',
line_dash='dashed',
line_width=.1)
sf_slider = RangeSlider(title="Square Footage",
start=sf_min,
end=sf_max,
value=(sf_min, sf_max),
step=500,
value_throttled=(250, 250))
price_slider = RangeSlider(title="Base Price",
start=price_min,
end=price_max,
value=(price_min, price_max),
step=500,
value_throttled=(50000, 50000))
lot_slider = RangeSlider(title="Lot Size (SF)",
start=lot_min,
end=lot_max,
value=(lot_min, lot_max),
step=500,
value_throttled=(250, 250))
# Create Column Data Source that will be used by the plot
source = ColumnDataSource(data=dict(
SF=[],
Price=[],
Builder=[],
Color=[],
Neighborhood=[],
LotSize=[],
Market=[],
Sales_Rate=[],
Total_Homes=[],
Homes_Sold=[],
Unsold_Homes=[],
Beds=[],
Baths=[],
HOA=[],
Tax_Rate=[],
Floors=[],
Lat=[],
Long=[],
Submarket=[],
OpenDate=[])) # source = ColumnDataSource(data=df)
# TOOLTIPS=[
# ("Title", "@title"),
# ("Year", "@year"),
# ("$", "@revenue")
# ]
floors = [str(x) for x in df['# of Floors (1)'].unique()]
markers = ['hex', 'circle_x', 'triangle', 'square']
# p = figure(plot_height=600, plot_width=700, title="", toolbar_location=None, tooltips=TOOLTIPS, sizing_mode="scale_both")
# Create plots and attributes
p = figure(plot_height=500,
plot_width=800,
title="Competitive Market Area")
p.scatter(
x='SF',
y='Price',
source=source,
size=15,
line_color='black',
fill_alpha=.25,
# legend='Floors',
# # marker=factor_mark('Floors', markers, floors),
# color=factor_cmap('Floors', 'Category10_4', floors)
)
map_options = GMapOptions(lat=df.Lat.mean(),
lng=df.Long.mean(),
map_type="roadmap",
zoom=8)
pmap = gmap(Google_API,
map_options,
plot_width=360,
plot_height=400,
title="CMA Map",
toolbar_location="above")
pmap.circle(x="Long",
y="Lat",
size=15,
fill_color='blue',
fill_alpha=0.25,
line_color='black',
line_width=.08,
source=source)
p.yaxis.formatter = NumeralTickFormatter(format="$ 0,0")
p.xaxis.formatter = NumeralTickFormatter(format="0,0")
# Filter dataframe based on widget inputs
def filter_df():
# builder_list = [builder_select_widget.labels[i] for i in builder_select_widget.active]
# market_list = [market_select_widget.labels[i] for i in market_select_widget.active]
# submarket_val = submarket.value.strip()
selected = df[
(df['Square Footage (1)'] <= float(sf_slider.value[1])) &
(df['Square Footage (1)'] >= float(sf_slider.value[0])) &
(df['Price (1)'] <= float(price_slider.value[1])) &
(df['Price (1)'] >= float(price_slider.value[0])) &
(df['Typical Lot Size'] <= float(lot_slider.value[1])) &
(df['Typical Lot Size'] >= float(lot_slider.value[0]))
# (df['Builder Name'].isin(builder_list)) &
# (df['City'].isin(market_list))
]
# if submarket_val != "All":
# selected = selected[selected['Submarket'].str.contains(submarket_val)==True]
return selected
# Update df
def update():
df = filter_df()
df = df.sort_values(by=['Project Name'])
p.xaxis.axis_label = x_axis.value
p.yaxis.axis_label = y_axis.value
# p.title.text = "%d Floorplans Selected, %d Communities Selected, %d Builders Selected" % (len(df), df.Neighborhood.nunique(), df.Builder.nunique())
source.data = dict(
# x=df[x_name],
# y=df[y_name],
SF=df['Square Footage (1)'],
Price=df['Price (1)'],
# Color=df['Color'],
Neighborhood=df['Project Name'],
Floors=df['# of Floors (1)'],
Builder=df['Builder Name'],
Market=df['City'],
Lot_Size=df['Typical Lot Size'],
Sales_Rate=df['Sales Rate'],
Total_Homes=df['Total Units Planned'],
Homes_Sold=df['Total Units Sold'],
Unsold_Homes=df['Total Remaining'],
HOA=df['HOA1'],
Tax_Rate=df['Assessments'],
Beds=df['# of Beds (1)'],
Baths=df['# of Baths (1)'],
Lat=df['Lat'],
Long=df['Long'],
# Submarket=df['Submarket'],
# Open_Date=df['Open Date']
)
# Add plot tools
hover_callback = CustomJS(
code="""var tooltips = document.getElementsByClassName("bk-tooltip");
for (var i = 0, len = tooltips.length; i < len; i ++) {
tooltips[i].style.top = "25px"; // unset what bokeh.js sets
tooltips[i].style.left = "100px";
tooltips[i].style.bottom = "";
tooltips[i].style.right = "";
""")
hover_callback2 = CustomJS(
code="""var tooltips = document.getElementsByClassName("bk-tooltip");
for (var i = 0, len = tooltips.length; i < len; i ++) {
tooltips[i].style.top = "25px"; // unset what bokeh.js sets
tooltips[i].style.left = "-700px";
tooltips[i].style.bottom = "";
tooltips[i].style.right = "";
}""")
hover = HoverTool(tooltips=[
('Neighborhood', '@Neighborhood'),
('Builder', '@Builder'),
('Market', '@Market'),
('Lot Size', '@{Lot_Size}{(0,0)}'
" SF"),
('Square Footage', '@{SF}{0,0SF}'
" SF"),
('Base Price', '@{Price}{$0,0}'),
('Floors', '@Floors'),
('Sales Rate', '@{Sales_Rate}{(0.0)}'
"/Mo"),
('Beds/Baths', '@{Beds}'
"/"
'@{Baths}{(0.0)}'),
],
callback=hover_callback,
show_arrow=False,
point_policy='snap_to_data')
hover2 = HoverTool(tooltips=[
('Neighborhood', '@Neighborhood'),
('Builder', '@Builder'),
('Market', '@Market'),
('Lot Size', '@{Lot_Size}{(0,0)}'
" SF"),
('Square Footage', '@{SF}{0,0SF}'
" SF"),
('Base Price', '@{Price}{$0,0}'),
('Floors', '@Floors'),
('Sales Rate', '@{Sales_Rate}{(0.0)}'
"/Mo"),
('Beds/Baths', '@{Beds}'
"/"
'@{Baths}{(0.0)}'),
],
callback=hover_callback2,
show_arrow=False,
point_policy='snap_to_data')
table_columns = [
TableColumn(field='Neighborhood', title='Neighborhood'),
TableColumn(field='Market', title='Market'),
TableColumn(field='Builder', title='Builder'),
TableColumn(field='SF',
title='Home Size',
formatter=NumberFormatter(format='0,0')),
TableColumn(field='Price',
title='Base Price',
formatter=NumberFormatter(format='$ 0,0[.]00'))
# TableColumn(field="Sales_Rate", title="Sales Rate",formatter=HTMLTemplateFormatter(template='<code><%= value +" / Mo" %></code>')),
# TableColumn(field="Lot_Size", title='Avg Lot Size',formatter=HTMLTemplateFormatter(template='<code><%= value +" SF" %></code>')),
# TableColumn(field="Open_Date", title="Open Date",formatter=DateFormatter(format="%m/%d/%Y"))
]
def floorplans_query():
print('something')
floorplans = pd.read_csv(
join(dirname(join(dirname(__file__))), 'data', 'floorplans.csv'))
con = sql.connect(join('data', 'Market.db'))
floorplans.to_sql('floorplans', con, if_exists='replace')
con.close()
print(floorplans)
return print('Data uploaded from CSV!')
scattergraph_tab(floorplans)
def download():
date = datetime.datetime.now().strftime("%m_%d_%y_h%Hm%Ms%S")
print('Testing')
df.to_csv(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'data',
'Downloads',
'scattergraph' + str(date) + '.csv')))
button_dl = Button(label="Download", button_type="success")
button_ul = Button(label="Upload data from CSV", button_type="success")
button_dl.on_click(download)
button_ul.on_click(floorplans_query)
table = DataTable(source=source,
columns=table_columns,
editable=True,
height=600,
width=1200,
fit_columns=True,
scroll_to_selection=True)
controls = [sf_slider, price_slider, lot_slider]
for control in controls:
control.on_change('value', lambda attr, old, new: update())
p.add_tools(hover)
pmap.add_tools(hover2)
pmap.add_tools(WheelZoomTool())
inputs = column([*controls, button_ul, button_dl], width=180, height=250)
inputs.sizing_mode = "fixed"
# Create layout
l = layout([[column([inputs]), p, pmap], table], sizing_mode="fixed")
tab = Panel(child=l, title='Scattergraph')
# Initial call of update
update()
# Return
return tab
class PeakPicking(Observer):
def __init__(self, logger, spectrumId, dic, udic, pdata, dataSource, reference):
Observer.__init__(self, logger)
self.logger = logger
self.id = spectrumId
self.dic = dic
self.udic = udic
self.pdata = pdata
self.mpdata = np.array(map(lambda x: -x, pdata))
self.dataSource = dataSource
reference.addObserver(lambda n: referenceObserver(self, n))
self.sources = dict()
self.sources['peaks'] = ColumnDataSource(data=dict(x=[], y=[]))
def create(self):
self.sources['table'] = ColumnDataSource(dict(x=[], y=[]))
self.sources['background'] = ColumnDataSource(dict(x=[], y=[]))
columns = [
TableColumn(field="x", title="ppm", formatter=NumberFormatter(format="0.00")),
TableColumn(field="y", title="y", formatter=NumberFormatter(format="0.00"))
]
self.dataTable = DataTable(source=self.sources['table'], columns=columns, reorderable=False, width=500)
self.sources['table'].on_change('selected', lambda attr, old, new: self.rowSelect(new['1d']['indices']))
self.sources['table'].on_change('data', lambda attr, old, new: self.dataChanged(old, new))
self.manual = CustomButton(label="Manual Peaks", button_type="success", width=500, error="Please select area using the peak picking tool.")
self.manual.on_click(self.manualPeakPicking)
self.peak = CustomButton(label="Peak By Peak", button_type="primary", width=250, error="Please select area using the peak by peak tool.")
self.peak.on_click(self.peakByPeakPicking)
self.peakTool = CustomTapTool.Create(self.peak, tapTool=PeakByPeakTapTool, auto=True, id="peakByPeakTool")
self.createManualTool()
self.createDeselectButton()
self.createDeleteButton()
self.chemicalShiftReportTitle = Div(text="<strong>Chemical Shift Report</strong>" if getLabel(self.udic) == "13C" else "")
self.chemicalShiftReport = Paragraph(text=self.getChemicalShiftReport(), width=500)
def createManualTool(self):
callback = CustomJS(args=dict(button=self.manual), code="""
/// get BoxSelectTool dimensions from cb_data parameter of Callback
var geometry = cb_data['geometry'];
button.data = {
x0: geometry['x0'],
x1: geometry['x1'],
y: geometry['y']
};
// Callback to the backend
button.clicks++;
""")
self.manualTool = BothDimensionsSelectTool(
tool_name = "Peak Picking By Threshold",
icon = "my_icon_peak_picking",
callback = callback,
id = "peakPickingByThresholdTool"
)
def dataChanged(self, old, new):
label = getLabel(self.udic)
if label == "13C":
added = [(peak, 'm') for peak in (set(new['x']) - set(old['x']))]
removed = [(peak, 'm') for peak in (set(old['x']) - set(new['x']))]
SpectrumDB.RemovePeaks(self.id, removed)
SpectrumDB.AddPeaks(self.id, added)
# Update Chemical Shift Report
self.updateChemicalShiftReport()
def updateChemicalShiftReport(self):
self.chemicalShiftReport.text = self.getChemicalShiftReport()
def getChemicalShiftReport(self):
label = getLabel(self.udic)
if label == "13C":
return getMetadata(self.dic, self.udic) + " δ " + ", ".join("{:0.2f}".format(x) for x in [round(x, 2) for x in self.sources['table'].data['x']]) + "."
else:
return ""
def createDeselectButton(self):
self.deselectButton = Button(label="Deselect all peaks", button_type="default", width=250)
self.deselectButton.on_click(lambda: deselectRows(self.sources['table']))
def createDeleteButton(self):
self.ids = []
self.deleteButton = Button(label="Delete selected peaks", button_type="danger", width=250)
self.deleteButton.on_click(self.deletePeaks)
def deletePeaks(self):
self.sources['peaks'].data = dict(x=[], y=[])
newX = list(self.sources['table'].data['x'])
newY = list(self.sources['table'].data['y'])
ids = self.sources['table'].selected['1d']['indices']
for i in sorted(ids, reverse=True):
try:
newX.pop(i)
newY.pop(i)
except IndexError:
pass
self.sources['table'].data = {
'x': list(newX),
'y': list(newY)
}
self.sources['background'].data = {
'x': list(newX),
'y': list(newY)
}
deselectRows(self.sources['table'])
self.notifyObservers()
def manualPeakPicking(self, dimensions, notify=True):
# Positive Peaks
self.peaksIndices = list(self.manualPeakPickingOnData(self.pdata, dimensions))
# Negative Peaks
self.peaksIndices.extend(self.manualPeakPickingOnData(self.mpdata, dimensions))
# Sort Peaks
self.peaksIndices = sorted(self.peaksIndices, reverse=True)
if len(self.peaksIndices) > 0:
self.updateDataValues({
'x': [self.dataSource.data['ppm'][i] for i in self.peaksIndices],
'y': [self.pdata[i] for i in self.peaksIndices]
})
if notify:
self.notifyObservers()
def manualPeakPickingOnData(self, data, dimensions):
threshold = abs(dimensions['y'])
if data.max() < threshold:
return []
peaks = ng.peakpick.pick(data, abs(dimensions['y']), algorithm="downward")
peaksIndices = [int(peak[0]) for peak in peaks]
# Filter left
peaksIndices = [i for i in peaksIndices if self.dataSource.data['ppm'][i] <= dimensions['x0']]
# Filter right
peaksIndices = [i for i in peaksIndices if self.dataSource.data['ppm'][i] >= dimensions['x1']]
return peaksIndices
def peakByPeakPicking(self, dimensions):
self.updateDataValues({
'x': [dimensions['x']],
'y': [dimensions['y']]
})
self.notifyObservers()
def updateDataValues(self, data):
# Update DataTable Values
newData = list(OrderedDict.fromkeys(
zip(
self.sources['table'].data['x'] + data['x'],
self.sources['table'].data['y'] + data['y']
)
))
newX, newY = zip(*sorted(newData, reverse=True))
self.sources['table'].data = {
'x': list(newX),
'y': list(newY)
}
self.sources['background'].data = {
'x': list(newX),
'y': list(newY)
}
def selectByPPM(self, peaks):
self.sources['table'].selected = {
'0d': {'glyph': None, 'indices': []},
'1d': {'indices': [self.sources['table'].data['x'].index(peak) for peak in peaks]},
'2d': {'indices': {}}
}
def rowSelect(self, ids):
self.sources['peaks'].data = {
'x': [self.sources['table'].data['x'][i] for i in ids],
'y': [self.sources['table'].data['y'][i] for i in ids]
}
def getPeaksInSpace(self, start, stop):
return [y for x, y in zip(self.sources['table'].data['x'], self.sources['table'].data['y']) if x <= start and x >= stop]
def getPPMInSpace(self, start, stop):
return [x for x in self.sources['table'].data['x'] if x <= start and x >= stop]
def draw(self, plot):
peak = Circle(
x="x",
y="y",
size=10,
line_color="#C0C0C0",
fill_color="#C0C0C0",
line_width=1
)
plot.add_glyph(self.sources['background'], peak, selection_glyph=peak, nonselection_glyph=peak)
selected = Circle(
x="x",
y="y",
size=10,
line_color="#ff0000",
fill_color="#ff0000",
line_width=1
)
plot.add_glyph(self.sources['peaks'], selected, selection_glyph=selected, nonselection_glyph=selected)
self.manualTool.addToPlot(plot)
plot.add_tools(self.peakTool)
def tab_testing():
data_list = glob.glob('./np/Regression/*.npy')
data_list = sorted(data_list)
select_data = Select(title="Data:", value="", options=data_list)
model_list = os.listdir('./model/Regression/')
model_list = sorted(model_list)
select_model = Select(title="Trained Model:", value="", options=model_list)
notifier = Paragraph(text=""" Notification """, width=200, height=100)
def refresh_handler():
data_list_new = glob.glob('./np/Regression/*.npy')
data_list_new = sorted(data_list_new)
select_data.options = data_list_new
model_list_new = os.listdir('./model/Regression/')
model_list_new = sorted(model_list_new)
select_model.options = model_list_new
button_refresh = Button(label="Refresh list")
button_refresh.on_click(refresh_handler)
button_test = Button(label="Test model")
select_result = MultiSelect(title="Key(result):")
src = ColumnDataSource()
df = pd.DataFrame(columns=['key', 'y', 'y_hat'])
table_src = ColumnDataSource(pd.DataFrame(columns=['Key', 'MSE', 'R^2']))
table_columns = [
TableColumn(field=col, title=col) for col in ['Key', 'MSE', 'R^2']
]
table_acc = DataTable(source=table_src,
columns=table_columns,
width=350,
height=400,
fit_columns=True,
name="Accuracy per Key")
def test_handler():
df.drop(df.index, inplace=True)
print("Start test")
tf.reset_default_graph()
K.clear_session()
notifier.text = """ Start testing """
if (select_data.value == ""):
data = np.load(select_data.options[0])
else:
data = np.load(select_data.value)
data = data.item()
notifier.text = """ Import data """
data_x = data.get('x')
if (data_x.shape[-1] == 1 and not 'cnn' in select_model.value):
data_x = np.squeeze(data_x, -1)
data_y = data.get('y')
data_key = data.get('key1')
print(data_x.shape)
print(data_y.shape)
df['key'] = data_key
df['y'] = data_y[:, 0]
op_list = []
for i in df['key'].unique():
op_list.append(str(i))
select_result.options = op_list
print(data_x.shape)
print(data_y.shape)
print(data.get('key1'))
if (select_model.value == ""):
model_name = select_model.options[0]
else:
model_name = select_model.value
model_save_dir = './model/Regression/' + model_name + '/'
model_dl = glob.glob(model_save_dir + '*.h5')
model_ml = glob.glob(model_save_dir + '*.sav')
print(model_save_dir + '*.h5')
print(model_dl)
print(model_ml)
if (len(model_dl) > len(model_ml)):
model = keras.models.load_model(model_save_dir + model_name +
'.h5')
target_hat = model.predict(data_x)
DL = True
elif (len(model_dl) < len(model_ml)):
model = pickle.load(
open(model_save_dir + model_name + '.sav', 'rb'))
data_x = data_x.reshape([data_x.shape[0], -1])
target_hat = model.predict(data_x)
target_hat = np.expand_dims(target_hat, -1)
DL = False
notifier.text = """ Model restored """
print("Model restored.")
xs = []
ys = []
keys = []
color = ['blue', 'red']
xs.append([i for i in range(data_y.shape[0])])
xs.append([i for i in range(data_y.shape[0])])
ys.append(data_y)
keys.append(data_key)
print(target_hat.shape)
K.clear_session()
ys.append(target_hat)
keys.append(data_key)
print(target_hat[:, 0])
df['y_hat'] = target_hat[:, 0]
src.data = ColumnDataSource(
data=dict(xs=xs, ys=ys, color=color, keys=keys)).data
figure_trend.multi_line('xs', 'ys', source=src, color='color')
line_mse = []
line_r_2 = []
for unit in df['key'].unique():
target = df[df['key'] == unit]
y = target['y'].values
y_hat = target['y_hat'].values
unit_mse = np.sum((y - y_hat)**2) / target.shape[0]
unit_r_2 = np.max([r2_score(y, y_hat), 0])
line_mse.append(unit_mse)
line_r_2.append(unit_r_2)
acc = pd.DataFrame(columns=['Key', 'MSE', 'R^2'])
acc['Key'] = df['key'].unique()
mse_mean = np.mean(line_mse)
r_2_mean = np.mean(line_r_2)
line_mse = list(map(lambda x: format(x, '.2f'), line_mse))
acc['MSE'] = line_mse
line_r_2 = list(map(lambda x: format(x, '.2f'), line_r_2))
acc['R^2'] = line_r_2
acc_append = pd.DataFrame(columns=acc.columns)
acc_append['Key'] = ['MSE average', 'R^2 average']
acc_append['MSE'] = [mse_mean, r_2_mean]
acc = pd.concat([acc, acc_append])
table_src.data = ColumnDataSource(acc).data
notifier.text = """ Drawing complete """
history.text = history.text + "\n\t" + model_name + "'s R^2 score: " + format(
np.mean(r_2_mean), '.2f')
def update(attr, old, new):
key_to_plot = select_result.value
xs = []
ys = []
keys = []
y = []
y_hat = []
key = []
key_type = type(df['key'].values[0])
for k in key_to_plot:
y += list(df[df['key'] == key_type(k)]['y'].values)
y_hat += list(df[df['key'] == key_type(k)]['y_hat'].values)
key += [k for _ in range(df[df['key'] == key_type(k)].shape[0])]
ys.append(y)
ys.append(y_hat)
xs.append([i for i in range(len(y))])
xs.append([i for i in range(len(y))])
keys.append(key)
keys.append(key)
color = ['blue', 'red']
src.data = ColumnDataSource(
data=dict(xs=xs, ys=ys, color=color, keys=keys)).data
select_result.on_change("value", update)
button_test.on_click(test_handler)
button_export = Button(label="Export result")
def handler_export():
df.to_csv('./Export/result.csv', index=False)
button_export.on_click(handler_export)
figure_trend = figure(title="Prediction result", width=800, height=460)
history = PreText(text="", width=300, height=460)
layout = Column(
Row(button_refresh),
Row(select_data, select_model, button_test, select_result, notifier),
Row(table_acc, figure_trend, history, button_export))
tab = Panel(child=layout, title='Regression Test')
return tab
session.store_document(document)
def radio_group_handler(active):
print("radio_group_handler: %s" % active)
session.store_document(document)
def checkbox_button_group_handler(active):
print("checkbox_button_group_handler: %s" % active)
session.store_document(document)
def radio_button_group_handler(active):
print("radio_button_group_handler: %s" % active)
session.store_document(document)
button = Button(label="Push button", icon=Icon(name="check"), type="primary")
button.on_click(button_handler)
toggle = Toggle(label="Toggle button", type="success")
toggle.on_click(toggle_handler)
menu = [("Item 1", "item_1"), ("Item 2", "item_2"), None, ("Item 3", "item_3")]
dropdown = Dropdown(label="Dropdown button", type="warning", menu=menu)
dropdown.on_click(dropdown_handler)
menu = [("Item 1", "foo"), ("Item 2", "bar"), None, ("Item 3", "baz")]
split = Dropdown(label="Split button", type="danger", menu=menu, default_action="baz")
split.on_click(split_handler)
checkbox_group = CheckboxGroup(labels=["Option 1", "Option 2", "Option 3"], active=[0, 1])
checkbox_group.on_click(checkbox_group_handler)
def ButtonWidget(*args, **kw):
kw['label'] = kw.pop('title')
cb = kw.pop('value')
button = Button(*args, **kw)
button.on_click(cb)
return button
def plot():
# FIGURES AND X-AXIS
fig1 = Figure(title = 'Dive Profile', plot_width = WIDTH, plot_height = HEIGHT, tools = TOOLS)
fig2 = Figure(title = 'Dive Controls', plot_width = WIDTH, plot_height = HEIGHT, tools = TOOLS, x_range=fig1.x_range)
fig3 = Figure(title = 'Attitude', plot_width = WIDTH, plot_height = HEIGHT, tools = TOOLS, x_range=fig1.x_range)
figs = gridplot([[fig1],[fig2],[fig3]])
# Formatting x-axis
timeticks = DatetimeTickFormatter(formats=dict(seconds =["%b%d %H:%M:%S"],
minutes =["%b%d %H:%M"],
hourmin =["%b%d %H:%M"],
hours =["%b%d %H:%M"],
days =["%b%d %H:%M"],
months=["%b%d %H:%M"],
years =["%b%d %H:%M %Y"]))
fig1.xaxis.formatter = timeticks
fig2.xaxis.formatter = timeticks
fig3.xaxis.formatter = timeticks
# removing gridlines
fig1.xgrid.grid_line_color = None
fig1.ygrid.grid_line_color = None
fig2.xgrid.grid_line_color = None
fig2.ygrid.grid_line_color = None
fig3.xgrid.grid_line_color = None
fig3.ygrid.grid_line_color = None
# INPUT WIDGETS
collection_list = CONN[DB].collection_names(include_system_collections=False)
gliders = sorted([platformID for platformID in collection_list if len(platformID)>2])
gliders = Select(title = 'PlatformID', value = gliders[0], options = gliders)
prev_glider = Button(label = '<')
next_glider = Button(label = '>')
glider_controlbox = HBox(children = [gliders, prev_glider, next_glider], height=80)
chunkations = Select(title = 'Chunkation', value = 'segment', options = ['segment', '24hr', '30days', '-ALL-'])
chunk_indicator = TextInput(title = 'index', value = '0')
prev_chunk = Button(label = '<')
next_chunk = Button(label = '>')
chunk_ID = PreText(height=80)
chunk_controlbox = HBox(chunkations,
HBox(chunk_indicator, width=25),
prev_chunk, next_chunk,
chunk_ID,
height = 80)
control_box = HBox(glider_controlbox,
chunk_controlbox)
# DATA VARS
deadby_date = ''
depth = ColumnDataSource(dict(x=[],y=[]))
vert_vel = ColumnDataSource(dict(x=[],y=[]))
mbpump = ColumnDataSource(dict(x=[],y=[]))
battpos = ColumnDataSource(dict(x=[],y=[]))
pitch = ColumnDataSource(dict(x=[],y=[]))
mfin = ColumnDataSource(dict(x=[],y=[]))
cfin = ColumnDataSource(dict(x=[],y=[]))
mroll = ColumnDataSource(dict(x=[],y=[]))
mheading = ColumnDataSource(dict(x=[],y=[]))
cheading = ColumnDataSource(dict(x=[],y=[]))
# AXIS setup
colors = COLORS[:]
fig1.y_range.flipped = True
fig1.yaxis.axis_label = 'm_depth (m)'
fig1.extra_y_ranges = {'vert_vel': Range1d(start=-50, end=50),
'dummy': Range1d(start=0, end=100)}
fig1.add_layout(place = 'right',
obj = LinearAxis(y_range_name = 'vert_vel',
axis_label = 'vertical velocity (cm/s)'))
fig1.add_layout(place = 'left',
obj = LinearAxis(y_range_name = 'dummy',
axis_label = ' '))
fig1.yaxis[1].visible = False
fig1.yaxis[1].axis_line_alpha = 0
fig1.yaxis[1].major_label_text_alpha = 0
fig1.yaxis[1].major_tick_line_alpha = 0
fig1.yaxis[1].minor_tick_line_alpha = 0
fig2.yaxis.axis_label = 'pitch (deg)'
fig2.y_range.start, fig2.y_range.end = -40,40
fig2.extra_y_ranges = {'battpos': Range1d(start=-1, end = 1),
'bpump': Range1d(start=-275, end=275)}
fig2.add_layout(place = 'right',
obj = LinearAxis(y_range_name = 'battpos',
axis_label = 'battpos (in)'))
fig2.add_layout(place = 'left',
obj = LinearAxis(y_range_name = 'bpump',
axis_label = 'bpump (cc)'))
fig2.yaxis[1].visible = False # necessary for spacing. later gets set to true
fig3.yaxis.axis_label = 'fin/roll (deg)'
fig3.y_range.start, fig3.y_range.end = -30, 30
fig3.extra_y_ranges = {'heading': Range1d(start=0, end=360), #TODO dynamic avg centering
'dummy': Range1d(start=0, end=100)}
fig3.add_layout(place = 'right',
obj = LinearAxis(y_range_name = 'heading',
axis_label = 'headings (deg)'))
fig3.add_layout(place = 'left',
obj = LinearAxis(y_range_name = 'dummy',
axis_label = ' '))
fig3.yaxis[1].visible = False
fig3.yaxis[1].axis_line_alpha = 0
fig3.yaxis[1].major_label_text_alpha = 0
fig3.yaxis[1].major_tick_line_alpha = 0
fig3.yaxis[1].minor_tick_line_alpha = 0
# PLOT OBJECTS
fig1.line( 'x', 'y', source = depth, legend = 'm_depth', color = 'red')
fig1.circle('x', 'y', source = depth, legend = 'm_depth', color = 'red')
fig1.line( 'x', 'y', source = vert_vel, legend = 'vert_vel', color = 'green', y_range_name = 'vert_vel')
fig1.circle('x', 'y', source = vert_vel, legend = 'vert_vel', color = 'green', y_range_name = 'vert_vel')
fig1.renderers.append(Span(location = 0, dimension = 'width', y_range_name = 'vert_vel',
line_color= 'green', line_dash='dashed', line_width=1))
fig2.line( 'x', 'y', source = pitch, legend = "m_pitch", color = 'indigo')
fig2.circle('x', 'y', source = pitch, legend = "m_pitch", color = 'indigo')
fig2.line( 'x', 'y', source = battpos, legend = 'm_battpos', color = 'magenta', y_range_name = 'battpos')
fig2.circle('x', 'y', source = battpos, legend = 'm_battpos', color = 'magenta', y_range_name = 'battpos')
fig2.line( 'x', 'y', source = mbpump, legend = "m_'bpump'", color = 'blue', y_range_name = 'bpump')
fig2.circle('x', 'y', source = mbpump, legend = "m_'bpump'", color = 'blue', y_range_name = 'bpump')
fig2.renderers.append(Span(location = 0, dimension = 'width',
line_color= 'black', line_dash='dashed', line_width=1))
fig3.line( 'x', 'y', source = mfin, legend = 'm_fin', color = 'cyan')
fig3.circle('x', 'y', source = mfin, legend = 'm_fin', color = 'cyan')
fig3.line( 'x', 'y', source = cfin, legend = 'c_fin', color = 'orange')
fig3.circle('x', 'y', source = cfin, legend = 'c_fin', color = 'orange')
fig3.line( 'x', 'y', source = mroll, legend = 'm_roll', color = 'magenta')
fig3.circle('x', 'y', source = mroll, legend = 'm_roll', color = 'magenta')
fig3.line( 'x', 'y', source = mheading, legend = 'm_heading', color = 'blue', y_range_name = 'heading')
fig3.circle('x', 'y', source = mheading, legend = 'm_heading', color = 'blue', y_range_name = 'heading')
fig3.line( 'x', 'y', source = cheading, legend = 'c_heading', color = 'indigo', y_range_name = 'heading')
fig3.circle('x', 'y', source = cheading, legend = 'c_heading', color = 'indigo', y_range_name = 'heading')
fig3.renderers.append(Span(location = 0, dimension = 'width', y_range_name = 'default',
line_color= 'black', line_dash='dashed', line_width=1))
# CALLBACK FUNCS
def update_data(attrib,old,new):
g = gliders.value
chnk = chunkations.value
chindex = abs(int(chunk_indicator.value))
depth.data = dict(x=[],y=[])
vert_vel.data = dict(x=[],y=[])
mbpump.data = dict(x=[],y=[])
battpos.data = dict(x=[],y=[])
pitch.data = dict(x=[],y=[])
mfin.data = dict(x=[],y=[])
cfin.data = dict(x=[],y=[])
mroll.data = dict(x=[],y=[])
mheading.data = dict(x=[],y=[])
cheading.data = dict(x=[],y=[])
depth.data,startend = load_sensor(g, 'm_depth', chnk, chindex)
if chnk == 'segment':
xbd = startend[2]
chunk_ID.text = '{} {} \n{} ({}) \nSTART: {} \nEND: {}'.format(g, xbd['mission'],
xbd['onboard_filename'], xbd['the8x3_filename'],
e2ts(xbd['start']), e2ts(xbd['end']))
if len(set(depth.data['x']))<=1 and attrib == 'chunk':
if old > new:
next_chunk.clicks += 1
else:
prev_chunk.clicks += 1
return
elif len(set(depth.data['x']))<=1 and chunk_indicator.value == 0:
chunk_indicator.value = 1
elif chnk in ['24hr', '30days']:
chunk_ID.text = '{} \nSTART: {} \nEND: {}'.format(g, e2ts(startend[0]), e2ts(startend[1]))
elif chnk == '-ALL-':
chunk_ID.text = '{} \nSTART: {} \nEND: {}'.format(g,e2ts(depth.data['x'][0] /1000),
e2ts(depth.data['x'][-1]/1000))
vert_vel.data = calc_vert_vel(depth.data)
mbpump.data,_ = load_sensor(g, 'm_de_oil_vol', chnk, chindex)
if len(mbpump.data['x']) > 1:
#for yax in fig2.select('mbpump'):
# yax.legend = 'm_de_oil_vol'
pass
else:
mbpump.data,_ = load_sensor(g, 'm_ballast_pumped', chnk, chindex)
#for yax in fig2.select('mbpump'):
# yax.legend = 'm_ballast_pumped'
battpos.data,_ = load_sensor(g, 'm_battpos', chnk, chindex)
pitch.data,_ = load_sensor(g, 'm_pitch', chnk, chindex)
pitch.data['y'] = [math.degrees(y) for y in pitch.data['y']]
mfin.data,_ = load_sensor(g, 'm_fin', chnk, chindex)
cfin.data,_ = load_sensor(g, 'c_fin', chnk, chindex)
mroll.data,_ = load_sensor(g, 'm_roll', chnk, chindex)
mheading.data,_ = load_sensor(g, 'm_heading', chnk, chindex)
cheading.data,_ = load_sensor(g, 'c_heading', chnk, chindex)
mfin.data['y'] = [math.degrees(y) for y in mfin.data['y']]
cfin.data['y'] = [math.degrees(y) for y in cfin.data['y']]
mheading.data['y'] = [math.degrees(y) for y in mheading.data['y']]
cheading.data['y'] = [math.degrees(y) for y in cheading.data['y']]
mroll.data['y'] = [math.degrees(y) for y in mroll.data['y']]
fig1.yaxis[1].visible = True
fig2.yaxis[1].visible = True
fig3.yaxis[1].visible = True
#GLIDER SELECTS
def glider_buttons(increment):
ops = gliders.options
new_index = ops.index(gliders.value) + increment
if new_index >= len(ops):
new_index = 0
elif new_index < 0:
new_index = len(ops)-1
gliders.value = ops[new_index]
chunkation_update(None, None, None) #reset chunk indicator and clicks
def next_glider_func():
glider_buttons(1)
def prev_glider_func():
glider_buttons(-1)
def update_glider(attrib,old,new):
chunk_indicator.value = '0'
#update_data(None,None,None)
gliders.on_change('value', update_glider)
next_glider.on_click(next_glider_func)
prev_glider.on_click(prev_glider_func)
#CHUNK SELECTS
def chunkation_update(attrib,old,new):
chunk_indicator.value = '0'
prev_chunk.clicks = 0
next_chunk.clicks = 0
update_data(None,None,None)
if new == '-ALL-':
chunk_indicator.value = '-'
def chunk_func():
chunkdiff = prev_chunk.clicks - next_chunk.clicks
if chunkdiff < 0:
prev_chunk.clicks = 0
next_chunk.clicks = 0
chunkdiff = 0
print (chunkdiff)
chunk_indicator.value = str(chunkdiff)
def chunk_indicator_update(attrib,old,new):
try:
if abs(int(old)-int(new))>1: #manual update, triggers new non-manual indicator update, ie else clause below
prev_chunk.clicks = int(new)
next_chunk.clicks = 0
else:
update_data('chunk',int(old),int(new))
print("UPDATE", old, new)
except Exception as e:
print(type(e),e, old, new)
chunkations.on_change('value', chunkation_update)
chunk_indicator.on_change('value', chunk_indicator_update)
next_chunk.on_click(chunk_func)
prev_chunk.on_click(chunk_func)
update_data(None,None,None)
return vplot(control_box, figs)
hover = t.select_one(HoverTool)
hover.point_policy = "follow_mouse"
hover.tooltips = [
("task type", "@tasktype"),
("#tasks", "@running_tasks"),
]
# Function that clears all checkboxes and calls checkbox to redraw the plots
def clear():
checkbox_group_p.active = []
checkbox("", "", "")
# Button to clear all checkboxes
clear_button = Button(label="clear all", width=20)
clear_button.on_click(clear)
# Function that sets all checkboxes and calls checkbox to redraw the plots
def select_all():
checkbox_group_p.active = [i for i in range(len(attributes[1:]))]
checkbox("", "", "")
# Button to select all checkboxes
all_button = Button(label="select all", width=20)
all_button.on_click(select_all)
layout = column(
#row(WidgetBox(dropdown, width=405, height=100), WidgetBox(width=500),WidgetBox(runID)),
row(WidgetBox(dropdown, width=410, height=100)),
row(runID, startTime),
# row(runID, numMessages),
class Chart_Panel(object):
def __init__(self, ut330):
self.ut330 = ut330
self.source = ColumnDataSource(data=dict(ts=[], t=[], h=[]))
self.plot = figure(x_axis_type="datetime")
self.plot.title = "Temperature and humidity vs. time"
self.plot.line(x="ts",
y="t",
source=self.source,
color="blue",
legend="Temperature",
line_width=2)
self.plot.xaxis.axis_label = "Timestamp"
self.plot.yaxis.axis_label = "Temperature (C)"
self.plot.extra_y_ranges = {"humidity": Range1d(0, 100)}
self.plot.line(x="ts",
y="h",
source=self.source,
y_range_name="humidity",
color="green",
legend="Humidity",
line_width=2)
self.plot.add_layout(LinearAxis(y_range_name="humidity",
axis_label="Relative humidity (%)"),
'right')
self.read = Button(label='Read data')
self.delete = Button(label='Delete data')
def _layout_(self):
return VBox(HBox(self.read, self.delete), self.plot)
def panel(self):
return Panel(child=self._layout_(), title="Chart")
def device_delete(self):
self.ut330.delete_data()
def device_read(self):
self.data = self.ut330.read_data()
df = pd.DataFrame(self.data)
self.source.data = dict(ts=df['timestamp'],
t=df['temperature'],
h=df['humidity'])
ymin = 10*int(df['temperature'].min()/10)
ymax = 10*int((10+df['temperature'].max())/10)
self.plot.y_range = Range1d(ymin, ymax)
ymin = 10*int(df['humidity'].min()/10)
ymax = 10*int((10+df['humidity'].max())/10)
self.plot.extra_y_ranges = {"humidity": Range1d(ymin, ymax)}
def callbacks(self):
self.read.on_click(self.device_read)
self.delete.on_click(self.device_delete)
print("split_handler: %s" % value)
def checkbox_group_handler(active):
print("checkbox_group_handler: %s" % active)
def radio_group_handler(active):
print("radio_group_handler: %s" % active)
def checkbox_button_group_handler(active):
print("checkbox_button_group_handler: %s" % active)
def radio_button_group_handler(active):
print("radio_button_group_handler: %s" % active)
button = Button(label="Push button", icon=Icon(name="check"), type="primary")
button.on_click(button_handler)
toggle = Toggle(label="Toggle button", type="success")
toggle.on_click(toggle_handler)
menu = [("Item 1", "item_1"), ("Item 2", "item_2"), None, ("Item 3", "item_3")]
dropdown = Dropdown(label="Dropdown button", type="warning", menu=menu)
dropdown.on_click(dropdown_handler)
menu = [("Item 1", "foo"), ("Item 2", "bar"), None, ("Item 3", "baz")]
split = Dropdown(label="Split button", type="danger", menu=menu, default_value="baz")
split.on_click(split_handler)
checkbox_group = CheckboxGroup(labels=["Option 1", "Option 2", "Option 3"], active=[0, 1])
checkbox_group.on_click(checkbox_group_handler)
def plotting(self):
#Tools = [hover, TapTool(), BoxZoomTool(), BoxSelectTool(), PreviewSaveTool(), ResetTool()]
TOOLS="crosshair,pan,wheel_zoom,box_zoom,reset,hover,previewsave"
tab_plots = []
#output_file("test.html")
self.all_elements = []
self.elements_comparison = []
for attr_id, i in zip(self.attribute_ids, range(len(self.attribute_ids))):
"""
create plots for each datafile and put them in a tab.
"""
list_of_datasets = getattr(self, attr_id)
y_axis_units = [x["y_unit"] for x in list_of_datasets]
x_axis_units = [x["x_unit"] for x in list_of_datasets]
figure_obj = figure(plot_width = 1000, plot_height = 800, y_axis_type = "log",
title = attr_id, tools = TOOLS)
#figure_obj.axes.major_label_text_font_size("12pt")
#figure_obj.major_label_text_font_size("12pt")
setattr(self, attr_id+"_"+"figure_obj",figure_obj)
figure_obj.yaxis.axis_label = y_axis_units[0]
figure_obj.xaxis.axis_label = x_axis_units[0]
if not all(x == y_axis_units[0] for x in y_axis_units):
for unit, data in zip(y_axis_units, list_of_datasets):
if not unit == y_axis_units[0]:
figure_obj.extra_y_ranges = {"foo": Range1d(start = np.amin(data["data"]["y"]),
end = np.amax(data["data"]["y"]))}
figure_obj.add_layout(LogAxis(y_range_name = "foo", axis_label = unit), "right")
break
if not all(x == x_axis_units[0] for x in x_axis_units):
for unit, data in zip(x_axis_units, list_of_datasets):
if not unit == x_axis_units[0]:
figure_obj.extra_x_ranges = {"bar": Range1d(start = np.amin(data["data"]["x"]),
end = np.amax(data["data"]["x"]))}
figure_obj.add_layout(LinearAxis(x_range_name = "bar", axis_label = unit), "above")
break
figure_obj.xaxis.axis_label = list_of_datasets[0]["x_unit"]
colour_list = Spectral11 + RdPu9 + Oranges9
colour_indices = [0, 2, 8, 10, 12, 14, 20, 22, 1, 3, 9, 11, 13, 15]
list_of_elements = []
for dataset, color_index in zip(list_of_datasets, colour_indices):
self.all_elements.append(dataset["sample element"]) #strip isotope number
color = colour_list[color_index]
source = ColumnDataSource(data = dataset["data"]) #Datastructure for source of plotting
setattr(self, attr_id+"_"+dataset["sample element"]+"_source", source) #Source element generalized for all plotting
list_of_elements.append(dataset["sample element"])
figure_obj.line("x", "y", source = getattr(self, attr_id+"_"+dataset["sample element"]
+"_source"), line_width = 2, line_color = color,
legend = dataset["sample element"], name = dataset["sample element"],
)
hover = figure_obj.select_one(HoverTool).tooltips = [("element", "@element"), ("(x,y)", "($x, $y)")]
radio_group = RadioGroup(labels = list_of_elements, active=0)
"""
Need to fetch default variables from input file and replace DEFAULT
Block of code produces the layout of buttons and callbacks
"""
#Calculations on the dataset
text_input_rsf = TextInput(value = "default", title = "RSF (at/cm^3): ")
do_integral_button = Button(label = "Calibration Integral")
smoothing_button = Button(label = "Smoothing on selected curve")
text_input_sputter = TextInput(value = "default", title = "Sputter speed: float unit")
text_input_crater_depth = TextInput(value = "default", title = "Depth of crater in: float")
radio_group.on_change("active", lambda attr, old, new: None)
text_input_xval_integral = TextInput(value = "0", title = "x-value for calibration integral ")
text_input_yval_integral = TextInput(value = "0", title = "y-value for calibration integral ")
#Save files for later use
save_flexDPE_button = Button(label = "Save element for FlexPDE")
save_all_flexDPE_button = Button(label = "Save all elements for FlexPDE")
#Pointers to methods on click / change handlers
do_integral_button.on_click(lambda identity = self.attribute_ids[i], radio = radio_group,
x_box = text_input_xval_integral, y_box = text_input_yval_integral:
self.integrate(identity, radio, x_box, y_box))
smoothing_button.on_click(lambda identity = self.attribute_ids[i], radio = radio_group:
self.smoothing(identity, radio) )
save_flexDPE_button.on_click(lambda identity = self.attribute_ids[i], radio = radio_group:
self.write_to_flexPDE(identity, radio))
save_all_flexDPE_button.on_click(lambda identity = self.attribute_ids[i], radio = radio_group:
self.write_all_to_flexPDE(identity, radio))
text_input_rsf.on_change("value", lambda attr, old, new, radio = radio_group,
identity = self.attribute_ids[i], text_input = text_input_rsf, which = "rsf":
self.update_data(identity, radio, text_input, new, which))
text_input_sputter.on_change("value", lambda attr, old, new, radio = radio_group,
identity = self.attribute_ids[i], text_input = text_input_sputter, which = "sputter":
self.update_data(identity, radio, text_input, new, which))
text_input_crater_depth.on_change("value", lambda attr, old, new, radio = radio_group,
identity = self.attribute_ids[i], text_input = text_input_crater_depth, which = "crater_depth":
self.update_data(identity, radio, text_input, new, which))
#Initialization of actual plotting.
tab_plots.append(Panel(child = hplot(figure_obj,
vform(radio_group, save_flexDPE_button, save_all_flexDPE_button),
vform(text_input_rsf, smoothing_button, text_input_sputter, text_input_crater_depth),
vform(text_input_xval_integral, text_input_yval_integral, do_integral_button)),
title = attr_id))
"""
Check to see if one or more element exists in the samples and creat a comparison plot for each
of those elements.
"""
for element in self.all_elements:
checkers = list(self.all_elements)
checkers.remove(element)
if element in checkers and not element in self.elements_comparison:
self.elements_comparison.append(element)
"""create plots for each element that is to be compared """
for comparison_element in self.elements_comparison:
colour_list = Spectral11 + RdPu9 + Oranges9
colour_indices = [0, 2, 8, 10, 12, 14, 20, 22, 1, 3, 9, 11, 13, 15]
figure_obj = figure(plot_width = 1000, plot_height = 800, y_axis_type = "log", title = comparison_element, tools = TOOLS)
#figure_obj.xaxis.major_label_text_font_size("12pt")
#figure_obj.yaxis.major_label_text_font_size("12pt")
y_axis_units = []
x_axis_units = []
comparison_datasets = []
for attr_id, color_index in zip(self.attribute_ids, colour_indices):
list_of_datasets = getattr(self, attr_id)
for dataset in list_of_datasets:
if dataset["sample element"] == comparison_element:
comparison_datasets.append(dataset)
y_axis_units.append(dataset["y_unit"])
x_axis_units.append(dataset["x_unit"])
figure_obj.xaxis.axis_label = comparison_datasets[-1]["x_unit"]
figure_obj.yaxis.axis_label = comparison_datasets[-1]["y_unit"]
if not all(x == y_axis_units[-1] for x in y_axis_units):
for unit, data in zip(y_axis_units, comparison_datasets):
if not unit == y_axis_units[-1]:
figure_obj.extra_y_ranges = {"foo": Range1d(start = np.amin(data["data"]["y"]),
end = np.amax(data["data"]["y"]))}
figure_obj.add_layout(LogAxis(y_range_name = "foo", axis_label = unit), "right")
break
if not all(x == x_axis_units[-1] for x in x_axis_units):
for unit, data in zip(x_axis_units, comparison_datasets):
if not unit == x_axis_units[-1]:
figure_obj.extra_x_ranges = {"bar": Range1d(start = np.amin(data["data"]["x"]),
end = np.amax(data["data"]["x"]))}
figure_obj.add_layout(LinearAxis(x_range_name = "bar", axis_label = unit), "above")
break
for attr_id, color_index in zip(self.attribute_ids, colour_indices):
list_of_datasets = getattr(self, attr_id)
for dataset in list_of_datasets:
if dataset["sample element"] == comparison_element:
color = colour_list[color_index]
"""
Logic that ensures that plots get put with correspoinding axes.
"""
if dataset["x_unit"] != x_axis_units[-1] or dataset["y_unit"] != y_axis_units[-1]:
if dataset["x_unit"] != x_axis_units[-1] and dataset["y_unit"] != y_axis_units[-1]:
figure_obj.line("x", "y", source = getattr(self, attr_id+"_"+dataset["sample element"]+"_source"), line_width = 2,
line_color = color, legend = attr_id, x_range_name = "bar", y_range_name = "foo")
elif dataset["x_unit"] != x_axis_units[-1]:
figure_obj.line("x", "y", source = getattr(self, attr_id+"_"+dataset["sample element"]+"_source"), line_width = 2,
line_color = color, legend = attr_id, x_range_name = "bar")
else:
figure_obj.line("x", "y", source = getattr(self, attr_id+"_"+dataset["sample element"]+"_source"), line_width = 2,
line_color = color, legend = attr_id, y_range_name = "foo")
else:
figure_obj.line("x", "y", source = getattr(self, attr_id+"_"+dataset["sample element"]+"_source"), line_width = 2,
line_color = color, legend = attr_id)
tab_plots.append(Panel(child = figure_obj, title = comparison_element))
tabs = Tabs(tabs = tab_plots)
session = push_session(curdoc())
session.show()
session.loop_until_closed()
tsne_source.on_change('selected', on_tsne_data_update)
# Undo button
undo_selected_points_button = Button(label='Undo last selection')
def on_button_undo_selection():
global previously_selected_spike_indices
tsne_source.data = {'tsne-x': tsne[0], 'tsne-y': tsne[1]}
tsne_source.selected['1d']['indices'] = previously_selected_spike_indices
old = new = tsne_source.selected
tsne_source.trigger('selected', old, new)
undo_selected_points_button.on_click(on_button_undo_selection)
# Layout
lay = column(tsne_figure, undo_selected_points_button)
session = push_session(curdoc())
session.show(lay) # open the document in a browser
session.loop_until_closed()
# ======================================================================================================================
import matplotlib.pyplot as plt
import numpy as np
from scipy.spatial import distance
import os
synonym_suggestions.menu = new_synonyms
processed_dir = "processed/" if len(sys.argv) < 2 else sys.argv[1]
start_year = 2012 if len(sys.argv) < 3 else sys.argv[2]
num_years = 5 if len(sys.argv) < 4 else sys.argv[3]
# Load all the pickle files.
year_data = load_data(processed_dir, start_year, num_years)
# Create the front-end
x_axis = range(start_year, start_year + num_years)
# User Input
text_input = TextInput(value="", title="Graph these comma-separated phrases:")
text_input.on_change("value", create_plot)
# Spell check
spelling_suggestions = Button(label="")
spelling_suggestions.on_click(suggestion_handler)
# Generate synonyms
generate_synonyms = Button(label="Generate synonyms")
generate_synonyms.on_click(synonym_button_handler)
# Dropdown to select replacement synonyms
synonym_suggestions = Dropdown(label="Synonyms", menu=[])
synonym_suggestions.on_change("value", replace_synonym)
# Plot!
p = figure(title="Ngram Viewer",
x_axis_label='Year',
y_axis_label='Frequency',
min_border=75,
plot_width=900,
plot_height=700)
source = ColumnDataSource({'xs': [], 'ys': [], 'labels': [], 'colors': []})
p.multi_line('xs',
fig = make_trial_figure()
# Setup callbacks for tools and sources
trialSourceDict['undefined'].on_change('selected', undefined_selected)
trialSourceDict['saccade'].on_change('selected', saccade_selected)
trialSourceDict['pursuit'].on_change('selected', pursuit_selected)
trialSourceDict['fixation'].on_change('selected', fixation_selected)
###########################################################################
# Add widgets and their callbacks
#widgets = add_widgets()
from bokeh.models.widgets import Button, TextInput
label_saccade_button = Button(label='saccade')
label_saccade_button.on_click(label_saccade_cb)
label_pursuit_button = Button(label='pursuit')
label_pursuit_button.on_click(label_pursuit_cb)
label_fixation_button = Button(label='fixation')
label_fixation_button.on_click(label_fixation_cb)
remove_button = Button(label='remove')
remove_button.on_click(remove_cb)
trial_text = TextInput(value=str(trialNum))
trial_text.on_change('value', trial_text_cb)
nextTrial_button = Button(label='+trial')
nextTrial_button .on_click(next_trial_cb)
def buildPlot():
#####################Setup
# Grab graph colors, pop undesireable ones
colors = SEABORN_PALETTES['bright']
#Grab and sort the FQs
quals = fruit_df.reset_index()
quals = quals['FruitQuality'].unique().tolist()
for idx, i in enumerate(list(quals)):
if type(i) == type(0.5):
quals.pop(idx)
unique_FQs = quals
#a little math to get the epoch time to set the initial x range
minDate = ts_to_epoch(fruit_df['Date'].min())
maxDate = ts_to_epoch(fruit_df['Date'].max())
###########Create and format the plot
plot = figure(
x_axis_type="datetime",
plot_width=600,
plot_height=400,
tools=[PanTool(),
WheelZoomTool(),
SaveTool(),
BoxZoomTool()],
x_range=DataRange1d(
start=minDate, end=maxDate
), #sets the initial date range to the limits of the data
y_range=DataRange1d(start=0, end=1),
name='the_plot',
toolbar_location='above')
#some styling
plot.title.text = "Historical Volatility"
plot.xaxis.axis_label = "Trade Date"
plot.yaxis.axis_label = "Vol"
plot.background_fill_color = '#EAEBF0'
plot.xgrid.grid_line_color = 'white'
plot.ygrid.grid_line_color = 'white'
plot.xaxis.axis_line_color = 'white'
plot.xaxis.major_tick_line_color = 'white'
plot.xaxis.minor_tick_line_color = 'white'
plot.yaxis.axis_line_color = 'white'
plot.yaxis.major_tick_line_color = 'white'
plot.yaxis.minor_tick_line_color = 'white'
plot.toolbar.logo = None
#a list for all of the lines to reside in
lines = []
legends = []
##############Create the widgets
#a console style window to show debug messages TODO: add on/off functionality
debug = PreText(text="", width=1200, height=500)
#echos the debug in a place more visiable for the user
user_message = Paragraph(text='')
#Asset_Class, Product, and From dropdown boxes. Sets dropdown's initial value.
asCls = Select(title="Asset Class",
options=ddOpts['Asset_Class'].unique().tolist())
asCls.value = asCls.options[0]
prod = Select(title="Products",
options=ddOpts[ddOpts['Asset_Class'] == asCls.value]
['Product'].unique().tolist())
prod.value = prod.options[0]
whereFrom = Select(title="From",
options=ddOpts[(ddOpts['Asset_Class'] == asCls.value)
& (ddOpts['Product'] == prod.value)]
['From'].unique().tolist())
whereFrom.value = whereFrom.options[0]
FQslider = Slider(title='Fruit Quality',
start=min(unique_FQs),
end=max(unique_FQs),
step=1)
#the amount of days back to look for the data
days_back = TextInput(title='Days ago', value='365')
days_back_buttons = RadioButtonGroup(
labels=['10', '30', '90', '180', '365', '730'], active=4)
#the date to linear fit to
fixed_date_buttons = RadioButtonGroup(
labels=['30', '60', '90', '120', '180', '365'], active=2)
fixed_date = TextInput(title='Days to Exp', value='90')
#the amount of days with which to calculate the rolling mean
rolling_days_buttons = RadioButtonGroup(labels=['1', '2', '5', '10'],
active=0)
rolling_days = TextInput(title='Rolling Mean Days', value='1')
#a dynamically resizing checkbox group that allows for the changing of the visablity of any line on the plot
line_onOff = CheckboxGroup(width=400, name='line_onOff')
#the associated colors to act as a legend for line_onOff
legendDiv = Div(width=50)
#button to add a line
addLine = Button(label="Add Line")
#an html rendered visualization of the data for each line
descriptions = Div(text='', width=500)
#resizes the plot
rszButton = Button(label='resize')
##########Define functions associated with the widgets
#concats any dubug call to the end of the current debug text, and changes the user message
def updateDebug(inString):
inString = str(inString)
user_message.text = inString
oldText = debug.text
newText = ("*- " + str(datetime.now()) + " : " + inString)
debug.text = oldText + '\n' + newText
#changes the potential products and contract categories to match the user selected asset class
def asClsChange(attrname, old, new):
prod.options = ddOpts[ddOpts['Asset_Class'] ==
asCls.value]['Product'].unique().tolist()
prod.value = prod.options[0]
#changes the potential contract categories to match the user selected product
def prodChange(attrname, old, new):
whereFrom.options = ddOpts[(ddOpts['Asset_Class'] == asCls.value) & (
ddOpts['Product'] == prod.value)]['From'].unique().tolist()
whereFrom.value = whereFrom.options[0]
#links the days back button and text box
def days_back_buttonChange(attrname, old, new):
days_back.value = days_back_buttons.labels[days_back_buttons.active]
#checks that the users input is an int
def days_backChange(attrname, old, new):
try:
days_back.value = str(int(days_back.value))
except ValueError:
days_back.value = '0'
updateDebug('please type an integer')
#links the fixed date button and text box
def fixed_date_buttonChange(attrname, old, new):
fixed_date.value = fixed_date_buttons.labels[fixed_date_buttons.active]
#checks that the users input is an int
def fixed_dateChange(attrname, old, new):
try:
fixed_date.value = str(int(fixed_date.value))
except ValueError:
fixed_date.value = '0'
updateDebug('please type an integer')
#links the rolling days button and text box
def rolling_days_buttonsChange(attrname, old, new):
rolling_days.value = rolling_days_buttons.labels[
rolling_days_buttons.active]
#checks that the users input is an int
def rolling_daysChange(attrname, old, new):
try:
rolling_days.value = str(int(rolling_days.value))
except ValueError:
rolling_days.value = '0'
updateDebug('please type an integer')
#fits the plot to the currently visiable lines
def resize():
if len(line_onOff.active) == 0 or len(line_onOff.labels) == 0:
plot.x_range.start = ts_to_epoch(fruit_df['Date'].min())
plot.x_range.end = ts_to_epoch(fruit_df['Date'].max())
plot.y_range.start = 0
plot.y_range.end = 100
else:
xmin, xmax, ymin, ymax = calc_range(lines)
plot.x_range.start = xmin
plot.x_range.end = xmax
plot.y_range.start = ymin
plot.y_range.end = ymax
#turn lines on or off
def line_onOffChange(attrname, old, new):
for i in range(len(line_onOff.labels)):
if i in line_onOff.active:
lines[i].glyph.visible = True
else:
lines[i].glyph.visible = False
legendDiv.text = '<div>'
for line in lines:
legendDiv.text += '<br><div style="background-color: %s; float:up; padding: 4px 4px 4px 4px"></div><br>' % line.glyph.line_color
legendDiv.text += '</div>'
resize()
#adds a line to the graph
def grphUpdt():
#adds some debug messages, grabs the current time as to later show the total time taken to calculate
updateDebug("Starting")
updateDebug("total dataframe size: " + str(fruit_df.shape))
stTime = datetime.now()
#the value to linear fit to
fit_to = int(fixed_date.value)
#instiantiate an empty dataframe that will eventually contain the graphs data
graphData = pd.DataFrame({
'Date': [],
'PriceVolatility': [],
'Days_to_Exp': []
})
#grab the appropriate subset of the whole dataframe based on the users input into the widgets
updateDebug("querying the data..")
try:
workingDf = fruit_df.loc[asCls.value, prod.value, whereFrom.value]
except KeyError:
updateDebug(
'no data with that combination of Asset Class, Product, From')
return
try:
workingDf = workingDf[[
'Date', 'PriceVolatility', 'Days_to_Exp'
]][(workingDf['Date'] >
(date.today() - timedelta(days=int(days_back.value))))]
except KeyError:
updateDebug(
'no data with that combination of Asset Class, Product, From, and days back'
)
return
updateDebug("done breaking down df")
#a hook in the case that the users inputs resulted in an empty dataframe
if (workingDf.empty):
updateDebug(
'no data with that combination of Asset Class, Product, From, and days back'
)
return
#widdle down the database to only contain the user specified FQ
try:
graphData = workingDf.loc[int(FQslider.value)].copy()
except KeyError:
updateDebug('no data with that FQ')
#another empty graph hook
if (graphData.empty):
updateDebug(
'no data with that combination of Asset Class, Product, Contract Category, FQ, and days back'
)
return
updateDebug('grabed correct FQs')
#calculate linear fit on the current subset
updateDebug('calculating linear fit...')
graphData = mu.linearFit(fit_to=fit_to,
group_on_column='Date',
df=graphData,
fit_column='Days_to_Exp',
on_columns=['PriceVolatility'])
updateDebug('finished with linear fit')
# a few more debug messages
updateDebug(
"working df qry: Asset_Class = %s and Product = %s and From = %s and Date > %s "
% (asCls.value, prod.value, whereFrom.value,
str(date.today() - timedelta(days=int(days_back.value)))))
updateDebug("graph data shape: " + str(workingDf.shape))
#makes sure graph data has at least 5 rows, so that rolling mean can be calculated
if graphData.shape[0] > int(rolling_days.value):
#make the graph legend, based on if there's a denominator specified or not
this_legend = '%s - %s FQ: %s Days to Exp: %s From: %s Rolling Days: %s' % (
prod.value, whereFrom.value, int(
FQslider.value), fixed_date.value,
str(date.today() - timedelta(days=int(days_back.value))),
rolling_days.value)
#add a new line to the graph, and add the accosiated GlyphRenderer created by adding the line to the lines list.
#Set the legend to the previously calculated legend, and set the color to the next color in the current theme (if there are more lines than colors, there will be multiple lines with the same color)
#Calculates a 5 day rolling mean on the y values. Maybe add a slider/text box/other widget so the user can set the rolling mean themselves
updateDebug('adding line to plot')
lines.append(
plot.line(graphData.index.values[int(rolling_days.value) - 1:],
graphData['PriceVolatility'].rolling(
window=int(rolling_days.value)).mean()
[int(rolling_days.value) - 1:],
line_width=3,
color=colors[len(lines) % len(colors)]))
legends.append(this_legend)
updateDebug("updated graph")
global descDf
#either creates, or adds to, a dataframe containing statistics about the data. stats come from pandas DataFrame.describe.
if descDf is None:
graphData[this_legend] = graphData['PriceVolatility']
descDf = graphData[[
this_legend
]].rolling(window=int(rolling_days.value)).mean(
)[int(rolling_days.value) -
1:].describe(percentiles=[]).transpose().copy()
else:
graphData[this_legend] = graphData['PriceVolatility']
descDf = pd.concat([
descDf, graphData[[
this_legend
]].rolling(window=int(rolling_days.value)).mean()
[int(rolling_days.value) -
1:].describe(percentiles=[]).transpose().copy()
])
descDf = descDf.round(1)
descriptions.text = descDf.to_html().replace('\\n', '')
graphData.drop(this_legend, 1, inplace=True)
#add the name of the line to the checkbox so that it can be turned off and o
line_onOff.labels.append(this_legend)
line_onOff.active.append(len(line_onOff.labels) - 1)
legendDiv.text = '<div>'
for line in lines:
legendDiv.text += '<br><div style="background-color: %s; float:up; padding: 4px 4px 4px 4px"></div><br>' % line.glyph.line_color
legendDiv.text += '</div>'
##leaving this in case we get around to figuring out the hover tool
##formats the date values for the hover tool, currently commented out until we, or bokeh, fix the hover tool for multiple lines
#formDates= pd.to_datetime(graphData['Date'] ,format="%m-%d-%Y")
#lines[-1].data_source.data['formDates'] = formDates.apply(lambda x: x.strftime('%m-%d-%Y'))
##Displays the amout of time it took to draw the line, as well as the number of points in the graph
updateDebug("updated y vals, with rolling mean calculated")
updateDebug(
str(datetime.now() - stTime) + " FOR " +
str(len(lines[-1].data_source.data['x'])) + " points")
else:
updateDebug("There's no data to display")
del graphData
del workingDf
#######Link widgets to their associated functions
asCls.on_change('value', asClsChange)
prod.on_change('value', prodChange)
days_back_buttons.on_change('active', days_back_buttonChange)
days_back.on_change('value', days_backChange)
fixed_date_buttons.on_change('active', fixed_date_buttonChange)
fixed_date.on_change('value', fixed_dateChange)
rolling_days_buttons.on_change('active', rolling_days_buttonsChange)
rolling_days.on_change('value', rolling_daysChange)
line_onOff.on_change('active', line_onOffChange)
addLine.on_click(grphUpdt)
rszButton.on_click(resize)
#Formatting
fixed_date_box = WidgetBox(fixed_date, fixed_date_buttons)
days_back_box = WidgetBox(days_back, days_back_buttons)
rolling_days_box = WidgetBox(rolling_days, rolling_days_buttons)
widgets = [
asCls, prod, whereFrom, FQslider, days_back_box, fixed_date_box,
rolling_days_box, addLine, rszButton, user_message
]
plot_w_description = VBox(plot, descriptions, width=700)
pwd_w_leg = HBox(plot_w_description,
VBox(legendDiv),
VBox(line_onOff),
width=plot_w_description.width + line_onOff.width + 100,
name='div_to_save')
input_box = VBox(*widgets, width=400, height=1200)
total_box = HBox(VBox(input_box),
VBox(pwd_w_leg),
width=input_box.width + pwd_w_leg.width + 100,
height=1200)
tot_w_debug = VBox(total_box, VBox(HBox(debug)))
resize()
return tot_w_debug