def generate_simulated_drawdown_table(self):
pp = [50, 30, 20, 10, 5, 1]
ddp = np.percentile(self.drawdowns, pp)
pp_formatted = [float(p) / 100. for p in pp]
source = ColumnDataSource(
dict(probabilities=pp_formatted, drawdowns=ddp))
probability_formatter = NumberFormatter(format='0.00')
drawdown_formatter = NumberFormatter(format='(0.00)%')
columns = [
TableColumn(field="probabilities",
title="Probabilities",
width=70,
formatter=probability_formatter),
TableColumn(field="drawdowns",
title="Drawdowns",
width=70,
formatter=drawdown_formatter)
]
# display(dd_tbl)
data_table = DataTable(source=source,
columns=columns,
height=220,
row_headers=False)
return data_table
def _get_columns(self):
if self.value is None:
return []
indexes = self.indexes
col_names = list(self.value.columns)
if not self.hierarchical or len(indexes) == 1:
col_names = indexes + col_names
else:
col_names = indexes[-1:] + col_names
df = self.value.reset_index() if len(indexes) > 1 else self.value
columns = []
for col in col_names:
if col in df.columns:
data = df[col]
else:
data = df.index
col_kwargs = {}
kind = data.dtype.kind
if kind == 'i':
formatter = NumberFormatter()
editor = IntEditor()
elif kind == 'f':
formatter = NumberFormatter(format='0,0.0[00000]')
editor = NumberEditor()
elif isdatetime(data) or kind == 'M':
formatter = DateFormatter(format='%Y-%m-%d %H:%M:%S')
editor = DateEditor()
else:
formatter = StringFormatter()
editor = StringEditor()
if col in self.editors:
editor = self.editors[col]
if col in indexes or editor is None:
editor = CellEditor()
if col in self.formatters:
formatter = self.formatters[col]
if str(col) != col:
self._renamed_cols[str(col)] = col
if isinstance(self.widths, int):
col_kwargs['width'] = self.widths
elif str(col) in self.widths:
col_kwargs['width'] = self.widths.get(str(col))
title = str(col)
if col in indexes and len(indexes) > 1 and self.hierarchical:
title = 'Index: %s' % ' | '.join(indexes)
column = TableColumn(field=str(col),
title=title,
editor=editor,
formatter=formatter,
**col_kwargs)
columns.append(column)
return columns
def get_default_formatters(
data: pd.DataFrame,
int_format: str = INT_FORMAT,
int_align: str = INT_ALIGN,
float_format: str = FLOAT_FORMAT,
float_align: str = FLOAT_ALIGN,
) -> Dict:
"""A dictionary of columns and formats for the the `DataFrame` Panel widget.
#### Example Use Case
```python
formatters = get_default_formatters(data)
dataframe_widget = pn.widgets.DataFrame(data, formatters=formatters))
```
### Notes
- if the index is 1-dimensional and the name is None we rename to 'index' in order to be able
to format the index.
- For the complete formattings specification see [numbrojs](http://numbrojs.com/format.html)
Args:
data (pd.DataFrame): A DataFrame of data
int_format (str, optional): The int format string. Defaults to INT_FORMAT.
int_align (str, optional): [description]. Defaults to INT_ALIGN.
float_format (str, optional): [description]. Defaults to FLOAT_FORMAT.
float_align (str, optional): [description]. Defaults to FLOAT_ALIGN.
Returns:
Dict[str,str] -- A dictionary of default formats for the columns
"""
formatters = {}
for column in data.columns:
if data[column].dtype == INT_DTYPE:
formatters[column] = NumberFormatter(format=int_format, text_align=int_align,)
elif data[column].dtype == FLOAT_DTYPE:
formatters[column] = NumberFormatter(format=float_format, text_align=float_align,)
# Hack: Without a proper name we cannot format the index
if len(data.index.names) == 1 and not data.index.name:
data.index.name = "index"
for (index, name,) in enumerate(data.index.names):
if name:
dtype = data.index.get_level_values(index).dtype
if dtype == INT_DTYPE:
formatters[name] = NumberFormatter(format=int_format, text_align=int_align,)
elif dtype == FLOAT_DTYPE:
formatters[name] = NumberFormatter(format=float_format, text_align=float_align,)
return formatters
def panel(self):
result = bootstrap
price_slider = pn.widgets.RangeSlider.from_param(
self.param.price_slider, step=10000, format='0.0a')
result.sidebar.append(price_slider)
result.sidebar.append(self.param.rooms_slider)
result.sidebar.append(self.param.bathrooms_slider)
result.sidebar.append(self.param.type)
result.sidebar.append(self.param.transit_time)
image_format = r'<div> <img src="<%= value %>" height="70" alt="<%= value %>" width="70" style="float: left; margin: 0px 15px 15px 0px;" border="2" ></img> </div>'
tabulator_formatters = {
'price': NumberFormatter(format='$0,0'),
'size': NumberFormatter(format='0,0 sqft'),
'photo': HTMLTemplateFormatter(template=image_format)
}
df_widget = pn.widgets.Tabulator(self.filter_df(),
pagination='remote',
page_size=10,
formatters=tabulator_formatters,
sizing_mode='scale_both')
df_widget.add_filter(self.param.price_slider, 'price')
df_widget.add_filter(self.param.rooms_slider, 'bedrooms')
# df_pins = pn.widgets.Tabulator(self.distance_df(), pagination='remote', page_size=10, sizing_mode='scale_both')
layout = pn.Row(
pn.Card(pn.bind(self.location,
x=self.stream.param.x,
y=self.stream.param.y),
title="Map",
sizing_mode='stretch_height'),
pn.Column(
pn.Card(df_widget,
title="Properties",
sizing_mode='scale_both')))
result.sidebar.append(
pn.Card(pn.bind(self.distance_df,
x=self.stream.param.x,
y=self.stream.param.y),
title="Pins",
sizing_mode='scale_both'))
bootstrap.main.append(layout)
bootstrap.main.append(pn.Card(self.details_area, title='Details'))
return result
def generate_stationarity_table(self):
# Test for non-stationarity
adf_res = adfuller(self.daily_returns)
# Output tables of stats
raw_data = dict(
test_stat=[adf_res[0]],
p_value=[adf_res[1]],
conclusion=[
'non-stationarity = {} @ 5%'.format(
'REJECTED' if adf_res[1] < 0.05 else 'NOT REJECTED')
])
source = ColumnDataSource(raw_data)
number_formatter = NumberFormatter(format='-0.0000')
columns = [
TableColumn(field="test_stat",
title="ADF Test Statistic",
width=150),
TableColumn(field="p_value",
title="p-value",
width=80,
formatter=number_formatter),
TableColumn(field="conclusion", title="Conclusion", width=400),
]
data_table = DataTable(source=source,
columns=columns,
width=650,
height=100,
row_headers=False)
return data_table
def test_dataframe_formatter(dataframe, document, comm):
formatter = NumberFormatter(format='0.0000')
table = DataFrame(dataframe, formatters={'float': formatter})
model = table.get_root(document, comm)
model_formatter = model.columns[2].formatter
assert model_formatter is not formatter
assert isinstance(model_formatter, NumberFormatter)
assert model_formatter.format == formatter.format
def table_elements(data, columns):
"""Table describing the contribution of each element"""
formatters = {
"weights": NumberFormatter(format="0 %"),
"element_net_gross": NumberFormatter(format="0 %"),
"contribution": NumberFormatter(format="0.0"),
}
return pn.widgets.Tabulator(
data.assign(
weights=lambda df: df.loc[:, "weights"] / 100,
element_net_gross=lambda df: df.loc[:, "element_net_gross"] / 100,
),
disabled=True,
titles=columns.as_dict(),
layout="fit_data_fill",
formatters=formatters,
sizing_mode="stretch_width",
)
def error_summary_stats_table(self):
from bokeh.models.widgets.tables import NumberFormatter
data = self.filter_data_rows()
stats_df = data.describe(
percentiles=[0.025, 0.2, 0.3, 0.5, 0.7, 0.8, 0.975]).T
stats_df = stats_df.round(3)
formatter = NumberFormatter(format='0.000')
return pn.pane.DataFrame(stats_df,
width=1350,
formatters={'float': formatter},
widths={'index': 300})
def generate_summary_table(self):
# Output tables of stats
rows = ['Input data'] + self.fitted_stats.keys()
means = [np.mean(self.daily_returns)
] + [float(s[0]) for s in self.fitted_stats.values()]
variances = [np.var(self.daily_returns)
] + [float(s[1]) for s in self.fitted_stats.values()]
skews = [skew(self.daily_returns)
] + [float(s[2]) for s in self.fitted_stats.values()]
kurtosises = [kurtosis(self.daily_returns)
] + [float(s[3]) for s in self.fitted_stats.values()]
raw_data = dict(rows=rows,
means=means,
variances=variances,
skews=skews,
kurtosises=kurtosises)
source = ColumnDataSource(raw_data)
number_formatter = NumberFormatter(format='-0.0000')
columns = [
TableColumn(field="rows", title="", width=200),
TableColumn(field="means",
title="Mean",
width=80,
formatter=number_formatter),
TableColumn(field="variances",
title="Variance",
width=80,
formatter=number_formatter),
TableColumn(field="skews",
title="Skewness",
width=80,
formatter=number_formatter),
TableColumn(field="kurtosises",
title="Kurtosis",
width=80,
formatter=number_formatter)
]
data_table = DataTable(source=source,
columns=columns,
width=550,
height=150,
row_headers=False)
return data_table
def _data(self, ):
# data is simple example. In the real world in would depend on among other self.frequency
data = pd.DataFrame(
{"int": [
1,
2,
3000,
]},
index=[
1,
2,
3,
],
)
formatters = {
"int": NumberFormatter(
format="0.0",
text_align="right",
)
}
return pn.widgets.DataFrame(
data,
formatters=formatters,
)
def __init__(self):
# app variables
self.active = True
self.playAnimation = None
self.start_epoch = None
self.stop_epoch = None
self.current_epoch = None
# get initial configuration
self.available_models = inspect.getmembers(StandardEphemerisModels,
inspect.isfunction)
self.ephemeris_model = self.available_models[0][1]()
self.spice_provider = SpiceProvider()
self.spice_provider.SPICE_IDS = self.ephemeris_model.objects
self.spice_provider.SPICE_NAMES = {
v: k
for k, v in self.ephemeris_model.objects.items()
}
# init data sources
self.plot_source = self.spice_provider.state_source
self.table_source = self.spice_provider.ephemeris_source
self.cum_source = self.spice_provider.cum_source
# gather options from ephemeris model and spice provider
self.allowed_models = {
model[1]().name: model[1]
for model in self.available_models
}
allowed_objects = [
self.spice_provider.fromId(name)
for name in self.ephemeris_model.objects
]
allowed_frames = self.ephemeris_model.FRAMES
allowed_corrections = [name for name in SpiceProvider.CORRECTIONS]
allowed_durations = [
str(v) for v in self.ephemeris_model.DURATION_DAYS
]
allowed_intervals = [name for name in SpiceProvider.INTERVALS]
# set up widgets
self.model = Select(title="Ephemeris Model",
options=list(self.allowed_models.keys()))
self.center = Select(title="Center",
value=self.ephemeris_model.center,
options=allowed_objects)
self.target = Select(title="Target",
value=self.ephemeris_model.target,
options=allowed_objects)
self.frames = Select(title="Frame",
value=self.ephemeris_model.frame,
options=allowed_frames)
self.planes = RadioButtonGroup(labels=['XY', 'YZ', 'XZ'], active=0)
self.vector = Select(title='Vector Type',
value=self.ephemeris_model.vector_type,
options=allowed_corrections)
self.epoch = DatePicker(title="Select Epoch",
value=datetime.strftime(
self.ephemeris_model.epoch, "%Y-%m-%d"))
self.offset = Slider(title="Days Since Epoch",
value=self.ephemeris_model.offset,
start=0,
end=self.ephemeris_model.duration,
step=1)
self.duration = Select(title="Duration (Days)",
value=str(self.ephemeris_model.duration),
options=allowed_durations)
self.interval = Select(title="Time Step",
value=str(self.ephemeris_model.step_size),
options=allowed_intervals)
# create buttons
self.play_button = Button(label="Play")
self.exportRange = Div(text="Start and Stop Epoch: ")
self.update_button = Button(label="Play")
self.export_button = Button(label="Export")
self.infoDiv = Div(
text=
"<hr>All ephemeris data shown on this website was obtained from publicly available "
"SPICE files located at <a href='https://naif.jpl.nasa.gov/naif/data.html'>"
"https://naif.jpl.nasa.gov/naif/data.html</a>, which is hosted by the "
"Navigation and Ancillary Information Facility (NAIF) at the NASA Jet Propulsion "
"Laboratory. The exception is the SPICE kernel for the Parker Solar Probe, which is "
"available at <a href='https://sppgway.jhuapl.edu/ancil_products'>"
"https://sppgway.jhuapl.edu/ancil_products</a>, hosted by the Johns Hopkins University "
"Applied Physics Laboratory. SpiceyPy is being used to process the SPICE files.",
sizing_mode='stretch_width')
# create plot tab objects
self.plot = figure(match_aspect=True,
sizing_mode="stretch_both",
title="Astropynamics",
tools="hover, pan, reset, save",
tooltips=[("name", "@index")])
self.plot.add_tools(BoxZoomTool(match_aspect=True))
self.plot.circle('px',
'py',
size='radii',
source=self.plot_source,
line_width=3,
line_alpha=0.5,
name='XY')
self.plot.circle('px',
'pz',
size='radii',
source=self.plot_source,
line_width=3,
line_alpha=0.5,
name='XZ').visible = False
self.plot.circle('py',
'pz',
size='radii',
source=self.plot_source,
line_width=3,
line_alpha=0.5,
name='YZ').visible = False
self.plot.line('px',
'py',
source=self.cum_source,
line_width=2,
line_alpha=0.5,
color='red',
name='XYOrbit')
self.plot.line('px',
'pz',
source=self.cum_source,
line_width=2,
line_alpha=0.5,
color='red',
name='XZOrbit').visible = False
self.plot.line('py',
'pz',
source=self.cum_source,
line_width=2,
line_alpha=0.5,
color='red',
name='YZOrbit').visible = False
self.plotLayout = column(self.plot,
self.offset,
sizing_mode="stretch_width")
self.plotTab = Panel(child=self.plotLayout, title="Display")
# create data table tab objects
fmt = NumberFormatter(format='0.000', text_align=TextAlign.right)
columns = [
TableColumn(field="index",
title="Epoch",
formatter=DateFormatter(format="%m/%d/%Y %H:%M:%S")),
TableColumn(field="px", title="PX", formatter=fmt, width=10),
TableColumn(field="py", title="PY", formatter=fmt),
TableColumn(field="pz", title="PZ", formatter=fmt),
TableColumn(field="vx", title="VX", formatter=fmt),
TableColumn(field="vy", title="VY", formatter=fmt),
TableColumn(field="vz", title="VZ", formatter=fmt)
]
self.ephemerisTable = DataTable(source=self.table_source,
columns=columns,
sizing_mode="stretch_both")
self.ephemerisLayout = column(self.exportRange,
self.ephemerisTable,
sizing_mode="stretch_width")
self.dataTab = Panel(child=self.ephemerisLayout, title="Table")
self.kernels = Div()
self.kernelTab = Panel(child=self.kernels, title="Kernels")
self.tabs = Tabs(tabs=[self.plotTab, self.dataTab, self.kernelTab])
# init data
self.model.value = "The Solar System"
self.update_model(None, 0, self.model.value)
self.update_epochs(None, 0, 0)
self.update_states(None, 0, 0)
self.model.on_change('value', self.update_model)
self.frames.on_change('value', self.update_epochs)
self.planes.on_change('active', self.update_plot_view)
self.center.on_change('value', self.update_epochs)
self.target.on_change('value', self.update_epochs)
self.offset.on_change('value', self.update_offset)
self.epoch.on_change('value', self.update_epochs)
self.duration.on_change('value', self.update_epochs)
self.interval.on_change('value', self.update_epochs)
self.update_button.on_click(self.update_onclick)
self.tabs.on_change('active', self.update_button_type)
self.inputs = column(self.model, self.frames, self.planes, self.center,
self.target, self.epoch, self.duration,
self.interval, self.update_button)
def test_dataframe_formatter(dataframe, document, comm):
formatter = NumberFormatter(format='0.0000')
table = DataFrame(dataframe, formatters={'float': formatter})
model = table.get_root(document, comm)
assert model.columns[2].formatter is formatter
def data_viewer(dataset, tables, columns_per_table, **widget_args):
"""Show all data in a downloadable table"""
formatters = {
"ng_vsh40_pct": NumberFormatter(format="0 %"),
"thickness_mtvd": NumberFormatter(format="0.0"),
"top_depth_mtvd": NumberFormatter(format="0.0"),
"base_depth_mtvd": NumberFormatter(format="0.0"),
}
def get_filename(table):
return f"{dataset}-{table}.xlsx"
def get_data(table_name):
columns = columns_per_table.get(table_name, {})
return (readers.read_all(
config.app.apps.reader, dataset,
table_name).loc[:, columns.keys()].sort_values(
by="ng_vsh40_pct", ascending=False).reset_index(drop=True))
table_name = pn.widgets.Select(
name="Choose table",
options={t.title(): t
for t in tables},
value=tables[0],
size=1,
)
filename = pn.widgets.TextInput(name="File name",
value=get_filename(table_name.value))
@pn.depends(table_name.param.value)
def table(table_name):
data = get_data(table_name).assign(
ng_vsh40_pct=lambda d: d.ng_vsh40_pct / 100 # Show N:G as percent
)
return pn.widgets.Tabulator(
data,
disabled=True,
titles=columns_per_table[table_name],
formatters=formatters,
layout="fit_data_table",
**widget_args,
)
@pn.depends(table_name.param.value, watch=True)
def update_filename(table_name):
filename.value = get_filename(table_name)
def download_file():
"""Download data as an Excel file"""
output_bytes = io.BytesIO()
# Write data as Excel
excel_writer = pd.ExcelWriter(output_bytes, engine="xlsxwriter")
get_data(table_name.value).to_excel(
excel_writer, sheet_name=table_name.value.title(), index=False)
excel_writer.save()
# Reset output stream and return it
output_bytes.seek(0)
return output_bytes
@pn.depends(filename.param.value)
def download_button(filename):
return pn.widgets.FileDownload(callback=download_file,
filename=filename,
button_type="success")
return pn.Row(
pn.Column(
table_name,
filename,
download_button,
),
pn.Column(table, sizing_mode="stretch_width"),
)
def __init__(self, model):
"""
Construct separation dashboard
"""
# Save reference to model
self.model = model
################################
# Process button
################################
self.process = Button(label="Generate",
button_type="primary",
name='process',
sizing_mode='scale_width',
css_classes=['generate'])
self.process.js_on_click(CustomJS(code="toggleLoading()"))
################################
# Widgets
################################
# Data type selection
self.data_type = RadioButtonGroup(
labels=["All Data", "Experimental", "Simulated"],
active=0,
css_classes=['dtypes'])
# Adsorbate drop-down selections
self.g1_sel = Select(title="Adsorbate 1",
options=self.model.ads_list,
value=self.model.g1,
css_classes=['g-selectors'])
self.g2_sel = Select(title="Adsorbate 2",
options=self.model.ads_list,
value=self.model.g2)
# Temperature selection
self.t_absolute = Spinner(value=self.model.t_abs,
title='Temperature:',
css_classes=['t-abs'])
self.t_tolerance = Spinner(value=self.model.t_tol,
title='Tolerance:',
css_classes=['t-tol'])
# Combined in a layout
self.dsel_widgets = layout([
[self.data_type],
[self.g1_sel, self.g2_sel, self.t_absolute, self.t_tolerance],
],
sizing_mode='scale_width',
name="widgets")
################################
# KPI Plots
################################
# Top graph generation
tooltip = load_tooltip()
self.p_henry, rend1 = self.top_graph("K", "Henry coefficient (log)",
self.model.data,
self.model.errors, tooltip)
self.p_loading, rend2 = self.top_graph(
"L", "Uptake at selected pressure (bar)", self.model.data,
self.model.errors, tooltip)
self.p_wc, rend3 = self.top_graph(
"W", "Working capacity in selected range (bar)", self.model.data,
self.model.errors, tooltip)
# Give graphs the same hover and select effect
sel = Circle(fill_alpha=1, fill_color="red", line_color=None)
nonsel = Circle(fill_alpha=0.2, fill_color="blue", line_color=None)
for rend in [rend1, rend2, rend3]:
rend.selection_glyph = sel
rend.nonselection_glyph = nonsel
rend.hover_glyph = sel
# Pressure slider
self.p_slider = Slider(
title="Pressure (bar)",
value=0.5,
start=0,
end=20,
step=0.5,
callback_policy='throttle',
callback_throttle=200,
)
# Working capacity slider
self.wc_slider = RangeSlider(
title="Working capacity (bar)",
value=(0.5, 5),
start=0,
end=20,
step=0.5,
callback_policy='throttle',
callback_throttle=200,
)
# Material datatable
self.mat_list = DataTable(
columns=[
TableColumn(field="labels", title="Material", width=300),
TableColumn(field="sel",
title="KH2/KH1",
width=35,
formatter=NumberFormatter(format='‘0.0a’')),
TableColumn(field="psa_W",
title="PSA-API",
width=35,
formatter=NumberFormatter(format='‘0.0a’')),
],
source=self.model.data,
index_position=None,
selectable='checkbox',
scroll_to_selection=True,
width=400,
fit_columns=True,
)
# Custom css classes for interactors
self.p_henry.css_classes = ['g-henry']
self.p_loading.css_classes = ['g-load']
self.p_wc.css_classes = ['g-wcap']
self.mat_list.css_classes = ['t-details']
# Generate the axis labels
self.top_graph_labels()
self.kpi_plots = layout([
[
gridplot([[self.mat_list, self.p_henry],
[self.p_loading, self.p_wc]],
sizing_mode='scale_width')
],
[self.p_slider, self.wc_slider],
],
sizing_mode='scale_width',
name="kpiplots")
self.kpi_plots.children[0].css_classes = ['kpi']
self.kpi_plots.children[1].css_classes = ['p-selectors']
################################
# Isotherm details explorer
################################
# Isotherm display graphs
self.p_g1iso = self.bottom_graph(self.model.g1_iso_sel, self.model.g1)
self.p_g2iso = self.bottom_graph(self.model.g2_iso_sel, self.model.g2)
# Isotherm display palette
self.c_cyc = cycle(gen_palette(20))
self.detail_plots = layout([
[self.p_g1iso, self.p_g2iso],
],
sizing_mode='scale_width',
name="detailplots")
self.detail_plots.children[0].css_classes = ['isotherms']
df = pd.DataFrame(
{
"int": [1, 2, 3],
"float": [3.14, 6.28, 9.42],
"str": ["A", "B", "C"],
"bool": [True, False, True],
"date":
[dt.date(2019, 1, 1),
dt.date(2020, 1, 1),
dt.date(2020, 1, 10)],
},
index=[1, 2, 3],
)
bokeh_formatters = {
"float": NumberFormatter(format="0.00"),
"bool": BooleanFormatter(),
}
theme = TABULATOR_THEME.get(template.theme, "site")
df_widget = pn.widgets.Tabulator(df,
formatters=bokeh_formatters,
theme=theme,
sizing_mode="stretch_both")
df_widget._configuration["layout"] = "fitDataFill"
df_widget._configuration["responsiveLayout"] = "collapse"
df_widget._configuration["columns"] = [
{
"int": "Name",
"field": "name",
"responsive": 0
def _get_column_definitions(self, col_names, df):
import pandas as pd
indexes = self.indexes
columns = []
for col in col_names:
if col in df.columns:
data = df[col]
else:
data = df.index
if isinstance(data, pd.DataFrame):
raise ValueError("DataFrame contains duplicate column names.")
col_kwargs = {}
kind = data.dtype.kind
if kind == 'i':
formatter = NumberFormatter()
editor = IntEditor()
elif kind == 'b':
formatter = StringFormatter()
editor = CheckboxEditor()
elif kind == 'f':
formatter = NumberFormatter(format='0,0.0[00000]')
editor = NumberEditor()
elif isdatetime(data) or kind == 'M':
if len(data) and isinstance(data.values[0], dt.date):
date_format = '%Y-%m-%d'
else:
date_format = '%Y-%m-%d %H:%M:%S'
formatter = DateFormatter(format=date_format)
editor = DateEditor()
else:
formatter = StringFormatter()
editor = StringEditor()
if col in self.editors and not isinstance(self.editors[col],
(dict, str)):
editor = self.editors[col]
if col in indexes or editor is None:
editor = CellEditor()
if col in self.formatters and not isinstance(
self.formatters[col], (dict, str)):
formatter = self.formatters[col]
if str(col) != col:
self._renamed_cols[str(col)] = col
if isinstance(self.widths, int):
col_kwargs['width'] = self.widths
elif str(col) in self.widths:
col_kwargs['width'] = self.widths.get(str(col))
title = self.titles.get(col, str(col))
if col in indexes and len(indexes) > 1 and self.hierarchical:
title = 'Index: %s' % ' | '.join(indexes)
column = TableColumn(field=str(col),
title=title,
editor=editor,
formatter=formatter,
**col_kwargs)
columns.append(column)
return columns
def create_ui(self, data):
self.logger.info("number of data items %d", len(data))
# Create data source and data table
# path, score, software_id, featcnt, featfreq, app name, app path, decision, status, comment, active in play, still voilating
decision_editor = SelectEditor(options=[
"Unprocessed", "GPL Violation", "LGPL Violation",
"Open Source App", "False Positive", "False Negative (LGPL)",
"False Negative (GPL)"
])
status_editor = SelectEditor(options=[
"Unprocessed", "Emailed", "Confirmed", "Denied", "Authorized"
])
if self.app_info:
columns = [
TableColumn(field="myindex", title="Id"),
TableColumn(field="path", title="File Path"),
TableColumn(field="score", title="Score"),
TableColumn(field="normscore",
title="NormScore",
formatter=NumberFormatter(format="0.00")),
TableColumn(field="partial", title="PartialMatch"),
TableColumn(field="repo_id", title="Repo ID"),
TableColumn(field="software_name", title="OSS"),
TableColumn(field="version", title="Version"),
TableColumn(
field="featcnt",
title="FeatCount",
),
TableColumn(
field="featfreq",
title="FeatFreq",
),
TableColumn(field="package_name", title="Package"),
TableColumn(field="app_path", title="App Path"),
TableColumn(field="app_count", title="App Count"),
TableColumn(field="decision",
title="Decision",
editor=decision_editor),
TableColumn(field="status",
title="Status",
editor=status_editor),
TableColumn(field="comment", title="Comment"),
# I am not sure whether we should add these two fields here.
# TableColumn(field="active", title="Active in Play"),
# TableColumn(field="still_violating", title="Still Violating"),
]
else:
template_str = '<a href="' + self.REPO_URL + '/<%= value %>"><%= value %></a>'
columns = [
TableColumn(
field="myindex",
title="Id",
),
TableColumn(field="name", title="Name"),
TableColumn(field="score",
title="Score",
formatter=NumberFormatter(format="0.00")),
TableColumn(field="normscore",
title="NormScore",
formatter=NumberFormatter(format="0.00")),
TableColumn(field="partial", title="PartialMatch"),
TableColumn(field="repo_id", title="RepoID"),
TableColumn(
field="software_name",
title="OSS",
formatter=HTMLTemplateFormatter(template=template_str)),
TableColumn(field="featcnt",
title="FeatCount",
formatter=NumberFormatter(format="0,000,000")),
TableColumn(field="featfreq",
title="FeatFreq",
formatter=NumberFormatter(format="0,000,000")),
TableColumn(field="version", title="Version"),
TableColumn(field="decision",
title="Decision",
editor=decision_editor),
TableColumn(field="status",
title="Status",
editor=status_editor),
TableColumn(field="comment", title="Comment"),
TableColumn(field="path", title="Path"),
]
# source is the displayed table, and can be modified by user
# original_source is the original data, it is the base, and can only be modified by the program
self.source = ColumnDataSource(self._data)
self.original_source = ColumnDataSource(self._data)
self.data_table = DataTable(source=self.source,
columns=columns,
width=2000,
height=2000,
editable=True,
sortable=True) # Disable sortable for now!
# selector or filters
# reference link for callback: https://gist.github.com/dennisobrien/450d7da20daaba6d39d0
min_matching_score_slider = Slider(start=0,
end=2,
value=0.3,
step=.01,
title="Minimum Matching Score")
max_matching_score_slider = Slider(start=0,
end=2,
value=0.7,
step=.01,
title="Maximum Matching Score")
featfreq_slider = Slider(start=0,
end=10000,
value=0,
step=1,
title="Minimum Matching Num of Features")
featcnt_slider = Slider(start=0,
end=10000,
value=50,
step=1,
title="Minimum Feature Count is OSS")
kind_select = Select(value="All", options=["All", "Java", "Native"])
file_select = Select(value="Name", options=["Name", "MD5", "Path"])
search_input = TextInput(value=None,
title="Enter library to search",
callback=None)
search_button = Button(label="Search", button_type="success")
download_callback_code = """
var data = source.get('data');
var filetext = 'Id,File Name,Matching Score,Normalized Matching Score,Repo ID,Software Name,Feature Count,Feature Freq.,Version,Decision,Status,Comment,File Path\\n';
var order = ['myindex', 'name', 'score', 'normscore', 'repo_id', 'software_name', 'featcnt', 'featfreq', 'version', 'decision', 'status', 'comment', 'path'];
for (var i = 0; i < data['path'].length; ++i) {
var currRow = [];
for (var item in order) {
key = order[item]
currRow.push(data[key][i]);
}
var joined = currRow.join().concat('\\n');
filetext = filetext.concat(joined);
}
var filename = 'violations.csv';
var blob = new Blob([filetext], { type: 'text/csv;charset=utf-8;' });
//addresses IE
if (navigator.msSaveBlob) {
//navigator.msSaveBlob(blob, filename);
}
else {
var link = document.createElement("a");
link = document.createElement('a');
link.href = URL.createObjectURL(blob);
link.download = filename;
link.target = "_blank";
link.style.visibility = 'hidden';
link.dispatchEvent(new MouseEvent('click'));
}
"""
# enable downloading of results as a csv file
download_button = Button(label="Download", button_type="success")
download_button.callback = CustomJS(args=dict(source=self.source),
code=download_callback_code)
# enable comparison of selected rows
compare_button = Button(label="Compare", button_type="success")
compare_button.on_click(self.compare_callback)
# update on change
#controls = [min_matching_score_slider, max_matching_score_slider, featfreq_slider, \
# featcnt_slider, kind_select, file_select, button]
#for item in controls:
# item.on_change('value', lambda attr, old, new: self.update_source(item))
combined_callback_code = """
var data = source.get('data');
var original_data = original_source.get('data');
var min_score = min_matching_score_slider.get('value');
var max_score = max_matching_score_slider.get('value');
var search_input = search_input.get('value');
var min_featfreq = featfreq_slider.get('value');
var min_featcnt = featcnt_slider.get('value');
var kind = kind_select.get('value');
console.log("min score: " + min_score + ", max score: " + max_score + ", min_featfreq: " + min_featfreq + ", min_featcnt" + min_featcnt + ", kind" + kind);
var java_suffix = ".dex";
var native_suffix = ".so";
console.log("searchinput: " + search_input);
var re;
if (search_input) {
re = new RegExp(search_input);
} else {
re = new RegExp(".*");
}
for (var key in original_data) {
data[key] = [];
for (var i = 0; i < original_data['path'].length; ++i) {
if ((original_data['normscore'][i] >= min_score) && (original_data['normscore'][i] <= max_score) && (original_data['featfreq'][i] >= min_featfreq) &&
(original_data['featcnt'][i] >= min_featcnt)) {
// filter by java
if (kind == "Java" && original_data['path'][i].indexOf(java_suffix, original_data['path'][i].length - java_suffix.length) === -1)
continue;
// filter by native
if (kind == "Native" && original_data['path'][i].indexOf(native_suffix, original_data['path'][i].length - native_suffix.length) === -1)
continue;
// filter by search regex
if (!re.test(original_data['name'][i])) {
console.log("mismatch: " + original_data['name'][i]);
continue;
}
// this row is the expected kind
data[key].push(original_data[key][i]);
}
}
}
source.trigger('change');
target.trigger('change');
"""
generic_callback = CustomJS(args=dict(
source=self.source,
original_source=self.original_source,
search_input=search_input,
max_matching_score_slider=max_matching_score_slider,
min_matching_score_slider=min_matching_score_slider,
featfreq_slider=featfreq_slider,
featcnt_slider=featcnt_slider,
kind_select=kind_select,
target=self.data_table),
code=combined_callback_code)
min_matching_score_slider.callback = generic_callback
max_matching_score_slider.callback = generic_callback
featfreq_slider.callback = generic_callback
featcnt_slider.callback = generic_callback
search_button.callback = generic_callback
kind_select.callback = generic_callback
# install callback when a row gets selected
self.source.on_change('selected', self.selected_callback)
###########################################################
# Main
###########################################################
controls = [min_matching_score_slider, max_matching_score_slider, featfreq_slider, \
featcnt_slider, kind_select, file_select, search_input, search_button, \
download_button, compare_button]
plots_box = widgetbox(*controls, width=800, sizing_mode="fixed")
layout = column(plots_box, self.data_table, sizing_mode="fixed")
return layout
