def _draw(self):
d = self._gene_data
for condition in self._conditions:
d[condition] = [
self._condition_data[condition][
self._condition_data["Sample"].index(sample)]
for sample in d["Sample"]
if sample in self._condition_data["Sample"]
]
template_update_1 = '<% if (value === "True") {print(\'<div style="height: 20px; width: 20px; background-color:'
template_update_2 = ';"></div>\')} %>'
def make_template(color):
return template_update_1 + str(color) + template_update_2
template_update_1_esc = template_update_1.replace("'", "\\'")
template_update_2_esc = template_update_2.replace("'", "\\'")
formatters = [
HTMLTemplateFormatter(template=make_template(
color=self._color_pickers[cond].color))
for cond in self._conditions
]
condition_columns = [
TableColumn(field=cond, title=cond, formatter=form, width=10)
for cond, form in zip(self._conditions, formatters)
]
columns = [
TableColumn(field="Sample", title="Sample", width=200),
TableColumn(field="Gene", title="Gene", width=10),
TableColumn(field="R1", title="R1"),
TableColumn(field="R2", title="R2"),
TableColumn(field="R3", title="R3"), *condition_columns
]
dt = DataTable(source=ColumnDataSource(d),
columns=columns,
width_policy="fit")
code = f"form.template = '{template_update_1_esc}' + cp.color + '{template_update_2_esc}'; dt.change.emit();"
for cp, formatter, col in zip(self._color_pickers.values(), formatters,
condition_columns):
cp.js_on_change(
"color",
CustomJS(args={
"cp": cp,
"col": col,
"form": formatter,
"dt": dt
},
code=code))
self._root.children.append(dt)
dt.source.patch({})
def _init_templates(self):
switch_case = ''
for c, v in CIRCLE_COLORS.items():
switch_case += "case '{}': return('{}'); break;\n".format(
str(c), v)
flag_template = """
<div style="font-weight: bold; color: <%=
(function colorfromint() {{
switch (value) {{
{}
default: return('black');
}}
}}()) %>; " ><%= value %>
</div>
""".format(switch_case)
self.flag_formatter = HTMLTemplateFormatter(
template=flag_template
) # TODO: to show float numbers NumberFormatter should be used
value_template = """
<div style="font-weight: <%=
(function colorfromint() {
switch (value) {
case 'NaN': return('bold'); break
default: return('normal');
}
}()) %>; color: <%=
(function colorfromint() {
switch (value) {
case 'NaN': return('red'); break
default: return('black');
}
}()) %>; " ><%= value %>
</div>
"""
# lg.info('>> TEMPLATE: {}'.format(value_template))
self.value_formatter = HTMLTemplateFormatter(template=value_template)
param_template = """
<div style="font-weight: bold;"><%= value %></div>
"""
# lg.info('>> TEMPLATE: {}'.format(param_template))
self.param_formatter = HTMLTemplateFormatter(template=param_template)
def get_table_formatters():
datetime_formatter = DateFormatter(format="%Y-%m-%dT%H:%M")
title_link_template = '''
<table style="table-layout: fixed; width: 100%;white-space: normal;">
<tr>
<td style="word-wrap: break-word;text-align:left">
<a href="<%= link %>"><%= value %></a>
</td>
</tr>
</table>
'''
title_link_formatter = HTMLTemplateFormatter(template=title_link_template)
return datetime_formatter, title_link_formatter
def generate_data_table(data_source: ColumnDataSource) -> DataTable:
"""
:param data_source:
:return:
"""
index_template = "<a href='https://www.python.org/dev/peps/pep-<%= index %>' " \
"target='_blank'>" \
"PEP <%= PEP %>" \
"</a>"
status_template = "<div style='background:<%= status_node_color %>;" \
" color:<%= status_font_color %>; text-align:center'>" \
"<%= Status %>" \
"</div>"
columns = [
TableColumn(field='index',
title='PEP',
width=60,
formatter=HTMLTemplateFormatter(template=index_template)),
TableColumn(field='Title', title='Title', width=280),
TableColumn(field='Status',
title='Status',
width=80,
formatter=HTMLTemplateFormatter(template=status_template)),
TableColumn(field='Created_str', title='Created', width=100),
]
# TODO: row_headers is deprecated.
# change code when bokeh 1.0 is released
data_table = DataTable(source=data_source,
columns=columns,
width=540,
height=420,
index_width=None)
return data_table
def panel(self):
result = bootstrap
price_slider = pn.widgets.RangeSlider.from_param(
self.param.price_slider, step=10000, format='0.0a')
result.sidebar.append(price_slider)
result.sidebar.append(self.param.rooms_slider)
result.sidebar.append(self.param.bathrooms_slider)
result.sidebar.append(self.param.type)
result.sidebar.append(self.param.transit_time)
image_format = r'<div> <img src="<%= value %>" height="70" alt="<%= value %>" width="70" style="float: left; margin: 0px 15px 15px 0px;" border="2" ></img> </div>'
tabulator_formatters = {
'price': NumberFormatter(format='$0,0'),
'size': NumberFormatter(format='0,0 sqft'),
'photo': HTMLTemplateFormatter(template=image_format)
}
df_widget = pn.widgets.Tabulator(self.filter_df(),
pagination='remote',
page_size=10,
formatters=tabulator_formatters,
sizing_mode='scale_both')
df_widget.add_filter(self.param.price_slider, 'price')
df_widget.add_filter(self.param.rooms_slider, 'bedrooms')
# df_pins = pn.widgets.Tabulator(self.distance_df(), pagination='remote', page_size=10, sizing_mode='scale_both')
layout = pn.Row(
pn.Card(pn.bind(self.location,
x=self.stream.param.x,
y=self.stream.param.y),
title="Map",
sizing_mode='stretch_height'),
pn.Column(
pn.Card(df_widget,
title="Properties",
sizing_mode='scale_both')))
result.sidebar.append(
pn.Card(pn.bind(self.distance_df,
x=self.stream.param.x,
y=self.stream.param.y),
title="Pins",
sizing_mode='scale_both'))
bootstrap.main.append(layout)
bootstrap.main.append(pn.Card(self.details_area, title='Details'))
return result
def create_datatable_columns(df_columns):
table_columns = []
for column in df_columns:
table_columns.append(TableColumn(field=column, title=column))
# 画像のimgとリンクを表示
if 'index' in df_columns:
table_columns.append(
TableColumn(
field='index',
title='image',
formatter=HTMLTemplateFormatter(
template=
'<a href="./mnist?index=<%= value %>" target="_blank"><img src="./mnist?index=<%= value %>"/></a>'
)))
return table_columns
def create_ui(self, data):
self.logger.info("number of data items %d", len(data))
# Create data source and data table
# path, score, software_id, featcnt, featfreq, app name, app path, decision, status, comment, active in play, still voilating
decision_editor = SelectEditor(options=[
"Unprocessed", "GPL Violation", "LGPL Violation",
"Open Source App", "False Positive", "False Negative (LGPL)",
"False Negative (GPL)"
])
status_editor = SelectEditor(options=[
"Unprocessed", "Emailed", "Confirmed", "Denied", "Authorized"
])
if self.app_info:
columns = [
TableColumn(field="myindex", title="Id"),
TableColumn(field="path", title="File Path"),
TableColumn(field="score", title="Score"),
TableColumn(field="normscore",
title="NormScore",
formatter=NumberFormatter(format="0.00")),
TableColumn(field="partial", title="PartialMatch"),
TableColumn(field="repo_id", title="Repo ID"),
TableColumn(field="software_name", title="OSS"),
TableColumn(field="version", title="Version"),
TableColumn(
field="featcnt",
title="FeatCount",
),
TableColumn(
field="featfreq",
title="FeatFreq",
),
TableColumn(field="package_name", title="Package"),
TableColumn(field="app_path", title="App Path"),
TableColumn(field="app_count", title="App Count"),
TableColumn(field="decision",
title="Decision",
editor=decision_editor),
TableColumn(field="status",
title="Status",
editor=status_editor),
TableColumn(field="comment", title="Comment"),
# I am not sure whether we should add these two fields here.
# TableColumn(field="active", title="Active in Play"),
# TableColumn(field="still_violating", title="Still Violating"),
]
else:
template_str = '<a href="' + self.REPO_URL + '/<%= value %>"><%= value %></a>'
columns = [
TableColumn(
field="myindex",
title="Id",
),
TableColumn(field="name", title="Name"),
TableColumn(field="score",
title="Score",
formatter=NumberFormatter(format="0.00")),
TableColumn(field="normscore",
title="NormScore",
formatter=NumberFormatter(format="0.00")),
TableColumn(field="partial", title="PartialMatch"),
TableColumn(field="repo_id", title="RepoID"),
TableColumn(
field="software_name",
title="OSS",
formatter=HTMLTemplateFormatter(template=template_str)),
TableColumn(field="featcnt",
title="FeatCount",
formatter=NumberFormatter(format="0,000,000")),
TableColumn(field="featfreq",
title="FeatFreq",
formatter=NumberFormatter(format="0,000,000")),
TableColumn(field="version", title="Version"),
TableColumn(field="decision",
title="Decision",
editor=decision_editor),
TableColumn(field="status",
title="Status",
editor=status_editor),
TableColumn(field="comment", title="Comment"),
TableColumn(field="path", title="Path"),
]
# source is the displayed table, and can be modified by user
# original_source is the original data, it is the base, and can only be modified by the program
self.source = ColumnDataSource(self._data)
self.original_source = ColumnDataSource(self._data)
self.data_table = DataTable(source=self.source,
columns=columns,
width=2000,
height=2000,
editable=True,
sortable=True) # Disable sortable for now!
# selector or filters
# reference link for callback: https://gist.github.com/dennisobrien/450d7da20daaba6d39d0
min_matching_score_slider = Slider(start=0,
end=2,
value=0.3,
step=.01,
title="Minimum Matching Score")
max_matching_score_slider = Slider(start=0,
end=2,
value=0.7,
step=.01,
title="Maximum Matching Score")
featfreq_slider = Slider(start=0,
end=10000,
value=0,
step=1,
title="Minimum Matching Num of Features")
featcnt_slider = Slider(start=0,
end=10000,
value=50,
step=1,
title="Minimum Feature Count is OSS")
kind_select = Select(value="All", options=["All", "Java", "Native"])
file_select = Select(value="Name", options=["Name", "MD5", "Path"])
search_input = TextInput(value=None,
title="Enter library to search",
callback=None)
search_button = Button(label="Search", button_type="success")
download_callback_code = """
var data = source.get('data');
var filetext = 'Id,File Name,Matching Score,Normalized Matching Score,Repo ID,Software Name,Feature Count,Feature Freq.,Version,Decision,Status,Comment,File Path\\n';
var order = ['myindex', 'name', 'score', 'normscore', 'repo_id', 'software_name', 'featcnt', 'featfreq', 'version', 'decision', 'status', 'comment', 'path'];
for (var i = 0; i < data['path'].length; ++i) {
var currRow = [];
for (var item in order) {
key = order[item]
currRow.push(data[key][i]);
}
var joined = currRow.join().concat('\\n');
filetext = filetext.concat(joined);
}
var filename = 'violations.csv';
var blob = new Blob([filetext], { type: 'text/csv;charset=utf-8;' });
//addresses IE
if (navigator.msSaveBlob) {
//navigator.msSaveBlob(blob, filename);
}
else {
var link = document.createElement("a");
link = document.createElement('a');
link.href = URL.createObjectURL(blob);
link.download = filename;
link.target = "_blank";
link.style.visibility = 'hidden';
link.dispatchEvent(new MouseEvent('click'));
}
"""
# enable downloading of results as a csv file
download_button = Button(label="Download", button_type="success")
download_button.callback = CustomJS(args=dict(source=self.source),
code=download_callback_code)
# enable comparison of selected rows
compare_button = Button(label="Compare", button_type="success")
compare_button.on_click(self.compare_callback)
# update on change
#controls = [min_matching_score_slider, max_matching_score_slider, featfreq_slider, \
# featcnt_slider, kind_select, file_select, button]
#for item in controls:
# item.on_change('value', lambda attr, old, new: self.update_source(item))
combined_callback_code = """
var data = source.get('data');
var original_data = original_source.get('data');
var min_score = min_matching_score_slider.get('value');
var max_score = max_matching_score_slider.get('value');
var search_input = search_input.get('value');
var min_featfreq = featfreq_slider.get('value');
var min_featcnt = featcnt_slider.get('value');
var kind = kind_select.get('value');
console.log("min score: " + min_score + ", max score: " + max_score + ", min_featfreq: " + min_featfreq + ", min_featcnt" + min_featcnt + ", kind" + kind);
var java_suffix = ".dex";
var native_suffix = ".so";
console.log("searchinput: " + search_input);
var re;
if (search_input) {
re = new RegExp(search_input);
} else {
re = new RegExp(".*");
}
for (var key in original_data) {
data[key] = [];
for (var i = 0; i < original_data['path'].length; ++i) {
if ((original_data['normscore'][i] >= min_score) && (original_data['normscore'][i] <= max_score) && (original_data['featfreq'][i] >= min_featfreq) &&
(original_data['featcnt'][i] >= min_featcnt)) {
// filter by java
if (kind == "Java" && original_data['path'][i].indexOf(java_suffix, original_data['path'][i].length - java_suffix.length) === -1)
continue;
// filter by native
if (kind == "Native" && original_data['path'][i].indexOf(native_suffix, original_data['path'][i].length - native_suffix.length) === -1)
continue;
// filter by search regex
if (!re.test(original_data['name'][i])) {
console.log("mismatch: " + original_data['name'][i]);
continue;
}
// this row is the expected kind
data[key].push(original_data[key][i]);
}
}
}
source.trigger('change');
target.trigger('change');
"""
generic_callback = CustomJS(args=dict(
source=self.source,
original_source=self.original_source,
search_input=search_input,
max_matching_score_slider=max_matching_score_slider,
min_matching_score_slider=min_matching_score_slider,
featfreq_slider=featfreq_slider,
featcnt_slider=featcnt_slider,
kind_select=kind_select,
target=self.data_table),
code=combined_callback_code)
min_matching_score_slider.callback = generic_callback
max_matching_score_slider.callback = generic_callback
featfreq_slider.callback = generic_callback
featcnt_slider.callback = generic_callback
search_button.callback = generic_callback
kind_select.callback = generic_callback
# install callback when a row gets selected
self.source.on_change('selected', self.selected_callback)
###########################################################
# Main
###########################################################
controls = [min_matching_score_slider, max_matching_score_slider, featfreq_slider, \
featcnt_slider, kind_select, file_select, search_input, search_button, \
download_button, compare_button]
plots_box = widgetbox(*controls, width=800, sizing_mode="fixed")
layout = column(plots_box, self.data_table, sizing_mode="fixed")
return layout
def index():
# 画面表示用変数のデフォルト値
warn = ''
# セッションクリアボタン押下時
if 'clear' in request.form:
for key in list(session.keys()):
session.pop(key)
session.clear()
logout_user()
##### セッションデータ読み込み開始
user_id = session.get('user_id', str(uuid.uuid4()))
session['user_id'] = user_id
user = User.query.get(user_id)
target_data = pd.DataFrame()
if user is None:
user = User(id=user_id, csv_data=pd.DataFrame().to_json())
db.session.add(user)
db.session.commit()
try:
target_data = pd.read_json(user.csv_data)
except Exception as e:
warn = str(e)
# 正解ラベルカラムと推論ラベルカラムの読み込み
label_column = session.get('label_column', '')
pred_column = session.get('pred_column', '')
##### 各フォーム変数読み込み開始
# label,predカラム設定読み込み
if 'label_column' in request.form:
label_column = request.form['label_column']
session['label_column'] = label_column
if 'pred_column' in request.form:
pred_column = request.form['pred_column']
session['pred_column'] = pred_column
# ファイルアップロード時の処理
if request.method == 'POST' and 'file' in request.files:
file = request.files['file']
if file.filename != '':
try:
target_data = pd.read_csv(file)
user.csv_data = target_data.to_json()
db.session.commit()
except Exception as e:
warn = str(e)
##### 各画面要素作成
# テーブル作成
df_columns = target_data.columns.tolist()
source = ColumnDataSource(data = target_data)
table_columns = []
for column in df_columns:
table_columns.append(TableColumn(field=column, title=column))
if 'index' in df_columns:
table_columns.append(TableColumn(field='index', title='image',
formatter=HTMLTemplateFormatter(template='<a href="./mnist?index=<%= value %>" "target=_blank"><img src="./mnist?index=<%= value %>"/></a>')))
layout_disp = DataTable(source=source,selectable = True, columns=table_columns,
sortable = True)
# コンフュージョンマトリックス作成
if label_column != '' and pred_column != '':
cm = confusion_matrix(target_data[label_column], target_data[pred_column])
length = len(cm)
cm = pd.DataFrame(cm)
cm['sum'] = cm.sum(axis='columns')
cm.columns = [str(col) for col in cm.columns]
cm = cm.append(cm.sum(axis='index'), ignore_index=True)
source = ColumnDataSource(data = cm)
# high = cm.max().max()
# low = cm.min().min()
# colors = ["#75968f", "#a5bab7", "#c9d9d3", "#e2e2e2", "#dfccce", "#ddb7b1", "#cc7878", "#933b41", "#550b1d"]
# mapper = LinearColorMapper(palette=colors, low=low, high=high)
# p = figure(title="confusion matrix",
# toolbar_location=None, tools="", x_axis_location="above")
# p.rect(x="pred", y="label", width=1, height=1, source=source,
# line_color=None, fill_color=transform('rate', mapper))
# color_bar = ColorBar(color_mapper=mapper,
# ticker=BasicTicker(desired_num_ticks=len(colors)),
# formatter=PrintfTickFormatter(format="%d"))
# p.add_layout(color_bar, 'right')
# layout_disp = p
# source = ColumnDataSource(data = cm)
table_columns = []
for column in cm.columns.tolist():
table_columns.append(TableColumn(field=column, title=column,
formatter=HTMLTemplateFormatter(template=f'<a href="./cm?label=<%= index %>&pred={column}" "target=_blank"><%= value %></a>')))
table =DataTable(source=source,selectable = True, columns=table_columns,
sortable = True)
layout_disp = Column(table, layout_disp)
# render template
script, div = components(layout_disp)
return render_template(
'index.html.j2',
warn=warn,
columns=df_columns,
label_column=label_column,
pred_column=pred_column,
plot_script=script,
plot_div=div,
js_resources=js_resources,
css_resources=css_resources,
)