def explore(df, channel='default', properties={}, bindings={}):
"""
Renders the urth-viz-explorer widget to the user output
If pandas.DataFrame assign with unique name to user namespace else use what was passed in the string
Parameters
----------
df The dataframe itself or the string representation of the variable
channel The channel to bind to defaulted to default
properties The properties e.g. {'selection-as-object': False, 'foo': 5}
bindings The bindings e.g. {'selection': 'sel'}
"""
global unique_explore_id
unique_explore_id += 1
explore_df = "unique_explore_df_name_" + str(unique_explore_id)
if isinstance(df, pandas.DataFrame) or isinstance(df, pyspark.sql.DataFrame):
get_ipython().user_ns[explore_df] = df
else:
explore_df = df
display(HTML(
"""<link rel='import' href='urth_components/declarativewidgets-explorer/urth-viz-explorer.html'
is='urth-core-import' package='jupyter-incubator/declarativewidgets_explorer'>
<template is="urth-core-bind" channel="{channel}">
<urth-viz-explorer ref='{ref}' {props} {binds}></urth-viz-explorer>
</template>"""
.format(ref=explore_df, channel=channel,
props=stringify_properties(properties), binds=stringify_bindings(bindings))
))
def stop_loader(identifier):
if util.in_ipnb():
display(Javascript("""
jQuery("#%s").remove();
""" % identifier))
else:
logger.debug("loading only works on Jupyter notebooks")
def _data_frame(solution):
df = self.nth_panel(solution - 1)
notice("biomass: {0:g}".format(df['biomass'].iat[0]))
notice("production: {0:g}".format(df['production'].iat[0]))
df = df.loc[abs(df['normalized_gaps']) >= non_zero_flux_threshold]
df.sort_values('normalized_gaps', inplace=True)
display(df)
def __call__(self):
bounds = dict(self.panels.bounds.limits())
js = "set_sliders(\'{0}\');".format(json.dumps(bounds))
display(Javascript(js))
def show_in_notebook(self, labels=None, predict_proba=True, **kwargs):
"""Shows html explanation in ipython notebook.
See as_html for parameters.
This will throw an error if you don't have IPython installed"""
from IPython.core.display import display, HTML
display(HTML(self.as_html(labels, predict_proba, **kwargs)))
def nbsample(draws, step, start=None, trace=None, chain=0, tune=None, model=None, random_seed=None):
try:
assert(hasattr(IPython.get_ipython(), 'comm_manager'))
except (AssertionError, NameError, KeyError) as e:
raise NotImplementedError(_no_notebook_error_message)
display.display_html(_javascript, raw=True)
w = ISampleWidget()
display.display(w)
t_start = time.time()
t_last = time.time()
w.max_samples = draws
w.current_samples = 0
for i,backend in enumerate(iter_sample(draws, step, start=start, trace=trace,
chain=chain, tune=tune, model=model, random_seed=None), 1):
elapsed = time.time() - t_start
elapsed_last = time.time() - t_last
if elapsed_last > 0.1:
t_last = time.time()
w.current_samples = i
w.clock = "%02i:%02i:%02i" % (elapsed / 60 / 60, elapsed / 60 % 60, elapsed % 60)
get_ipython().kernel.do_one_iteration()
if w.stopped:
break
w.current_samples = i
return backend
def show(close=None, block=None):
"""Show all figures as SVG/PNG payloads sent to the IPython clients.
Parameters
----------
close : bool, optional
If true, a ``plt.close('all')`` call is automatically issued after
sending all the figures. If this is set, the figures will entirely
removed from the internal list of figures.
block : Not used.
The `block` parameter is a Matplotlib experimental parameter.
We accept it in the function signature for compatibility with other
backends.
"""
if close is None:
close = InlineBackend.instance().close_figures
try:
for figure_manager in Gcf.get_all_fig_managers():
display(
figure_manager.canvas.figure,
metadata=_fetch_figure_metadata(figure_manager.canvas.figure)
)
finally:
show._to_draw = []
# only call close('all') if any to close
# close triggers gc.collect, which can be slow
if close and Gcf.get_all_fig_managers():
matplotlib.pyplot.close('all')
def display_diagrams():
if os.path.isfile(output_name + ".svg") and os.path.getmtime(input_path) <= os.path.getmtime(output_name + ".svg"):
if not notebook_options.no_mcd:
display(SVG(filename=output_name + ".svg"))
if notebook_options.mld:
display(HTML(filename=output_name + ".html"))
return True
def __display_compound_html(inchi):
svg = Designer.__generate_svg(inchi)
display(HTML("""
<p>
%s
</p>
""" % svg))
def animate(self, i, elapsed):
percentage = int(self.fraction(i))
display(Javascript(
"$('div#%s').width('%i%%')" % (self.divid, percentage)))
display(Javascript("$('label#%s').text('%i%% in %.1f sec')" %
(self.sec_id, fraction, round(elapsed, 1))))
def leet_puzzle(problem):
html = '<a href="https://leetcode.com/problems/' + problem
html += '">Source : https://leetcode.com/problems/' + problem + '</a>'
response = get('https://leetcode.com/problems/' + problem)
soup = BeautifulSoup(response.content)
content = soup.select('.question-content')[0]
for a in content.select('a'):
if a['href'] == '/subscribe/':
a.nextSibling.extract()
a.extract()
else:
a['href'] = 'https://leetcode.com' + a['href']
for tags in content.select('#tags'):
tags.extract()
for similar in content.select('#similar'):
similar.extract()
html += str(content)
display(HTML(html))
def fancy_table(df, div_id):
def _repr_html_(self):
if self._info_repr():
buf = StringIO(u(""))
self.info(buf=buf)
return '<pre>' + buf.getvalue() + '</pre>'
max_rows = pd.get_option("display.max_rows")
max_cols = pd.get_option("display.max_columns")
html = self.to_html(max_rows=max_rows,
max_cols=max_cols,
show_dimensions=False,
classes='table table-bordered table-striped')
text = '<div>\n'
text += html
text += '\n</div>'
#text = text.replace('border="1" ','border="2" ')
text = text.replace('dataframe ','')
text = text.replace('<table ','<table id="{}" '.format(div_id))
return text
pd.DataFrame._repr_html_ = _repr_html_
display(df)
pd.DataFrame._repr_html_ = old_rep
def init():
# JavaScript code to load the declarative widgets extension.
# Code sent to the front end from here may be executed after
# extension initialization (iterative cell execution) or
# before (Run All, Reload). This code works together with the
# extension initialization to ensure that the required API is
# available in all scenarios.
#
# Urth._initialized is a deferred that is resolved by the extension
# initialization after the global Urth instance has been setup.
# If extension initialization has not completed a new deferred is
# initialized which extension initialization will resolve.
#
# Urth.whenReady is a public API defined by extension initialization
# to delay javascript execution until dependencies have loaded. If
# extension initialization has not completed a wrapper implementation
# is setup which will invoke the real implementation when it is available.
code = '''
window.Urth = window.Urth || {};
Urth._initialized = Urth._initialized || $.Deferred();
Urth.whenReady = Urth.whenReady || function(cb) {
Urth._initialized.then(function() {
Urth.whenReady(cb);
});
};
Urth.whenReady(function() { console.log("Declarative widgets connected.") });
'''
# Send the code to the browser.
display(Javascript(code))
def draw_viz(self, json_data):
js = """
(function (){
debugger;
var viz_data = %s;
var visualization = d3plus.viz()
.legend(false)
.container("#%s")
.type("tree_map")
.size({
'value': "value",
'threshold': false
})
.id(%s)
.color(%s)
.text(%s)
.depth(1)
.data(viz_data)
.draw();
})();
""" % (json_data, self.container_id, self.id, self.color, self.name)
display(Javascript(lib=self.JS_LIBS, data=js))
def showmat(m, labels=('','',''), prec=3):
'''Display a numpy.ndarray as a latex matrix in an IPython notebook with optional caption.
m: np.ndarray
array to display
labels: (str, str, str), optional
Latex to insert before, between, and after matrices (default ('','','')).
prec: int, optional
Maximum number of decimal places. Hardcoding this as opposed to using ordinary string
formatting because the Numpy->SymPy->IPython chain makes things confusing. Feel free to
propose a better method @_@'''
if type(labels) != tuple and type(labels) != list:
labels = (labels, '', '')
elif len(labels) < 2:
labels = (labels[0], '', '')
elif len(labels) < 3:
labels = (labels[0], labels[1], '')
if type(m) != list and type(m) != tuple:
m = [m]
display(
Latex(
'%s$$' % labels[0] +
labels[1].join([
latex(Matrix(limitprec(a, prec)), mat_str='matrix')
for a in m
]) + '$$%s' % labels[2]
)
)
def draw_viz(self, json_data):
js = """
(function (){
var viz_data = %s;
var visualization = d3plus.viz()
.legend(false)
.container("#%s")
.data(viz_data)
.type("scatter")
.id("%s")
.color("%s")
.text("%s")
.x("%s")
.y("%s")
.depth(1)
.size("value")
.draw();
})();
""" % (json_data, self.container_id, self.id, self.color,
self.name, self.x, self.y)
display(Javascript(lib=self.JS_LIBS, data=js))
def draw_viz(self, json_data):
js = """
(function (){
debugger;
var viz_data = %s;
var graph_data = %s;
var visualization = d3plus.viz()
.legend(false)
.labels(false)
.container("#%s")
.type("network")
.nodes(graph_data.nodes)
.edges(graph_data.edges)
.size("%s")
.active({
"value": function(d){
return d.M == 1;
},
"spotlight":true
})
.id('%s')
.color('%s')
.text('%s')
.data(viz_data)
.draw();
})();
""" % (json_data, self.GRAPH_DATA, self.container_id, self.size, "id", self.color, self.name)
display(Javascript(lib=self.JS_LIBS, data=js))
def DrawDecisionTree(fac, datasetName, methodName):
m = GetMethodObject(fac, datasetName, methodName)
if m==None:
return None
tr = TreeReader(str(m.GetWeightFileName()))
variables = tr.getVariables();
def clicked(b):
if treeSelector.value>tr.getNTrees():
treeSelector.value = tr.getNTrees()
clear_output()
toJs = {
"variables": variables,
"tree": tr.getTree(treeSelector.value)
}
json_str = json.dumps(toJs)
JPyInterface.JsDraw.Draw(json_str, "drawDecisionTree", True)
mx = str(tr.getNTrees()-1)
treeSelector = widgets.IntText(value=0, font_weight="bold")
drawTree = widgets.Button(description="Draw", font_weight="bold")
label = widgets.HTML("<div style='padding: 6px;font-weight:bold;color:#333;'>Decision Tree [0-"+mx+"]:</div>")
drawTree.on_click(clicked)
container = widgets.HBox([label,treeSelector, drawTree])
display(container)
def DrawDNNWeights(fac, datasetName, methodName="DNN"):
m = GetMethodObject(fac, datasetName, methodName)
if m == None:
return None
try:
net = GetDeepNetwork(str(m.GetWeightFileName()), True)
except AttributeError:
print("STANDARD architecture not supported! If you want to use this function you must use CPU or GPU architecture")
numOfLayers = len(net["layers"])
options = []
vals=[]
for layer in xrange(numOfLayers):
options.append(str(layer)+"->"+str(layer+1))
vals.append(layer)
selectLayer=widgets.Dropdown(
options=options,
value=options[0],
description='Layer'
)
def drawWrapper(e):
CreateWeightHist(net, selectLayer.value)
pass
button = widgets.Button(description="Draw", font_weight="bold", font_size="16")
button.on_click(drawWrapper)
box = widgets.HBox([selectLayer, button])
display(box)
def draw(graphs, n_graphs_per_line=5, size=250, title_key=None, titles=None, smiles=False):
# we want a list of graphs
if isinstance(graphs, nx.Graph):
print "give me a list of graphs"
# make molecule objects
chem = [nx_to_rdkit(graph) for graph in graphs]
# print chem
# calculate coordinates:
set_coordinates(chem)
# take care of the subtitle of each graph
if title_key:
legend = [g.graph[title_key] for g in graphs]
elif titles:
legend = titles
else:
legend = [str(i) for i in range(len(graphs))]
if smiles == False:
# make the image
image = Draw.MolsToGridImage(chem, molsPerRow=n_graphs_per_line, subImgSize=(size, size), legends=legend)
# display on the spot
display(image)
else:
for m in chem:
print Chem.MolToSmiles(m)
print '\n'
def calculate_confusion_matrix(results):
(tp, fn, fp, tn) = (0, 0, 0, 0);
n = len(results);
for result in results:
(a, b) = result;
if a:
if b: tp += 1;
else: fp += 1;
else:
if b: fn += 1;
else: tn += 1;
htmlString = '<h1>Confusion Matrix</h1><table>';
htmlString += '<tr><th></th><th>Predicted Condition Positive</th><th>Predicted Condition Negative</th></tr>';
htmlString += '<tr><th>Actual Condition Positive</th>';
htmlString += '<td style="background-color:aquamarine; text-align: center">True Positive (TP): <br />';
htmlString += str(tp) +'/'+ str(n) + ' = ' + str(tp/n*100) + '%</td>';
htmlString += '<td style="background-color:pink; text-align: center">False Negative (FN): <br />';
htmlString += str(fn) +'/'+ str(n) + ' = ' + str(fn/n*100) + '%</td></tr>';
htmlString += '<tr><th>Actual Condition Negative</th>';
htmlString += '<td style="background-color:pink; text-align: center">False Positive (FP): <br />';
htmlString += str(fp) +'/'+ str(n) + ' = ' + str(fp/n*100) + '%</td>';
htmlString += '<td style="background-color:aquamarine; text-align: center">True Negative (TN): <br />';
htmlString += str(tn) +'/'+ str(n) + ' = ' + str(tn/n*100) + '%</td></tr>';
htmlString += '</table>';
htmlString += '<h2>Error rate</h2>' + str((fn+fp)/n*100) + '%';
htmlString += '<h2>True positive rate</h2>' + str(tp/(tp+fn)*100) + '%';
htmlString += '<h2>True negative rate</h2>' + str(tn/(tn+fp)*100) + '%';
display(HTML(htmlString));
def __init__(self,
token="72500331:AAGPGC4dI8Co3NJWHxno_HBGhLf57F1xHWA",
name="logger",
reader_chat_id=None,
print_dual_logging=True,
max_local_log_size=5):
self.name = name
self.log_queue = []
self.reader_chat_id = None
self.print_dual_logging = print_dual_logging
if print_dual_logging:
self._stdout = sys.stdout
self.max_local_log_size = max_local_log_size
self.bot = None
self.updater = Updater(token=token)
self.dispatcher = self.updater.dispatcher
self.dispatcher.addTelegramCommandHandler('reader', self.set_reader)
self.dispatcher.addTelegramCommandHandler('ping', self.pong)
self.dispatcher.addTelegramCommandHandler('flush', lambda b, u: self.flush())
self.dispatcher.addTelegramCommandHandler('help', self.help)
# I think it's a good choise for logger to launch bot in __init__
try:
display(HTML(GREETINGS_MSG.format(url=BOT_URL, link=BOT_URL.rsplit('/', 1)[1], name=name)))
except:
pass
self.launch()
def style(dpi=120):
plt.rc('savefig', dpi=dpi)
css_file = './static/custom.css'
css_txt = open(css_file, "r").read()
css = HTML(css_txt)
display( css )
def test_std_tryit(df_std):
print('df_std')
display(df_std)
sample = df_std.ix['man-1'][RAW_FEATURES]
correct = [15.154425, 36.328485, 18.901917, 54.902340]
failmsg = 'The raw man-1 values returned were not correct.\nExpected: {} for {}'.format(correct, RAW_FEATURES)
return feedback(np.allclose(sample, correct, .001), failmsg)
def notebook_widget(delay=0.5, **kwargs):
"""
Poll the queues and update a html table.
"""
from IPython.core.display import HTML, Javascript, display
# Connect to the server and retrieve status.
conn = client.form_client_connection(**kwargs)
display(HTML(TABLE))
while True:
try:
# Retrieve the data for each queue.
data = client.simple_job_query(conn)
# Reduce the information and update the table.
display(Javascript(JS.format(*[len(x) for x in data.values()])))
# Poll every half second.
time.sleep(delay)
except (KeyboardInterrupt, SystemExit):
return
def showImage(self,img):
"""
**SUMMARY**
Show the image.
**PARAMETERS**
* *img = a SimpleCV Image object to be displayed
**Example**
>>> img = Image('lenna')
>>> d = Display()
>>> d.showImage(img)
"""
# so that only the last few images are saved, newer ones over-write the old ones
img.save(NBDisplay.staticDir + os.sep + str(img.getUID() % NBDisplay.__cache__) + '.png' )
#print uid%10
options = {}
options['imageID'] = img.getUID()
options['width'] = img.width
options['height'] = img.height
options['displayID'] = self.getUID()
command = "window.disp%(displayID)s.show(%(imageID)s,%(width)s,%(height)s)" % options
#print command
#pass the id to javascript and do the rest there
display(JS(command))
def test_features_tryit(asl):
print('asl.df sample')
display(asl.df.head())
sample = asl.df.ix[98, 1][GROUND_FEATURES].tolist()
correct = [9, 113, -12, 119]
failmsg = 'The values returned were not correct. Expected: {} Found: {}'.format(correct, sample)
return feedback(sample == correct, failmsg)
def display_highlighted_source(source):
'''For use inside an IPython notebook: Display highlighted source.'''
from pygments import highlight
from pygments.lexers import CythonLexer
from pygments.formatters import HtmlFormatter
from IPython.core.display import HTML, display
display(HTML(highlight(source, CythonLexer(), HtmlFormatter(full=True))))
def show_js(name, countries):
print("%d keys" % (len(countries)))
display(HTML("""
<div id="%s" style="width: 1000px; height: 550px;"></div>
<!-- XXX didnt find https -->
<link rel="stylesheet" href="http://jvectormap.com/css/jquery-jvectormap-2.0.0.css" type="text/css" media="screen"/>
<script>
require.config({paths: {jvectormap: "http://jvectormap.com/js/jquery-jvectormap-2.0.0.min", // XXX didnt find https
worldmap: "http://jvectormap.com/js/jquery-jvectormap-world-mill-en", // XXX didnt find https
jquery: "https://code.jquery.com/jquery-1.11.1.min"}});
require(["jvectormap", "worldmap", "jquery"], function(jvectormap, worldmap, jquery) {
$(function(){
countriesdata = %s;
$('#%s').vectorMap({
map: 'world_mill_en',
series: {
regions: [{
values: countriesdata,
scale: ['#C8EEFF', '#0071A4'],
normalizeFunction: 'linear',
}]
},
onRegionTipShow: function(e, el, code){
el.html(el.html()+' (%s: '+countriesdata[code]+')');
}
});
}); });
</script>""" % (name, countries, name, name) ))
def initialize_notebook():
"""Initialize the IPython notebook display elements"""
try:
from IPython.core.display import display, Javascript
except ImportError:
print('IPython Notebook could not be loaded.')
require_js = '''
if (window['d3'] === undefined) {{
require.config({{ paths: {{d3: "http://d3js.org/d3.v3.min"}} }});
require(["d3"], function(d3) {{
window.d3 = d3;
{0}
}});
}};
'''
d3_geo_projection_js_url = "http://d3js.org/d3.geo.projection.v0.min.js"
d3_layout_cloud_js_url = ("http://wrobstory.github.io/d3-cloud/"
"d3.layout.cloud.js")
topojson_js_url = "http://d3js.org/topojson.v1.min.js"
vega_js_url = 'http://trifacta.github.com/vega/vega.js'
dep_libs = '''$.getScript("%s", function() {
$.getScript("%s", function() {
$.getScript("%s", function() {
$.getScript("%s", function() {
$([IPython.events]).trigger("vega_loaded.vincent");
})
})
})
});''' % (d3_geo_projection_js_url, d3_layout_cloud_js_url,
topojson_js_url, vega_js_url)
load_js = require_js.format(dep_libs)
display(Javascript(load_js))
def show_sim_map(ev):
cpd_id = tools.get_value(w_input_id.value.strip())
clear_output()
w_input_id.value = ""
display(self.new_list_from_ids(cpd_id).sim_map())
def InsertCSS(cssName):
display(
HTML('<link rel="stylesheet" href="' + JsDraw.__jsMVACSSDir + '/' +
cssName + '"></link>'))
def show_test(elt) -> str:
"Show associated tests for a fastai function/class"
md = ''.join(build_tests_markdown(elt))
display(Markdown(md))
def display_all(df):
with pd.option_context("display.max_rows", 1000, "display.max_columns",
1000):
display(df.T)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(font_scale=1.2, style='darkgrid')
get_ipython().magic(u'matplotlib inline')
import warnings
warnings.filterwarnings("ignore")
# In[2]:
#change display into using full screen
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
# In[3]:
#import .csv file
data = pd.read_csv('FB_data_targeting.csv')
# # 1. Data Exploration & Cleaning
# ### 1. Have a first look at data
# In[53]:
data.head()
# In[ ]:
def show(df=None, **kwargs):
"""Show a dataframe"""
html = _datatables_repr_(df, **kwargs)
display(HTML(html))
def image_to_text(shap_values):
""" Plots SHAP values for image inputs with test outputs.
Parameters
----------
shap_values : [numpy.array]
List of arrays of SHAP values. Each array has the shap (# width x height x channels x num output tokens). One array
for each sample
"""
if len(shap_values.values.shape) == 5:
for i in range(shap_values.values.shape[0]):
display(HTML(f"<br/><b>{ordinal_str(i)} instance:</b><br/>"))
image_to_text(shap_values[i])
return
uuid = ''.join(random.choices(string.ascii_lowercase, k=20))
# creating input html tokens
model_output = shap_values.output_names
output_text_html = ''
for i in range(model_output.shape[0]):
output_text_html += "<div style='display:inline; text-align:center;'>" \
+ f"<div id='{uuid}_output_flat_value_label_" + str(i) + "'" \
+ "style='display:none;color: #999; padding-top: 0px; font-size:12px;'>" \
+ "</div>" \
+ f"<div id='{uuid}_output_flat_token_" + str(i) + "'" \
+ "style='display: inline; background:transparent; border-radius: 3px; padding: 0px;cursor: default;cursor: pointer;'" \
+ f"onmouseover=\"onMouseHoverFlat_{uuid}(this.id)\" " \
+ f"onmouseout=\"onMouseOutFlat_{uuid}(this.id)\" " \
+ f"onclick=\"onMouseClickFlat_{uuid}(this.id)\" " \
+ ">" \
+ model_output[i].replace("<", "<").replace(">", ">").replace(' ##', '').replace('▁',
'').replace(
'Ġ', '') \
+ " </div>" \
+ "</div>"
# computing gray scale images
image_data = shap_values.data
image_height = image_data.shape[0]
image_width = image_data.shape[1]
# computing gray scale image
image_data_gray_scale = np.ones((image_height, image_width, 4)) * 255 * 0.5
image_data_gray_scale[:, :, 0] = np.mean(image_data, axis=2).astype(int)
image_data_gray_scale[:, :, 1] = image_data_gray_scale[:, :, 0]
image_data_gray_scale[:, :, 2] = image_data_gray_scale[:, :, 0]
# computing shap color values for every pixel and for every output token
shap_values_color_maps = shap_values.values[:, :, 0, :]
max_val = np.nanpercentile(np.abs(shap_values.values), 99.9)
shap_values_color_dict = {}
for index in range(model_output.shape[0]):
shap_values_color_dict[f'{uuid}_output_flat_token_{index}'] = (colors.red_transparent_blue(
0.5 + 0.5 * shap_values_color_maps[:, :, index] / max_val) * 255).astype(int).tolist()
# converting to json to be read in javascript
image_data_json = json.dumps(shap_values.data.astype(int).tolist())
shap_values_color_dict_json = json.dumps(shap_values_color_dict)
image_data_gray_scale_json = json.dumps(image_data_gray_scale.astype(int).tolist())
image_viz_html = f"""
<div id="{uuid}_image_viz" class="{uuid}_image_viz_content">
<div id="{uuid}_image_viz_header" style="padding:15px;margin:5px;font-family:sans-serif;font-weight:bold;">
<div style="display:inline">
<span style="font-size: 20px;"> Input/Output - Heatmap </span>
</div>
</div>
<div id="{uuid}_image_viz_content" style="display:flex;">
<div id="{uuid}_image_viz_input_container" style="padding:15px;border-style:solid;margin:5px;flex:2;">
<div id="{uuid}_image_viz_input_header" style="margin:5px;font-weight:bold;font-family:sans-serif;margin-bottom:10px">
Input Image
</div>
<div id="{uuid}_image_viz_input_content" style="margin:5px;font-family:sans-serif;">
<canvas id="{uuid}_image_canvas" style="cursor:grab;width:100%;max-height:500px;"></canvas>
<br>
<br>
<div id="{uuid}_tools">
<div id="{uuid}_zoom">
<span style="font-size:12px;margin-right:15px;"> Zoom </span>
<button id="{uuid}_minus_button" class="zoom-button" onclick="{uuid}_zoom(-1)" style="background-color: #555555;color: white; border:none;font-size:15px;">-</button>
<button id="{uuid}_plus_button" class="zoom-button" onclick="{uuid}_zoom(1)" style="background-color: #555555;color: white; border:none;font-size:15px;">+</button>
<button id="{uuid}_reset_button" class="zoom-button" onclick="{uuid}_reset()" style="background-color: #555555;color: white; border:none;font-size:15px;"> Reset </button>
</div>
<br>
<div id="{uuid}_opacity" style="display:none">
<span style="font-size:12px;margin-right:15px;"> Shap-Overlay Opacity </span>
<input type="range" min="1" max="100" value="35" style="width:100px" oninput="{uuid}_set_opacity(this.value)">
</div>
</div>
</div>
</div>
<div id="{uuid}_image_viz_output_container" style="padding:15px;border-style:solid;margin:5px;flex:1;">
<div id="{uuid}_image_viz_output_header" style="margin:5px;font-weight:bold;font-family:sans-serif;margin-bottom:10px">
Output Text
</div>
<div id="{uuid}_image_viz_output_content" style="margin:5px;font-family:sans-serif;">
{output_text_html}
</div>
</div>
</div>
</div>
"""
image_viz_script = f"""
<script>
var {uuid}_heatmap_flat_state = null;
var {uuid}_opacity = 0.35
function onMouseHoverFlat_{uuid}(id) {{
if ({uuid}_heatmap_flat_state === null) {{
document.getElementById(id).style.backgroundColor = "grey";
{uuid}_update_image_and_overlay(id);
}}
}}
function onMouseOutFlat_{uuid}(id) {{
if ({uuid}_heatmap_flat_state === null) {{
document.getElementById(id).style.backgroundColor = "transparent";
{uuid}_update_image_and_overlay(null);
}}
}}
function onMouseClickFlat_{uuid}(id) {{
if ({uuid}_heatmap_flat_state === null) {{
document.getElementById(id).style.backgroundColor = "grey";
document.getElementById('{uuid}_opacity').style.display = "block";
{uuid}_update_image_and_overlay(id);
{uuid}_heatmap_flat_state = id;
}}
else {{
if ({uuid}_heatmap_flat_state === id) {{
document.getElementById(id).style.backgroundColor = "transparent";
document.getElementById('{uuid}_opacity').style.display = "none";
{uuid}_update_image_and_overlay(null);
{uuid}_heatmap_flat_state = null;
}}
else {{
document.getElementById({uuid}_heatmap_flat_state).style.backgroundColor = "transparent";
document.getElementById(id).style.backgroundColor = "grey";
{uuid}_update_image_and_overlay(id)
{uuid}_heatmap_flat_state = id
}}
}}
}}
const {uuid}_image_data_matrix = {image_data_json};
const {uuid}_image_data_gray_scale = {image_data_gray_scale_json};
const {uuid}_image_height = {image_height};
const {uuid}_image_width = {image_width};
const {uuid}_shap_values_color_dict = {shap_values_color_dict_json};
{uuid}_canvas = document.getElementById('{uuid}_image_canvas');
{uuid}_context = {uuid}_canvas.getContext('2d');
var {uuid}_imageData = {uuid}_convert_image_matrix_to_data({uuid}_image_data_matrix, {image_height}, {image_width}, {uuid}_context);
var {uuid}_currImagData = {uuid}_imageData;
{uuid}_trackTransforms({uuid}_context);
initial_scale_factor = Math.min({uuid}_canvas.height/{uuid}_image_height,{uuid}_canvas.width/{uuid}_image_width);
{uuid}_context.scale(initial_scale_factor, initial_scale_factor);
function {uuid}_update_image_and_overlay(selected_id) {{
if (selected_id == null) {{
{uuid}_currImagData = {uuid}_imageData;
{uuid}_redraw();
}}
else {{
{uuid}_currImagData = {uuid}_blend_image_shap_map({uuid}_image_data_gray_scale, {uuid}_shap_values_color_dict[selected_id], {image_height}, {image_width}, {uuid}_opacity, {uuid}_context);
{uuid}_redraw();
}}
}}
function {uuid}_set_opacity(value) {{
{uuid}_opacity = value/100;
if ({uuid}_heatmap_flat_state !== null ) {{
{uuid}_currImagData = {uuid}_blend_image_shap_map({uuid}_image_data_gray_scale, {uuid}_shap_values_color_dict[{uuid}_heatmap_flat_state], {image_height}, {image_width}, {uuid}_opacity, {uuid}_context);
{uuid}_redraw();
}}
}}
function {uuid}_redraw() {{
// Clear the entire canvas
var p1 = {uuid}_context.transformedPoint(0, 0);
var p2 = {uuid}_context.transformedPoint({uuid}_canvas.width, {uuid}_canvas.height);
{uuid}_context.clearRect(p1.x, p1.y, p2.x - p1.x, p2.y - p1.y);
{uuid}_context.save();
{uuid}_context.setTransform(1, 0, 0, 1, 0, 0);
{uuid}_context.clearRect(0, 0, {uuid}_canvas.width, {uuid}_canvas.height);
{uuid}_context.restore();
createImageBitmap({uuid}_currImagData, {{ premultiplyAlpha: 'premultiply' }}).then(function(imgBitmap) {{
{uuid}_context.drawImage(imgBitmap, 0, 0);
}});
}}
{uuid}_redraw();
{uuid}_context.save();
var lastX = {uuid}_canvas.width / 2,
lastY = {uuid}_canvas.height / 2;
var dragStart, dragged;
{uuid}_canvas.addEventListener('mousedown', function(evt) {{
document.body.style.mozUserSelect = document.body.style.webkitUserSelect = document.body.style.userSelect = 'none';
lastX = evt.offsetX || (evt.pageX - {uuid}_canvas.offsetLeft);
lastY = evt.offsetY || (evt.pageY - {uuid}_canvas.offsetTop);
dragStart = {uuid}_context.transformedPoint(lastX, lastY);
dragged = false;
document.getElementById('{uuid}_image_canvas').style.cursor = 'grabbing';
}}, false);
{uuid}_canvas.addEventListener('mousemove', function(evt) {{
lastX = evt.offsetX || (evt.pageX - {uuid}_canvas.offsetLeft);
lastY = evt.offsetY || (evt.pageY - {uuid}_canvas.offsetTop);
dragged = true;
if (dragStart) {{
var pt = {uuid}_context.transformedPoint(lastX, lastY);
{uuid}_context.translate(pt.x - dragStart.x, pt.y - dragStart.y);
{uuid}_redraw();
}}
}}, false);
{uuid}_canvas.addEventListener('mouseup', function(evt) {{
dragStart = null;
document.getElementById('{uuid}_image_canvas').style.cursor = 'grab';
}}, false);
var scaleFactor = 1.1;
var {uuid}_zoom = function(clicks) {{
var pt = {uuid}_context.transformedPoint(lastX, lastY);
{uuid}_context.translate(pt.x, pt.y);
var factor = Math.pow(scaleFactor, clicks);
{uuid}_context.scale(factor, factor);
{uuid}_context.translate(-pt.x, -pt.y);
{uuid}_redraw();
}}
var {uuid}_reset = function(clicks) {{
{uuid}_context.restore();
{uuid}_redraw();
{uuid}_context.save();
}}
var handleScroll = function(evt) {{
var delta = evt.wheelDelta ? evt.wheelDelta / 40 : evt.detail ? -evt.detail : 0;
if (delta) {uuid}_zoom(delta);
return evt.preventDefault() && false;
}}
{uuid}_canvas.addEventListener('DOMMouseScroll', handleScroll, false);
{uuid}_canvas.addEventListener('mousewheel', handleScroll, false);
function {uuid}_trackTransforms(ctx) {{
var svg = document.createElementNS("http://www.w3.org/2000/svg", 'svg');
var xform = svg.createSVGMatrix();
ctx.getTransform = function() {{
return xform;
}}
var savedTransforms = [];
var save = ctx.save;
ctx.save = function() {{
savedTransforms.push(xform.translate(0, 0));
return save.call(ctx);
}}
var restore = ctx.restore;
ctx.restore = function() {{
xform = savedTransforms.pop();
return restore.call(ctx);
}}
var scale = ctx.scale;
ctx.scale = function(sx, sy) {{
xform = xform.scaleNonUniform(sx, sy);
return scale.call(ctx, sx, sy);
}}
var rotate = ctx.rotate;
ctx.rotate = function(radians) {{
xform = xform.rotate(radians * 180 / Math.PI);
return rotate.call(ctx, radians);
}}
var translate = ctx.translate;
ctx.translate = function(dx, dy) {{
xform = xform.translate(dx, dy);
return translate.call(ctx, dx, dy);
}}
var transform = ctx.transform;
ctx.transform = function(a, b, c, d, e, f) {{
var m2 = svg.createSVGMatrix();
m2.a = a;
m2.b = b;
m2.c = c;
m2.d = d;
m2.e = e;
m2.f = f;
xform = xform.multiply(m2);
return transform.call(ctx, a, b, c, d, e, f);
}}
var setTransform = ctx.setTransform;
ctx.setTransform = function(a, b, c, d, e, f) {{
xform.a = a;
xform.b = b;
xform.c = c;
xform.d = d;
xform.e = e;
xform.f = f;
return setTransform.call(ctx, a, b, c, d, e, f);
}}
var pt = svg.createSVGPoint();
ctx.transformedPoint = function(x, y) {{
pt.x = x;
pt.y = y;
return pt.matrixTransform(xform.inverse());
}}
}}
function {uuid}_convert_image_matrix_to_data(image_data_matrix, image_height, image_width, context) {{
var imageData = context.createImageData(image_height, image_width);
for(var row_index = 0; row_index < image_height; row_index++) {{
for(var col_index = 0; col_index < image_width; col_index++) {{
index = (row_index * image_width + col_index) * 4;
imageData.data[index + 0] = image_data_matrix[row_index][col_index][0];
imageData.data[index + 1] = image_data_matrix[row_index][col_index][1];
imageData.data[index + 2] = image_data_matrix[row_index][col_index][2];
imageData.data[index + 3] = 255;
}}
}}
return imageData;
}}
function {uuid}_blend_image_shap_map(image_data_matrix, shap_color_map, image_height, image_width, alpha, context) {{
var blendedImageData = context.createImageData(image_height, image_width);
for(var row_index = 0; row_index < image_height; row_index++) {{
for(var col_index = 0; col_index < image_width; col_index++) {{
index = (row_index * image_width + col_index) * 4;
blendedImageData.data[index + 0] = image_data_matrix[row_index][col_index][0] * alpha + (shap_color_map[row_index][col_index][0]) * ( 1 - alpha);
blendedImageData.data[index + 1] = image_data_matrix[row_index][col_index][1] * alpha + (shap_color_map[row_index][col_index][1]) * ( 1 - alpha);
blendedImageData.data[index + 2] = image_data_matrix[row_index][col_index][2] * alpha + (shap_color_map[row_index][col_index][2]) * ( 1 - alpha);
blendedImageData.data[index + 3] = image_data_matrix[row_index][col_index][3] * alpha + (shap_color_map[row_index][col_index][3]) * ( 1 - alpha);
}}
}}
return blendedImageData;
}}
</script>
"""
display(HTML(image_viz_html + image_viz_script))
def notebook_max_width():
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
def plot_tree(tree, metric: str = "probability", pic_path: str = ""):
try:
from anytree import Node, RenderTree
except:
raise ImportError(
"The anytree module seems to not be installed in your environment.\nTo be able to use this method, you'll have to install it."
)
check_types([("metric", metric, [str],), ("pic_path", pic_path, [str],)])
try:
import shutil
screen_columns = shutil.get_terminal_size().columns
except:
import os
screen_rows, screen_columns = os.popen("stty size", "r").read().split()
tree_id, nb_nodes, tree_depth, tree_breadth = (
tree["tree_id"][0],
len(tree["node_id"]),
max(tree["node_depth"]),
sum([1 if item else 0 for item in tree["is_leaf"]]),
)
print("-" * int(screen_columns))
print("Tree Id: {}".format(tree_id))
print("Number of Nodes: {}".format(nb_nodes))
print("Tree Depth: {}".format(tree_depth))
print("Tree Breadth: {}".format(tree_breadth))
print("-" * int(screen_columns))
tree_nodes = {}
if "probability/variance" in tree:
metric_tree = "probability/variance"
else:
metric_tree = "log_odds"
for idx in range(nb_nodes):
op = "<" if not (tree["is_categorical_split"][idx]) else "="
if tree["is_leaf"][idx]:
tree_nodes[tree["node_id"][idx]] = Node(
"[{}] => {} ({} = {})".format(
tree["node_id"][idx],
tree["prediction"][idx],
metric,
tree[metric_tree][idx],
)
)
else:
tree_nodes[tree["node_id"][idx]] = Node(
"[{}] ({} {} {} ?)".format(
tree["node_id"][idx],
tree["split_predictor"][idx],
op,
tree["split_value"][idx],
)
)
for idx, node_id in enumerate(tree["node_id"]):
if not (tree["is_leaf"][idx]):
tree_nodes[node_id].children = [
tree_nodes[tree["left_child_id"][idx]],
tree_nodes[tree["right_child_id"][idx]],
]
for pre, fill, node in RenderTree(tree_nodes[1]):
print("%s%s" % (pre, node.name))
if pic_path:
from anytree.dotexport import RenderTreeGraph
RenderTreeGraph(tree_nodes[1]).to_picture(pic_path)
if isnotebook():
from IPython.core.display import HTML, display
display(HTML("<img src='{}'>".format(pic_path)))
return RenderTree(tree_nodes[1])
def md(str):
display(HTML(markdown.markdown(str + "<br />")))
def animate(self):
display.clear_output()
display.display(self)
sys.stdout.flush()
def MeshViewer(R, L, F, U, f1, f2):
source = """
<!--<div id="info"><a href="http://threejs.org" target="_blank" rel="noopener">three.js</a> - dashed lines example</div>-->
<div id="container"></div>
<script src="https://threejs.org/build/three.js"></script>
<script src="https://threejs.org/examples/js/WebGL.js"></script>
<script src="https://threejs.org/examples/js/libs/stats.min.js"></script>
<script src="https://threejs.org/examples/js/libs/dat.gui.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>
<style>
.dg li {
background: #f7f7f7 !important;
}
.dg {
color: #111;
text-shadow: none;
}
.dg.main .close-button {
background: none;
}
.dg.main .close-button:hover {
background: none;
}
.dg .cr.boolean {
border-left: 1px solid #cfcfcf;
}
.dg .cr.number {
border-left: 1px solid #cfcfcf;
}
.dg .c input[type=text] {
background: #fffefe00;
outline: none;
color: #111 !important;
}
.dg .c input[type=text]:hover {
background: #fffefe00;
outline: none;
color: #111 !important;
}
.dg .c .slider {
background: #d6d6d6;
cursor: ew-resize;
border-radius: 5px;
}
.dg .c .slider:hover {
background: #d6d6d6;
}
.dg .c .slider-fg {
background: #747575;
border-radius: 5px;
}
.dg .c .slider:hover .slider-fg {
background: #42a5f5;
}
.dg li:not(.folder) {
border: 1px solid #cfcfcf;
border-radius: 2px;
}
</style>
<script>
function NewArray(type, base64) {
var binary_string = window.atob(base64);
var len = binary_string.length;
var bytes = new Uint8Array( len );
for (var i = 0; i < len; i++) {
bytes[i] = binary_string.charCodeAt(i);
}
return new type(bytes.buffer);
}
//if ( WEBGL.isWebGLAvailable() === false ) {
// document.body.appendChild( WEBGL.getWebGLErrorMessage() );
//}
var renderer, scene, camera, stats, controls;
var objects = [];
var gui;
factor_mesh = %f;
factor_force = %f;
var WIDTH = window.innerWidth, HEIGHT = window.innerHeight;
init();
animate();
function init() {
camera = new THREE.PerspectiveCamera( 60, WIDTH / HEIGHT, 1, 200 );
camera.position.z = 150;
scene = new THREE.Scene();
scene.background = new THREE.Color( 0xFFFFFF);//0x111111 );
scene.fog = new THREE.Fog( 0xFFFFFF, 50, 200);
renderer = new THREE.WebGLRenderer( { antialias: true } );
renderer.setPixelRatio( window.devicePixelRatio );
renderer.setSize( WIDTH, HEIGHT );
var container = document.getElementById( 'container' );
container.appendChild( renderer.domElement );
//stats = new Stats();
//container.appendChild( stats.dom );
//
addMesh(%s, %s, %s, %s)
window.addEventListener( 'resize', onWindowResize, false );
controls = new THREE.OrbitControls( camera, renderer.domElement );
//controls.minDistance = 10;
//controls.maxDistance = 500;
initGui();
}
function addMesh(points1, lines1, F1, U1) {
points = points1;
lines = lines1;
F = F1;
U = U1;
for(var i=0; i < points.length; i++) {
points[i] *= factor_mesh;
U[i] *= factor_mesh;
}
//var h = size * 0.5;
var geometry = new THREE.BufferGeometry();
var position = [];
//console.log(points.length, tets.length);
for(var t=0; t < lines1.length/2; t++) {
var t1 = lines1[t*2+0];
var t2 = lines1[t*2+1];
for(var x=0; x < 3; x++)
position.push(points[t1*3+x]);
for(var x=0; x < 3; x++)
position.push(points[t2*3+x]);
//console.log(t);
}
console.log("ready");
geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( position, 3 ) );
//var geometryCube = cube( 50 );
//var lineSegments = new THREE.LineSegments( geometry, new THREE.LineDashedMaterial( { color: 0xffaa00, dashSize: 3, gapSize: 1 } ) );
mesh_lines = new THREE.LineSegments( geometry, new THREE.LineBasicMaterial( { color: 0xffaa00, linewidth: 0.5, transparent: true, opacity: 0.5} ) );
mesh_lines.computeLineDistances();
objects.push( mesh_lines );
scene.add( mesh_lines );
var geometry = new THREE.BufferGeometry();
var position = [];
var force_tips = [];
for(var i=0; i < U.length/3; i++) {
f_abs = Math.sqrt(F[i*3+0]**2 + F[i*3+1]**2 + F[i*3+2]**2);
factor = factor_force*factor_mesh;//1/f_abs/3000 * f_abs * 100000;
for(var x=0; x < 3; x++)
position.push((points[i*3+x]));
for(var x=0; x < 3; x++) {
position.push(points[i * 3 + x] + F[i * 3 + x] * factor);
force_tips.push(points[i * 3 + x] + F[i * 3 + x] * factor);
}
}
geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( position, 3 ) );
force_mat = new THREE.LineBasicMaterial( { color: 0xaa0000, linewidth: 3,} );
force_lines = new THREE.LineSegments( geometry, force_mat );
force_lines.computeLineDistances();
objects.push( force_lines );
scene.add( force_lines );
var sprite = new THREE.TextureLoader().load( 'https://threejs.org/examples/textures/sprites/disc.png' );
var geometry = new THREE.BufferGeometry();
geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( points, 3 ) );
mesh_points = new THREE.Points( geometry, new THREE.PointsMaterial( { size: 8, sizeAttenuation: false, color: 0xffaa00, map: sprite, alphaTest: 0.5, transparent: true } ) );
scene.add( mesh_points );
var geometry = new THREE.BufferGeometry();
geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( force_tips, 3 ) );
force_points = new THREE.Points( geometry, new THREE.PointsMaterial( { size: 10, sizeAttenuation: false, color: 0xaa0000, map: sprite, alphaTest: 0.5, transparent: true } ) );
scene.add( force_points );
// Displacements
var geometry = new THREE.BufferGeometry();
var position = [];
var displacement_tips = [];
for(var i=0; i < U.length/3; i++) {
for(var x=0; x < 3; x++)
position.push((points[i*3+x]));
for(var x=0; x < 3; x++) {
position.push(points[i * 3 + x] + U[i * 3 + x]);
displacement_tips.push(points[i * 3 + x] + U[i * 3 + x]);
}
}
geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( position, 3 ) );
displacement_mat = new THREE.LineBasicMaterial( { color: 0x00aa00, linewidth: 2,} );
displacement_lines = new THREE.LineSegments( geometry, displacement_mat );
displacement_lines.computeLineDistances();
objects.push( displacement_lines );
scene.add( displacement_lines );
var geometry = new THREE.BufferGeometry();
geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( displacement_tips, 3 ) );
displacement_points = new THREE.Points( geometry, new THREE.PointsMaterial( { size: 10, sizeAttenuation: false, color: 0x00aa00, map: sprite, alphaTest: 0.5, transparent: true } ) );
scene.add( displacement_points );
}
function onWindowResize() {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize( window.innerWidth, window.innerHeight );
}
function animate() {
requestAnimationFrame( animate );
render();
renderer.render( scene, camera );
//stats.update();
}
function render() {
var time = Date.now() * 0.001;
scene.traverse( function ( object ) {
//if ( object.isLine ) {
//object.rotation.y = 0.25 * time;
//object.rotation.y = 0.25 * time;
//}
} );
renderer.render( scene, camera );
}
function initGui() {
gui = new dat.GUI();
var param = {
'mesh': true,
'forces': true,
'force scale': 1,
'displacements': true,
'view_range' : 200,
};
gui.add( param, 'mesh' ).onChange( function ( val ) {
mesh_lines.visible = val;
mesh_points.visible = val;
} );
gui.add( param, 'forces' ).onChange( function ( val ) {
force_lines.visible = val;
force_points.visible = val;
} );
gui.add( param, 'force scale', 1, 8, 0.1 ).onChange( function ( val ) {
var position = [];
var force_tips = [];
for(var i=0; i < U.length/3; i++) {
f_abs = Math.sqrt(F[i * 3 + 0] ** 2 + F[i * 3 + 1] ** 2 + F[i * 3 + 2] ** 2);
factor = factor_force * factor_mesh * val;//1/f_abs/3000 * f_abs * 100000;
for (var x = 0; x < 3; x++)
position.push((points[i * 3 + x]));
for (var x = 0; x < 3; x++) {
position.push(points[i * 3 + x] + F[i * 3 + x] * factor);
force_tips.push(points[i * 3 + x] + F[i * 3 + x] * factor);
}
}
force_lines.geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( position, 3 ) );
force_points.geometry.addAttribute( 'position', new THREE.Float32BufferAttribute( force_tips, 3 ) );
} );
gui.add( param, 'displacements' ).onChange( function ( val ) {
displacement_lines.visible = val;
displacement_points.visible = val;
} );
gui.add( param, 'view_range', 10, 300, 1 ).onChange( function ( val ) {
scene.fog.far = val;
} );
}
</script>
"""
source = source.replace("'", "\"")
def wrap(array):
if array.dtype == "float64":
data_type = "Float64Array"
elif array.dtype == "int32":
data_type = "Int32Array"
else:
raise TypeError(array.dtype)
return "NewArray(" + data_type + ", \"" + repr(
base64.b64encode(array))[2:-1] + "\")"
here = source % (f1, f2, wrap(R - np.mean(R, axis=0)), wrap(L), wrap(F),
wrap(U))
from IPython.core.display import HTML, display
code = "<h1></h1><iframe srcdoc='{0}' scrolling=no style='border:none; width: 100%; height: 600px'></iframe>".format(
here)
display(HTML(code))
def iplot_histogram(data,
figsize=None,
number_to_keep=None,
sort='asc',
legend=None):
""" Create a histogram representation.
Graphical representation of the input array using a vertical bars
style graph.
Args:
data (list or dict): This is either a list of dicts or a single
dict containing the values to represent (ex. {'001' : 130})
figsize (tuple): Tuple giving figure width and height.
number_to_keep (int): The number of terms to plot and
rest is made into a single bar called other values
sort (string): Could be 'asc' or 'desc'
legend (list): A list of strings to use for labels of the data.
The number of entries must match the length of data.
Raises:
VisualizationError: When legend is provided and the length doesn't
match the input data.
"""
# HTML
html_template = Template("""
<p>
<div id="histogram_$divNumber"></div>
</p>
""")
# JavaScript
javascript_template = Template("""
<script>
requirejs.config({
paths: {
qVisualization: "https://qvisualization.mybluemix.net/q-visualizations"
}
});
require(["qVisualization"], function(qVisualizations) {
qVisualizations.plotState("histogram_$divNumber",
"histogram",
$executions,
$options);
});
</script>
""")
# Process data and execute
div_number = str(time.time())
div_number = re.sub('[.]', '', div_number)
# set default figure size if none provided
if figsize is None:
figsize = (7, 5)
options = {
'number_to_keep': 0 if number_to_keep is None else number_to_keep,
'sort': sort,
'show_legend': 0,
'width': int(figsize[0]),
'height': int(figsize[1])
}
if legend:
options['show_legend'] = 1
data_to_plot = []
if isinstance(data, dict):
data = [data]
if legend and len(legend) != len(data):
raise VisualizationError("Length of legendL (%s) doesn't match number "
"of input executions: %s" %
(len(legend), len(data)))
for item, execution in enumerate(data):
exec_data = process_data(execution, options['number_to_keep'])
out_dict = {'data': exec_data}
if legend:
out_dict['name'] = legend[item]
data_to_plot.append(out_dict)
html = html_template.substitute({'divNumber': div_number})
javascript = javascript_template.substitute({
'divNumber': div_number,
'executions': data_to_plot,
'options': options
})
display(HTML(html + javascript))
def model_view(attention, tokens, sentence_b_start=None, prettify_tokens=True):
"""Render model view
Args:
attention: list of ``torch.FloatTensor``(one for each layer) of shape
``(batch_size(must be 1), num_heads, sequence_length, sequence_length)``
tokens: list of tokens
sentence_b_start: index of first wordpiece in sentence B if input text is sentence pair (optional)
prettify_tokens: indicates whether to remove special characters in wordpieces, e.g. Ġ
"""
# Generate unique div id to enable multiple visualizations in one notebook
vis_id = 'bertviz-%s'%(uuid.uuid4().hex)
if sentence_b_start is not None:
vis_html = """
<div id='%s'>
<span style="user-select:none">
Attention: <select id="filter">
<option value="all">All</option>
<option value="aa">Sentence A -> Sentence A</option>
<option value="ab">Sentence A -> Sentence B</option>
<option value="ba">Sentence B -> Sentence A</option>
<option value="bb">Sentence B -> Sentence B</option>
</select>
</span>
<div id='vis'></div>
</div>
"""%(vis_id)
else:
vis_html = """
<div id='%s'>
<div id='vis'></div>
</div>
"""%(vis_id)
if prettify_tokens:
tokens = format_special_chars(tokens)
attn = format_attention(attention)
attn_data = {
'all': {
'attn': attn.tolist(),
'left_text': tokens,
'right_text': tokens
}
}
if sentence_b_start is not None:
slice_a = slice(0, sentence_b_start) # Positions corresponding to sentence A in input
slice_b = slice(sentence_b_start, len(tokens)) # Position corresponding to sentence B in input
attn_data['aa'] = {
'attn': attn[:, :, slice_a, slice_a].tolist(),
'left_text': tokens[slice_a],
'right_text': tokens[slice_a]
}
attn_data['bb'] = {
'attn': attn[:, :, slice_b, slice_b].tolist(),
'left_text': tokens[slice_b],
'right_text': tokens[slice_b]
}
attn_data['ab'] = {
'attn': attn[:, :, slice_a, slice_b].tolist(),
'left_text': tokens[slice_a],
'right_text': tokens[slice_b]
}
attn_data['ba'] = {
'attn': attn[:, :, slice_b, slice_a].tolist(),
'left_text': tokens[slice_b],
'right_text': tokens[slice_a]
}
params = {
'attention': attn_data,
'default_filter': "all",
'root_div_id': vis_id
}
attn_seq_len = len(attn_data['all']['attn'][0][0])
if attn_seq_len != len(tokens):
raise ValueError(f"Attention has {attn_seq_len} positions, while number of tokens is {len(tokens)}")
display(HTML(vis_html))
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
vis_js = open(os.path.join(__location__, 'model_view.js')).read().replace("PYTHON_PARAMS", json.dumps(params))
display(Javascript(vis_js))
def show_video(url):
"Display video in `url`."
data = f'<iframe width="560" height="315" src="{url}" frameborder="0" allowfullscreen></iframe>'
return display(HTML(data))
def display(self):
display(self._widget_box, self._submit)
def _display_df(df: pd.DataFrame, df_name: str):
display(_html_from_str(df_name, tag_type='h3'))
display(df)
def jekyll_div(s, c, h, icon=None):
icon = ifnone(icon, c)
res = f'<div markdown="span" class="alert alert-{c}" role="alert"><i class="fa fa-{c}-circle"></i> <b>{h}: </b>{s}</div>'
display(Markdown(res))
def print_dict(dictionary):
html = dict_to_html(dictionary)
display(HTML(html))
"""Internal services for MavenWorks in Python."""
# can't do `import .pkg-name` in Python
from .expression_evaluator import * # noqa F401 F403
from .KernelPartManager import * # noqa F401 F403
from .MetadataComm import * # noqa F401 F403
from IPython.core.display import display
display({
"text/plain": ""
}, transient={
"mavenomics_state_ok": True
}, raw=True)
def display_answers(answers):
if not answers: return
df = _answers2df(answers)
from IPython.core.display import display, HTML
return display(HTML(df.to_html(render_links=True, escape=False)))
def display_dfs(df_list: Sequence[pd.DataFrame], title: str=None) -> None:
if title is not None:
display(_html_from_str(title, tag_type='h3'))
display(*df_list)
def display_mention(mention):
res = JupyterVisualizer.mention_to_html(mention)
display(HTML(res))
def display(self):
table = self.dataframe.astype('str')
table = table.applymap(_correct_format)
display(HTML(table.to_html(index=False, col_space=100)))
def display(self):
from IPython.core.display import display
display(self)
return self
def display_graph(s, graph_name="stanford-collapsed", css=None, distance=None, div_id=None):
res = JupyterVisualizer.graph_to_html(s=s, graph_name=graph_name, css=css, distance=distance, div_id=div_id)
display(HTML(data=res))
def view(struct_or_mol, host="http://localhost:8050"):
"""
Generate a temporary link to view your structure online using Material Project's
Crystal Toolkit.
Note that your structure or molecule is sent to the remote host to be viewed.
Structures or molecules are not stored permanently or otherwise processed, but
only retained for the purposes of visualization for the user who requested the link.
The full source code of the viewer is available online.
Links expire by default after 7 days of last being viewed, but no guarantees
are provided while this service is currently in beta.
"""
warnings.warn("This feature is currently in beta.")
try:
req = requests.request("GET", urllib.parse.urljoin(host, "/version"))
except:
raise ConnectionError(
"Could not get a response from Crystal Toolkit, it may be offline "
"or down for maintenance, or you may need to upgrade pymatgen.")
if req.status_code != 200:
raise ConnectionError(
"Could not get correct response from Crystal Toolkit, it may be "
"offline or down for maintenance, or you may need to upgrade pymatgen."
)
try:
json = req.json()
except JSONDecodeError:
raise ConnectionError(
"Could not get correct reponse from Crystal Toolkit, the version "
"deployed online may need to be upgraded.")
if json.get("crystal_toolkit_api_version", None) != 1:
raise RuntimeError(
"Crystal Toolkit version has changed, you may need to upgrade "
"pymatgen for view function to work correctly.")
# note molecules and graphs don't have a verbosity option
payload = dumps(struct_or_mol.as_dict(verbosity=0)).encode("ascii")
payload = zlib.compress(payload)
payload = base64.urlsafe_b64encode(payload).decode("ascii")
payload = "?structure={}".format(payload)
if len(payload) > 2048:
raise ValueError(
"The structure or molecule is currently too large to be sent to the "
"remote server, please try again with a later version.")
url = urllib.parse.urljoin(host, payload)
try:
if get_ipython().__class__.__name__ == "ZMQInteractiveShell":
in_jupyter = True
else:
in_jupyter = False
except NameError:
in_jupyter = False
if in_jupyter:
from IPython.core.display import display, HTML
display(
HTML("<a href='{}' target='_blank'>Click to open structure or "
"molecule in viewer.</a>".format(url)))
else:
print(url)
def show_window(window_name, file_path, button_text = None, **kwargs):
html_str = Display._get_window_html(window_name,file_path, button_text, **kwargs)
display(HTML(html_str))
def train_and_evaluate(config, train_env, eval_env, eval_callback, tb_log_name):
"""Trains and evaluates on separate environments with config parameters.
Args:
config (Dict): Relevant parameters.
train_env (RampupEnv2): Environment to train with.
eval_env (RampupEnv2): Environment to evaluate with.
eval_callback (EvalCallback): Callback wrapper for evaluation
callbacks during training.
tb_log_name (str): Log name to identify metrics on Tensorboard.
"""
best_model = None
best_mean = -np.inf
# sched_LR = LinearSchedule(config["TIMESTEPS"], 0.005, 0.00001)
for i in range(config["REPETITIONS"]):
print(f"\nRunning repetition {i+1}/{config['REPETITIONS']}...")
model = A2C(
"MlpPolicy",
train_env,
learning_rate=config["LEARNING_RATE"],
policy_kwargs=config["POLICY_KWARGS"],
tensorboard_log=config["TENSORBOARD_LOG"],
verbose=0,
)
model.learn(
total_timesteps=config["TIMESTEPS"],
callback=[eval_callback, TensorboardCallback(tb_log_name)],
tb_log_name=tb_log_name,
)
if config["SHOW_TABLE"]:
eval_env.fill_table = True
obs = eval_env._set_initial_state(initial_state_status=3)
while not eval_env.done:
action, _states = model.predict(obs, deterministic=False)
obs, reward, done, info = eval_env.step(action)
mean_reward, std_reward = evaluate_policy(
model, eval_env, n_eval_episodes=config["EVAL_EPISODES"]
)
if mean_reward > best_mean:
best_mean = mean_reward
best_model = model
if config["PUNISH_ILLEGAL"]:
economic_potential = eval_env.demand.economic_potential_no_illegal()
else:
economic_potential = eval_env.demand.economic_potential()
lost_potential = economic_potential - max(mean_reward, 0)
lost_potential_perc = round(lost_potential / economic_potential * 100, 4)
summary = "POLICY EVALUATION RESULTS"
summary += f"\nEvaluated episodes:\t{config['EVAL_EPISODES']}"
summary += f"\nMean reward:\t\t{mean_reward}"
summary += f"\nStandard deviation:\t{std_reward}"
summary += f"\nEconomic potential:\t{economic_potential}"
summary += f"\nLost potential:\t\t{lost_potential} ({lost_potential_perc}%)"
print(summary)
if config["SHOW_TABLE"]:
print("Sample Episode Table")
display(eval_env.episode_table)
return best_model, train_env, eval_env
def start(self):
self.box = self.updateBox()
display(self.box)
if len(self.workflow.to_ext_words) > 0:
self.initNextStepWhileReviewing()
pass
