#!/usr/bin/env python
# coding: utf-8
# In[ ]:
import sys
sys.path.insert(0, "./../") #so we can import our modules properly
# In[ ]:
# iPhython
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:90% !important; }</style>"))
from matplotlib import rcParams
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'notebook')
import numpy as np
import pandas as pd
# # House E
# In[ ]:
path_to_house = "./datasets/dfE_300s.hdf"
df_house = pd.read_hdf(path_to_house)
print('\n\nStart time: {}'.format(df_house.index[1]))
import pymc3 as pm
import numpy as np
from scipy.stats import norm
import matplotlib.pyplot as plt
import matplotlib.style as style
from IPython.core.display import HTML
# PLOTTING CONFIG
%matplotlib inline
style.use('fivethirtyeight')
plt.rcParams["figure.figsize"] = (14, 7)
HTML("""
<style>
.output_png {
display: table-cell;
text-align: center;
vertical-align: center;
}
</style>
""")
plt.figure(dpi=100)
##### SIMULATION #####
# MODEL BUILDING
with pm.Model() as model:
lam = pm.Uniform("lambda", upper=20)
normal = pm.Poisson("poisson", mu=lam, observed=sample)
# MODEL RUN
with model:
step = pm.Metropolis()
import re
p = re.compile('([^a-z_1-9]+)([a-z_1-9]*)')
import itertools
from io import StringIO
import requests
def read_url(url):
return pd.read_csv(StringIO(requests.get(url).text))
from IPython.core.display import display, HTML
display(
HTML(
"<style>.container { width:100%; margin-left: 0%; margin-right:auto;}</style>"
))
display(
HTML(
"<style>div.cell { width:100%; margin-left: 0%; margin-right:auto;}</style>"
))
import statsmodels.formula.api as sm
import statsmodels.api
import sklearn.linear_model
import lightgbm as lgbm
import shap
import xgboost
from sklearn.impute import SimpleImputer
from scipy.stats import mode
def _handle_scatterplot(self, df):
parser = self._get_scatterplot_parser()
try:
args = parser.parse_args(self.options)
except SystemExit:
return
if not isinstance(df, pandas.core.frame.DataFrame):
raise ValueError('Not of DataFrame type')
tid = str(self.get_tid('scatterplot'))
if df.shape[0] > args.limit and self.kernel:
self.kernel.warn(
f"Only the first {args.limit} of the {df.shape[0]} records are plotted. Use option --limit to set a new limit.")
# replacing ' ' with   and '-' with unicode hyphen will disallow webpage to separate words
# into lines
indexes = [str(x).replace(' ', ' ').replace('-', '‑') for x in df.index]
data = df.head(args.limit)
nrow = data.shape[0]
if not args.cols:
if df.shape[1] == 1:
args.cols = ['_index', df.columns[0]]
elif df.shape[1] == 2:
args.cols = list(df.columns)
else:
raise ValueError(f'Please specify columns for plot. Available columns are {" ".join(df.columns)}')
if len(args.cols) == 1:
args.cols = ['_index', args.cols[0]]
if args.cols[0] == '_index':
args.tooltip = [args.cols[0]] + (args.tooltip if args.tooltip else [])
else:
args.tooltip = ['_index', args.cols[0]] + (args.tooltip if args.tooltip else [])
# check datatype
for col in args.cols + args.tooltip + (args.by if args.by else []):
if col == '_index':
continue
if col not in data.columns:
raise ValueError(f"Invalid column name {col}")
# tooltip and --by columns does not have to be numeric
for col in args.cols:
if col == '_index':
continue
if not self._is_numeric_type(data[col].dtype):
raise ValueError(f"Column {col} is not of numeric type")
if args.cols[0] == '_index':
val_x = list(range(0, nrow))
else:
val_x = self._to_list(data[args.cols[0]])
all_series = []
if args.by:
# create seris with _by
vals = []
for by_col in args.by:
vals.append(sorted(list(set(data[by_col]))))
# outer product
import itertools
categories = list(itertools.product(*vals))
for col in args.cols[1:]:
series = {}
series['clickable'] = True
series['hoverable'] = True
if col == '_index':
val_y = list(range(1, nrow + 1))
else:
val_y = self._to_list(data[col])
tooltip = ['<br>'.join([f'{"index" if t == "_index" else t}: {idxvalue if t == "_index" else df[t][idx]}'
for t in args.tooltip])
for idx,idxvalue in enumerate(indexes)]
all_data = [(x, y, z) for x, y, z in zip(val_x, val_y, tooltip)]
if args.by:
for cat in categories:
series = {}
series['label'] = col + ' (' + ' '.join(f'{x}={y}' for x, y in zip(args.by, cat)) + ')'
# find index of values that falls into the category
series['data'] = [
all_data[i] for i in range(len(all_data)) if
all(data[b][i]==v for b,v in zip(args.by, cat))
]
if len(series['data']) > 0:
all_series.append(series)
else:
series['label'] = col
series['data'] = all_data
all_series.append(series)
options = defaultdict(dict)
options['xaxis'] = {}
options['yaxis'] = {}
options['series']['lines'] = {'show': is_sorted(val_x) and not args.by if not args.show or 'lines' in args.show else False }
options['series']['points'] = {'show': True if not args.show or 'points' in args.show else False }
options['grid']['hoverable'] = True
options['grid']['clickable'] = True
# if there are actual indexes... and plot by x
class_name = 'scatterplot'
if args.cols[0] == '_index' and not isinstance(df.index, pandas.RangeIndex):
options['xaxis']['ticks'] = [[x,str(y)] for x,y in enumerate(indexes)]
class_name = 'scatterplot_by_rowname'
if args.xlim:
options['xaxis']['min'] = args.xlim[0]
options['xaxis']['max'] = args.xlim[1]
if args.ylim:
options['yaxis']['min'] = args.ylim[0]
options['yaxis']['max'] = args.ylim[1]
optfunc = ''
if args.log and 'x' in args.log:
range_x = [min(val_x), min(val_x)]
optfunc = '''
options['xaxis']['transform'] = function(v) { return Math.log(v); }
options['xaxis']['inverseTransform'] = function(v) { return Math.exp(v); }
'''
ticks = self._natural_ticks(range_x)
if ticks:
optfunc += f'''
options['xaxis']['ticks'] = {ticks!r};
'''
if not args.xlim:
options['xaxis']['min'] = range_x[0]
options['xaxis']['max'] = range_x[1]
if args.log and 'y' in args.log:
range_y = [min([min([x[1] for x in series['data']]) for series in all_series]),
max([max([x[1] for x in series['data']]) for series in all_series])]
optfunc += '''
options['yaxis']['transform'] = function(v) { return Math.log(v); };
options['yaxis']['inverseTransform'] = function(v) { return Math.exp(v); };
'''
ticks = self._natural_ticks(range_y)
if ticks:
optfunc += f'''
options['yaxis']['ticks'] = {ticks!r};
'''
# flot does not seems to scale correctly without min/max
if not args.ylim:
options['yaxis']['min'] = range_y[0]
options['yaxis']['max'] = range_y[1]
code = """
<div class='scatterplot_container'>
<div class='""" + class_name + """' id='dataframe_scatterplot_""" + tid + """' width='""" + args.width + """' height='""" + args.height + """'></div>
<script language="javascript" type="text/javascript" src="http://www.flotcharts.org/flot/jquery.flot.js"></script>
<script>
var options = """ + json.dumps(options) + """;""" + optfunc + """
function plotScatterPlot""" + tid + """() {
plot = $.plot('#dataframe_scatterplot_""" + tid + """', """ + json.dumps(all_series) + """, options)
if ($('#dftooltip').length == 0) {
$("<div id='dftooltip'></div>").css({
position: "absolute",
display: "none",
border: "1px solid #fdd",
padding: "2px",
"background-color": "#fee",
"z-index": 1000,
opacity: 0.80
}).appendTo("body");
}
$('#dataframe_scatterplot_""" + tid + """').bind("plothover", function (event, pos, item) {
if (item) {
$("#dftooltip").html((item.series.label + ": " +
item.series.data[item.dataIndex][1].toString() + "<br>" +
item.series.data[item.dataIndex][2]).trim()).css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#dftooltip").hide();
}
});
}
""" + """
// we will wait for the div to be displayed on the HTML/Jupyter side before we plot
// the figure. This might not be necessary but at least this is a chance for us
// to resize the div and avoid some flot error
var dt = 100;
// the frontend might be notified before the table is inserted as results.
function showFigure""" + tid + """() {
if ( $('#dataframe_scatterplot_""" + tid + """').length === 0 || $.plot === undefined ) {
dt = dt * 1.5; // slow-down checks for datatable as time goes on;
setTimeout(showFigure""" + tid + """, dt);
return;
} else {
$('#dataframe_scatterplot_""" + tid + """').css('width', '""" + args.width + """').css('height', '""" + args.height + """');
plotScatterPlot""" + tid + """();
}
}
showFigure""" + tid + """()
</script>
</div>"""
return {'text/html': HTML(code).data}
def style_nb(path):
styles = open(path, "r").read()
return HTML("""<style>{}<style>""".format(styles))
def iplot_paulivec(rho, options=None):
""" Create a paulivec representation.
Graphical representation of the input array.
Args:
rho (array): Density matrix
options (dict): Representation settings containing
- width (integer): graph horizontal size
- height (integer): graph vertical size
- slider (bool): activate slider
- show_legend (bool): show legend of graph content
"""
# HTML
html_template = Template("""
<p>
<div id="paulivec_$divNumber"></div>
</p>
""")
# JavaScript
javascript_template = Template("""
<script>
requirejs.config({
paths: {
qVisualization: "https://qvisualization.mybluemix.net/q-visualizations"
}
});
require(["qVisualization"], function(qVisualizations) {
qVisualizations.plotState("paulivec_$divNumber",
"paulivec",
$executions,
$options);
});
</script>
""")
if not options:
options = {}
# Process data and execute
div_number = str(time.time())
div_number = re.sub('[.]', '', div_number)
if 'slider' in options and options['slider'] is True:
options['slider'] = 1
else:
options['slider'] = 0
if 'show_legend' in options and options['show_legend'] is False:
options['show_legend'] = 0
else:
options['show_legend'] = 1
data_to_plot = []
rho_data = process_data(rho)
data_to_plot.append(dict(data=rho_data))
html = html_template.substitute({'divNumber': div_number})
javascript = javascript_template.substitute({
'divNumber': div_number,
'executions': data_to_plot,
'options': options
})
display(HTML(html + javascript))
def tfidf_kmeans_classify_feature__transform(df,
df_name,
feat,
tfidf_vectorizer,
idx_term_map,
kmeans,
df_kmeans_clusters,
verbosity=1,
display_max_rows=25):
"""
Other notes:
nan values MUST be dealt with beforehand!
"""
df_copy = df.copy()
# finally, fit docs to the new tfidf kmeans class - this adds new f"{feat}_tfidf_kmeans_class" feature
# note that this is what should be use to classify, for example, the test/validation date set
# a new model should NOT be built for the test/validation date set
df_copy, feat_name_class = _tfidf_kmeans_classify_feature(
df_copy, feat, kmeans, tfidf_vectorizer, idx_term_map)
if verbosity > 0:
display(
HTML(f"<h3><i>{feat}</i> to <i>{feat_name_class}</i> Mapping:</h3>"
))
display(
HTML(df_copy[[feat, feat_name_class
]].to_html(notebook=True,
justify='left',
max_rows=display_max_rows)))
display(
HTML(f"<p><br>building distribution plot of {feat_name_class}..."))
display(HTML(f"<h3><i>{feat_name_class}</i> Distribution:</h3>"))
plt.figure(figsize=(15, 6))
df_copy[feat_name_class].hist(bins=len(df_kmeans_clusters))
plt.show()
display(HTML(f"<p><br>computing <i>frequency</i> of {feat_name_class}..."))
# there may be some classes from the training set which do not occur in test/validation
# so reset to the class having median frequency in X
df_kmeans_clusters_copy = df_kmeans_clusters.copy()
df_kmeans_clusters_copy = df_kmeans_clusters_copy.reset_index()
df_kmeans_clusters_copy['frequency'] = 0
for _class, class_freq in df_copy[feat_name_class].value_counts().items():
df_kmeans_clusters_copy['frequency'] = np.where(
df_kmeans_clusters_copy['centroid_idx'] == _class, class_freq,
df_kmeans_clusters_copy['frequency'])
df_kmeans_clusters_copy = df_kmeans_clusters_copy.set_index('centroid_idx')
display(HTML(f"<pre>{s_all_done}</pre>"))
if verbosity > 0:
df_mapped_classes = df_kmeans_clusters_copy[
df_kmeans_clusters_copy.frequency > 0]
df_unmapped_classes = df_kmeans_clusters_copy[
df_kmeans_clusters_copy.frequency == 0]
display(
HTML(
f"<h3><code>{len(df_mapped_classes)} KMeans</code> Cluster Centroids (<i>{feat_name_class}</i>) from fit occur in X, ordered by <i>frequency</i>:</h3>"
))
display(
HTML(df_kmeans_clusters_copy[
df_kmeans_clusters_copy.frequency > 0].sort_values(
by='frequency',
ascending=False).to_html(notebook=True,
justify='left',
max_rows=display_max_rows)))
if len(df_unmapped_classes) > 0:
display(
HTML(
f"<h3><code>{len(df_unmapped_classes)} KMeans</code> Cluster Centroids from fit do not ocur in X</h3>"
))
return df_copy, df_kmeans_clusters_copy, feat_name_class
def model_summary_string(keras_model, mode='keras', show_parameters=True, display=False):
"""Model summary in a formatted string, similar to Keras model summary function.
Parameters
----------
keras_model : keras model
Keras model
mode : str
Summary mode ['extended', 'keras']. In case 'keras', standard Keras summary is returned.
Default value keras
show_parameters : bool
Show model parameter count and input / output shapes
Default value True
display : bool
Display summary immediately, otherwise return string
Default value False
Returns
-------
str
Model summary
"""
if is_jupyter():
ui = FancyHTMLStringifier()
html_mode = True
else:
ui = FancyStringifier()
html_mode = False
output = ''
output += ui.line('Model summary') + '\n'
if mode == 'extended' or mode == 'extended_wide':
layer_name_map = {
'BatchNormalization': 'BatchNorm',
}
layer_type_html_tags = {
'InputLayer': '<span class="label label-default">{0:s}</span>',
'Dense': '<span class="label label-primary">{0:s}</span>',
'TimeDistributed': '<span class="label label-primary">{0:s}</span>',
'BatchNorm': '<span class="label label-default">{0:s}</span>',
'Activation': '<span class="label label-default">{0:s}</span>',
'Dropout': '<span class="label label-default">{0:s}</span>',
'Flatten': '<span class="label label-success">{0:s}</span>',
'Reshape': '<span class="label label-success">{0:s}</span>',
'Permute': '<span class="label label-success">{0:s}</span>',
'Conv1D': '<span class="label label-warning">{0:s}</span>',
'Conv2D': '<span class="label label-warning">{0:s}</span>',
'MaxPooling1D': '<span class="label label-success">{0:s}</span>',
'MaxPooling2D': '<span class="label label-success">{0:s}</span>',
'MaxPooling3D': '<span class="label label-success">{0:s}</span>',
'AveragePooling1D': '<span class="label label-success">{0:s}</span>',
'AveragePooling2D': '<span class="label label-success">{0:s}</span>',
'AveragePooling3D': '<span class="label label-success">{0:s}</span>',
'GlobalMaxPooling1D': '<span class="label label-success">{0:s}</span>',
'GlobalMaxPooling2D': '<span class="label label-success">{0:s}</span>',
'GlobalMaxPooling3D': '<span class="label label-success">{0:s}</span>',
'GlobalAveragePooling1D': '<span class="label label-success">{0:s}</span>',
'GlobalAveragePooling2D': '<span class="label label-success">{0:s}</span>',
'GlobalAveragePooling3D': '<span class="label label-success">{0:s}</span>',
'RNN': '<span class="label label-danger">{0:s}</span>',
'SimpleRNN': '<span class="label label-danger">{0:s}</span>',
'GRU': '<span class="label label-danger">{0:s}</span>',
'CuDNNGRU': '<span class="label label-danger">{0:s}</span>',
'LSTM': '<span class="label label-danger">{0:s}</span>',
'CuDNNLSTM': '<span class="label label-danger">{0:s}</span>',
'Bidirectional': '<span class="label label-danger">{0:s}</span>'
}
import keras
from distutils.version import LooseVersion
import keras.backend as keras_backend
table_data = {
'layer_type': [],
'output': [],
'parameter_count': [],
'name': [],
'connected_to': [],
'activation': [],
'initialization': []
}
row_separators = []
prev_name = None
for layer_id, layer in enumerate(keras_model.layers):
connections = []
if LooseVersion(keras.__version__) >= LooseVersion('2.1.3'):
for node_index, node in enumerate(layer._inbound_nodes):
for i in range(len(node.inbound_layers)):
inbound_layer = node.inbound_layers[i].name
inbound_node_index = node.node_indices[i]
inbound_tensor_index = node.tensor_indices[i]
connections.append(
inbound_layer + '[' + str(inbound_node_index) + '][' + str(inbound_tensor_index) + ']'
)
else:
for node_index, node in enumerate(layer.inbound_nodes):
for i in range(len(node.inbound_layers)):
inbound_layer = node.inbound_layers[i].name
inbound_node_index = node.node_indices[i]
inbound_tensor_index = node.tensor_indices[i]
connections.append(
inbound_layer + '[' + str(inbound_node_index) + '][' + str(inbound_tensor_index) + ']'
)
config = DictContainer(layer.get_config())
layer_name = layer.__class__.__name__
if layer_name in layer_name_map:
layer_name = layer_name_map[layer_name]
if html_mode and layer_name in layer_type_html_tags:
layer_name = layer_type_html_tags[layer_name].format(layer_name)
if config.get_path('kernel_initializer.class_name') == 'VarianceScaling':
init = str(config.get_path('kernel_initializer.config.distribution', '---'))
elif config.get_path('kernel_initializer.class_name') == 'RandomUniform':
init = 'uniform'
else:
init = '-'
name_parts = layer.name.split('_')
if prev_name != name_parts[0]:
row_separators.append(layer_id)
prev_name = name_parts[0]
table_data['layer_type'].append(layer_name)
table_data['output'].append(str(layer.output_shape))
table_data['parameter_count'].append(str(layer.count_params()))
table_data['name'].append(layer.name)
table_data['connected_to'].append(str(connections[0]) if len(connections) > 0 else '-')
table_data['activation'].append(str(config.get('activation', '-')))
table_data['initialization'].append(init)
trainable_count = int(
numpy.sum([keras_backend.count_params(p) for p in set(keras_model.trainable_weights)])
)
non_trainable_count = int(
numpy.sum([keras_backend.count_params(p) for p in set(keras_model.non_trainable_weights)])
)
# Show row separators only if they are useful
if len(row_separators) == len(keras_model.layers):
row_separators = None
if mode == 'extended':
output += ui.table(
cell_data=[table_data['name'], table_data['layer_type'], table_data['output'], table_data['parameter_count']],
column_headers=['Layer name', 'Layer type', 'Output shape', 'Parameters'],
column_types=['str30', 'str20', 'str25', 'str20'],
column_separators=[1, 2],
row_separators=row_separators,
indent=4
)
elif mode == 'extended_wide':
output += ui.table(
cell_data=[table_data['name'], table_data['layer_type'], table_data['output'], table_data['parameter_count'],
table_data['activation'], table_data['initialization']],
column_headers=['Layer name', 'Layer type', 'Output shape', 'Parameters', 'Act.', 'Init.'],
column_types=['str30', 'str20', 'str25', 'str20', 'str15', 'str15'],
column_separators=[1, 2, 3],
row_separators=row_separators,
indent=4
)
if show_parameters:
output += ui.line('') + '\n'
output += ui.line('Parameters', indent=4) + '\n'
output += ui.data(indent=6, field='Total', value=trainable_count + non_trainable_count) + '\n'
output += ui.data(indent=6, field='Trainable', value=trainable_count) + '\n'
output += ui.data(indent=6, field='Non-Trainable', value=non_trainable_count) + '\n'
else:
output_buffer = []
keras_model.summary(print_fn=output_buffer.append)
for line in output_buffer:
if is_jupyter():
output += ui.line('<code>'+line+'</code>', indent=4) + '\n'
else:
output += ui.line(line, indent=4) + '\n'
model_config = keras_model.get_config()
if show_parameters:
output += ui.line('') + '\n'
output += ui.line('Input', indent=4) + '\n'
output += ui.data(indent=6, field='Shape', value=keras_model.input_shape) + '\n'
output += ui.line('Output', indent=4) + '\n'
output += ui.data(indent=6, field='Shape', value=keras_model.output_shape) + '\n'
if isinstance(model_config, dict) and 'layers' in model_config:
output += ui.data(
indent=6,
field='Activation',
value=model_config['layers'][-1]['config'].get('activation')
) + '\n'
elif isinstance(model_config, list):
output += ui.data(
indent=6,
field='Activation',
value=model_config[-1].get('config', {}).get('activation')
) + '\n'
if display:
if is_jupyter():
from IPython.core.display import display, HTML
display(HTML(output))
else:
print(output)
else:
return output
def htmlCanvas(self) -> HTML:
htmlCode = f'''
<canvas id="{self.id}" width="{self.width}" height="{self.height}" style="border: 1px solid black; margin: 0; padding: 0;"></canvas>
'''
return HTML(htmlCode)
def tfidf_kmeans_classify_feature__fit(df,
df_name,
feat,
mean_cluster_size=None,
verbosity=1):
"""
IMPORTANT! Set mean_cluster_size only if you want to OVERRIDE the default beahvior to base KMeans n_clusters on entropy of TF-IDF doc distribution.
*** IN GENERAL, THIS IS A BAD IDEA UNLESS YOU HAVE AN EXPLICIT REASON FOR DOING SO! ***
Other notes:
nan values MUST be dealt with beforehand!
"""
df_copy = df.copy()
# fit TF-IDF to the corpus
corpus, tfidf, tfidf_vectorizer = _tfidf_fit_corpus_from_feat(
df_copy, feat)
# for display to the reader to show the evolution from DIRTY to TF-IDF "cleaned"
# add the result of the first step of preprocessing: coverting to lower-case
feat_name_stripped_lcase = f"{feat}_stripped_lcase"
df_copy[feat_name_stripped_lcase] = df_copy[feat].apply(
preprocess__lcase_strip)
# add the result of the next step of preprocessing: tokenization
feat_name_word_tokenized = f"{feat}_word_tokenized"
df_copy[feat_name_word_tokenized] = df_copy[
feat_name_stripped_lcase].apply(preprocess__tokenize)
# add the result of the next step of preprocessing: remove stop-words
feat_name_word_tokenized_no_stopwords = f"{feat}_word_tokenized_no_stopwords"
df_copy[feat_name_word_tokenized_no_stopwords] = df_copy[
feat_name_word_tokenized].apply(
lambda feat_word_tokenized: preprocess__filter_stopwords(
feat_word_tokenized, is_list=True)[0])
# do this beforehand to avoid recomputing it every time, should we pass in more than one document (installer)... which we do below
display(HTML(f"<p><br>building the idx term map..."))
idx_term_map = tfidf_vocab_to_idx_map(tfidf_vectorizer.vocabulary_)
display(HTML(f"<pre>{s_all_done}</pre>"))
feat_name_after_tfidf = f"{feat}_after_tfidf"
# now fit docs to tf-idf vectors
display(
HTML(
f"<p><br>fitting DIRTY <i>{feat}</i> documents to <code>TF-IDF</code> vectors..."
))
df_copy[feat_name_after_tfidf] = df_copy[feat].apply(
lambda _feat: doc_to_tfidf_fit(_feat, tfidf_vectorizer, idx_term_map)[
0][0])
display(HTML(f"<pre>{s_all_done}</pre>"))
if verbosity > 1:
cols_for_this_feat = [
feat, feat_name_stripped_lcase, feat_name_word_tokenized,
feat_name_word_tokenized_no_stopwords, feat_name_after_tfidf
]
display(
HTML(
f"<h3>First few rows of {df_name} TF-IDF DataFrame (verbosity>1)</h3>"
))
display(HTML(df_copy[cols_for_this_feat].head(10).to_html()))
# THIS PART IS KEY! Entropy is the basis for setting the proper cluster size and hence the proper n_clusters parameter to build the KMeans model!
dist_normalized = df_copy[feat_name_after_tfidf].value_counts(
normalize=True)
_entropy = entropy(dist_normalized, base=2)
display(
HTML(
f"<p><br>info: mean_cluster_size=={mean_cluster_size}; calculated entropy: {_entropy}"
))
if mean_cluster_size is None:
mean_cluster_size = _entropy
display(HTML(f"<p><br>set mean_cluster_size={mean_cluster_size}"))
# build KMeans model
n_clusters = int(
len(corpus) /
mean_cluster_size) # 8 is default n_clusters value for KMeans
kmeans, df_kmeans_clusters = _kmeans_from_tfidf(tfidf, idx_term_map,
n_clusters)
# clean up df_copy
df_copy = df_copy.drop([
feat_name_stripped_lcase, feat_name_word_tokenized,
feat_name_word_tokenized_no_stopwords, feat_name_after_tfidf
],
axis=1)
return df_copy, corpus, tfidf, tfidf_vectorizer, idx_term_map, kmeans, df_kmeans_clusters
def putHTML(color, msg):
source = """<font color={}>{}</font><br/>""".format(color, msg)
return HTML(source)
def show_in_notebook(self, exp, true_class=False, predict_proba_fn=None):
"""Bla"""
out = self.as_html(exp, true_class, predict_proba_fn)
from IPython.core.display import display, HTML
display(HTML(out))
def toggle_code():
display(HTML(toggle_code_str))
from IPython.core.display import display, HTML
toggle_code_str = '''
<form action="javascript:code_toggle()"><input type="submit" id="toggleButton" value="Show Sloution"></form>
'''
toggle_code_prepare_str = '''
<script>
function code_toggle() {
if ($('div.cell.code_cell.rendered.selected div.input').css('display')!='none'){
$('div.cell.code_cell.rendered.selected div.input').hide();
} else {
$('div.cell.code_cell.rendered.selected div.input').show();
}
}
</script>
'''
display(HTML(toggle_code_prepare_str + toggle_code_str))
def toggle_code():
display(HTML(toggle_code_str))
def animate_fates(adata,
basis='umap',
dims=None,
n_steps=100,
cell_states=None,
color='ntr',
fig=None,
ax=None,
logspace=False,
max_time=None,
frame_color=None,
interval=100,
blit=True,
save_show_or_return='show',
save_kwargs={},
**kwargs):
"""Animating cell fate commitment prediction via reconstructed vector field function.
This class creates necessary components to produce an animation that describes the exact speed of a set of cells
at each time point, its movement in gene expression and the long range trajectory predicted by the reconstructed
vector field. Thus it provides intuitive visual understanding of the RNA velocity, speed, acceleration, and cell
fate commitment in action.
This function is originally inspired by https://tonysyu.github.io/animating-particles-in-a-flow.html and relies on
animation module from matplotlib. Note that you may need to install `imagemagick` in order to properly show or save
the animation. See for example, http://louistiao.me/posts/notebooks/save-matplotlib-animations-as-gifs/ for more
details.
Parameters
----------
adata: :class:`~anndata.AnnData`
AnnData object that already went through the fate prediction.
basis: `str` or None (default: `None`)
The embedding data to use for predicting cell fate. If `basis` is either `umap` or `pca`, the reconstructed
trajectory will be projected back to high dimensional space via the `inverse_transform` function.
space.
dims: `list` or `None` (default: `None')
The dimensions of low embedding space where cells will be drawn and it should corresponds to the space
fate prediction take place.
n_steps: `int` (default: `100`)
The number of times steps (frames) fate prediction will take.
cell_states: `int`, `list` or `None` (default: `None`)
The number of cells state that will be randomly selected (if `int`), the indices of the cells states (if
`list`) or all cell states which fate prediction executed (if `None`)
fig: `matplotlib.figure.Figure` or None (default: `None`)
The figure that will contain both the background and animated components.
ax: `matplotlib.Axis` (optional, default `None`)
The matplotlib axes object that will be used as background plot of the vector field animation. If `ax`
is None, `topography(adata, basis=basis, color=color, ax=ax, save_show_or_return='return')` will be used
to create an axes.
logspace: `bool` (default: `False`)
Whether or to sample time points linearly on log space. If not, the sorted unique set of all time points
from all cell states' fate prediction will be used and then evenly sampled up to `n_steps` time points.
interval: `float` (default: `200`)
Delay between frames in milliseconds.
blit: `bool` (default: `False`)
Whether blitting is used to optimize drawing. Note: when using blitting, any animated artists will be drawn
according to their zorder; however, they will be drawn on top of any previous artists, regardless of their
zorder.
save_show_or_return: `str` {'save', 'show', 'return'} (default: `save`)
Whether to save, show or return the figure. By default a gif will be used.
save_kwargs: `dict` (default: `{}`)
A dictionary that will passed to the anim.save. By default it is an empty dictionary and the save_fig function
will use the {"filename": 'fate_ani.gif', "writer": "imagemagick"} as its parameters. Otherwise you can
provide a dictionary that properly modify those keys according to your needs. see
https://matplotlib.org/api/_as_gen/matplotlib.animation.Animation.save.html for more details.
kwargs:
Additional arguments passed to animation.FuncAnimation.
Returns
-------
Nothing but produce an animation that will be embedded to jupyter notebook or saved to disk.
Examples 1
----------
>>> from matplotlib import animation
>>> progenitor = adata.obs_names[adata.obs.clusters == 'cluster_1']
>>> fate_progenitor = progenitor
>>> info_genes = adata.var_names[adata.var.use_for_transition]
>>> dyn.pd.fate(adata, basis='umap', init_cells=fate_progenitor, interpolation_num=100, direction='forward',
... inverse_transform=False, average=False)
>>> dyn.mv.animate_fates(adata)
See also:: :func:`StreamFuncAnim`
"""
from matplotlib import animation
instance = StreamFuncAnim(
adata=adata,
basis=basis,
dims=dims,
n_steps=n_steps,
cell_states=cell_states,
color=color,
fig=fig,
ax=ax,
logspace=logspace,
max_time=max_time,
frame_color=frame_color,
)
anim = animation.FuncAnimation(instance.fig,
instance.update,
init_func=instance.init_background,
frames=np.arange(n_steps),
interval=interval,
blit=blit,
**kwargs)
if save_show_or_return == 'save':
save_kwargs_ = {"filename": 'fate_ani.gif', "writer": "imagemagick"}
save_kwargs_.update(save_kwargs)
anim.save(**save_kwargs_) # save as gif file.
elif save_show_or_return == 'show':
from IPython.core.display import HTML
HTML(anim.to_jshtml()) # embedding to jupyter notebook.
else:
anim
def md(str):
display(HTML(markdown.markdown(str + "<br />")))
def iplot_state_paulivec(rho, figsize=None, slider=False, show_legend=False):
""" Create a paulivec representation.
Graphical representation of the input array.
Args:
rho (array): State vector or density matrix.
figsize (tuple): Figure size in inches.
slider (bool): activate slider
show_legend (bool): show legend of graph content
"""
# HTML
html_template = Template("""
<p>
<div id="paulivec_$divNumber"></div>
</p>
""")
# JavaScript
javascript_template = Template("""
<script>
requirejs.config({
paths: {
qVisualization: "https://qvisualization.mybluemix.net/q-visualizations"
}
});
require(["qVisualization"], function(qVisualizations) {
qVisualizations.plotState("paulivec_$divNumber",
"paulivec",
$executions,
$options);
});
</script>
""")
rho = _validate_input_state(rho)
# set default figure size if none given
if figsize is None:
figsize = (7, 5)
options = {
'width': figsize[0],
'height': figsize[1],
'slider': int(slider),
'show_legend': int(show_legend)
}
# Process data and execute
div_number = str(time.time())
div_number = re.sub('[.]', '', div_number)
data_to_plot = []
rho_data = process_data(rho)
data_to_plot.append(dict(data=rho_data))
html = html_template.substitute({'divNumber': div_number})
javascript = javascript_template.substitute({
'divNumber': div_number,
'executions': data_to_plot,
'options': options
})
display(HTML(html + javascript))
def jupyter_no_margins():
"""Cause Jupyter notebook to take up 100% of window width."""
display(HTML("<style>.container { width:100% !important; }</style>"))
'people-with-disabilities-that-limit-their-day-to-day-activities2':
'people-with-disabilities-that-limit-their-day-to-day-activities',
'people-with-disabilities-that-limit-their-day-to-day-activities3':
'people-with-disabilities-that-limit-their-day-to-day-activities',
'people-with-work-limiting-disabilities1':
'people-with-work-limiting-disabilities',
'people-with-work-limiting-disabilities2':
'people-with-work-limiting-disabilities'
}
})
from IPython.core.display import HTML
for col in tidy:
if col not in ['Value']:
tidy[col] = tidy[col].astype('category')
display(HTML(f"<h2>{col}</h2>"))
display(tidy[col].cat.categories)
# In[57]:
destinationFolder = Path('out')
destinationFolder.mkdir(exist_ok=True, parents=True)
TAB_NAME = 'observations'
tidy.drop_duplicates().to_csv(destinationFolder / f'{TAB_NAME}.csv',
index=False)
scraper.dataset.family = 'disability'
with open(destinationFolder / f'{TAB_NAME}.csv-metadata.trig',
def css_styling():
styles = open("../styles/custom.css", "r").read()
return HTML(styles)
def print_dict(dictionary):
html = dict_to_html(dictionary)
display(HTML(html))
async def setup(self, author, document, db_url=None):
"""Setup cowork.
The `db_url` should be formatted as a database url like:
'driver://*****:*****@host:port/database'
For SQLite, there should be four slashes for an absolute path:
'sqlite:////absolute/path/to/db.sqlite3'
When `db_url` is None (the default), the SQLite driver will be used
with the current working directory path like:
cwd = os.getcwd()
name = f'cowork-{document}.sqlite3'
f'sqlite:///{os.path.join(cwd, name)}'
"""
# TODO: `db_url` param is too strong of a requirement. It works for
# local development with a database but the more generic solution is to
# use a server url and not run the server in the local ioloop.
if db_url is None:
cwd = os.getcwd()
name = f'cowork-{document}.sqlite3'
db_url = f'sqlite:///{os.path.join(cwd, name)}'
os.environ.setdefault('DATABASE_URL', db_url)
from django.core import management
from .wsgi import application
with ThreadPoolExecutor(1) as executor:
future = executor.submit(
management.call_command,
'migrate',
verbosity=0,
)
future.result()
self.author = author
self.document = document
container = tornado.wsgi.WSGIContainer(application)
http_server = tornado.httpserver.HTTPServer(container)
http_server.listen(0)
port = next(iter(http_server._sockets.values())).getsockname()[1]
self.port = port
if os.environ.get('SERVER_SOFTWARE', '').startswith('voila/'):
tags = """
<meta name="htmx-config" content='{"withCredentials": true}'>
<script src="https://unpkg.com/[email protected]"></script>
<script>htmx.logAll();</script>
"""
else:
tags = """
<script>
require(
["https://unpkg.com/[email protected]"],
function (htmx) {
window.htmx = htmx;
htmx.config.withCredentials = true;
}
);
</script>
"""
return HTML(tags)
def run_simulation(X, n_frames, grow_func, ignite_func, burn_func):
"""
calls the function "grow_func", "ignite_func" and "burn_func" on X n_frames times.
Overwrites the X matrice, and plots the result as animation.
Returns a list of wildfire burned area for each iteration.
"""
# Colours for visualization: brown for EMPTY, dark green for TREE and orange
# for FIRE. Note that for the colormap to work, this list and the bounds list
# must be one larger than the number of different values in the array.
colors_list = ['orange', (0, 0, 0), (0, 0.7, 0), (0, 0.5, 0), (0, 0.7, 0)]
cmap = colors.ListedColormap(colors_list)
bounds = [-1, 0, 1, 2, 3]
norm = colors.BoundaryNorm(bounds, cmap.N)
burned_list = []
fig = plt.figure(figsize=(15, 8))
ax1 = fig.add_subplot(121)
#ax.set_axis_off()
im = ax1.imshow(X, cmap=cmap, norm=norm) #, interpolation='nearest')
ax2 = fig.add_subplot(122)
n_trees1 = np.nan * np.ones((n_frames, ))
n_trees1[0] = np.sum(X == TREE1)
ax2.plot(np.arange(n_frames), n_trees1, 'r') #, interpolation='nearest')
n_trees2 = np.nan * np.ones((n_frames, ))
n_trees2[0] = np.sum(X == TREE2)
ax2.plot(np.arange(n_frames), n_trees2, 'b') #, interpolation='nearest')
ax2.set_xlabel('iteration')
ax2.set_ylabel('# of trees')
bar = PB(min_value=0, max_value=n_frames)
# The animation function: called to produce a frame for each generation.
def animate(i):
bar.update(i)
im.set_data(animate.X)
grow_func(animate.X)
ignite_func(animate.X)
burned = burn_func(animate.X)
burned_list.append(burned)
animate.n_trees1[i] = np.sum(animate.X == TREE1)
animate.n_trees2[i] = np.sum(animate.X == TREE2)
ax2.plot(np.arange(n_frames), animate.n_trees1, 'r')
ax2.plot(np.arange(n_frames), animate.n_trees2, 'b')
ax2.set_xlim(0, n_frames)
ax2.set_ylim(0, np.multiply(*animate.X.shape))
# Bind our grid to the identifier X in the animate function's namespace.
animate.X = X
animate.n_trees1 = n_trees1
animate.n_trees2 = n_trees2
anim = animation.FuncAnimation(fig, animate, frames=n_frames)
display(HTML(anim.to_jshtml(fps=10)))
return burned_list
def docstring2html(function_or_string,
format="html",
fLOG=noLOG,
writer="html",
keep_warnings=False,
directives=None,
language="en",
layout='docutils',
document_name="<<string>>",
filter_nodes=None,
**options):
"""
Converts a docstring into a :epkg:`HTML` format.
@param function_or_string function, class, method or doctring
@param format output format (``'html'`` or '``rawhtml``')
@param fLOG logging function
@param writer ``'html'`` for :epkg:`HTML` format or ``'rst'`` for RST format
@param keep_warnings keep_warnings in the final :epkg:`HTML`
@param directives new directives to add (see below)
@param language language
@param layout ``'docutils'``, ``'sphinx'``, ``'sphinx_body'``, see below.
@param document_name document_name for this string
@param filter_nodes transform the doctree before writing the results
(layout must be 'sphinx')
@param options Sphinx options see `Render math as images
<http://www.sphinx-doc.org/en/stable/ext/math.html#module-sphinx.ext.imgmath>`_,
a subset of options is used, see @see fn default_sphinx_options.
By default, the theme (option *html_theme*) will ``'basic'``.
@return (str) :epkg:`HTML` format or (IPython.core.display.HTML)
.. exref::
:title: Produce HTML documentation for a function or class
The following code can display the dosstring in :epkg:`HTML` format
to display it in a :epkg:`notebook`.
::
from pyquickhelper.helpgen import docstring2html
import sklearn.linear_model
docstring2html(sklearn.linear_model.LogisticRegression)
The output format is defined by:
* ``'html'``: IPython :epkg:`HTML` object
* ``'rawhtml'``: :epkg:`HTML` as text + style
* ``'rst'``: :epkg:`rst`
* ``'text'``: raw text
"""
if not isinstance(function_or_string, str):
doc = function_or_string.__doc__
else:
doc = function_or_string
if format == "text":
return doc
if doc is None:
return ""
javadoc = migrating_doxygen_doc(doc, "None", log=False)[1]
rows = javadoc.split("\n")
from .utils_sphinx_doc import _private_migrating_doxygen_doc
rst = _private_migrating_doxygen_doc(rows,
index_first_line=0,
filename="None")
rst = "\n".join(rst)
ded = textwrap.dedent(rst)
try:
html = rst2html(ded,
fLOG=fLOG,
writer=writer,
keep_warnings=keep_warnings,
directives=directives,
language=language,
filter_nodes=filter_nodes,
document_name=document_name,
layout=layout,
**options)
except Exception:
# we check the indentation
ded = correct_indentation(ded)
try:
html = rst2html(ded,
fLOG=fLOG,
writer=writer,
keep_warnings=keep_warnings,
directives=directives,
language=language,
filter_nodes=filter_nodes,
document_name=document_name,
layout=layout,
**options)
except Exception as e:
lines = ded.split("\n")
lines = [
"%04d %s" % (i + 1, _.strip("\n\r"))
for i, _ in enumerate(lines)
]
raise HelpGenConvertError("unable to process:\n{0}".format(
"\n".join(lines))) from e
if writer in ('doctree', 'rst'):
return html
if format == "html":
from IPython.core.display import HTML
return HTML(html)
elif format in ("rawhtml", 'rst'):
return html
else:
raise ValueError("Unexpected format: '" + format +
"', should be html, rawhtml, text, rst.")
from IPython.core.display import HTML
HTML(open('../css/custom.css', 'r').read())
# A function I'll use later to extract FPGA resource usage stats from Yosys log output.
def print_stats(yosys_log):
stat_start_line = yosys_log.grep(r'^2\.27\. ')
stat_end_line = yosys_log.grep(r'^2\.28\. ')
start_index = yosys_log.index(stat_start_line[0])
end_index = yosys_log.index(stat_end_line[0])
print('\n'.join(yosys_log[start_index + 2:end_index - 1]))
from pygmyhdl import *
@chunk
def ram(clk_i, wr_i, addr_i, data_i, data_o):
'''
Inputs:
clk_i: Data is read/written on the rising edge of this clock input.
wr_i: When high, data is written to the RAM; when low, data is read from the RAM.
addr_i: Address bus for selecting which RAM location is being read/written.
data_i: Data bus for writing data into the RAM.
Outputs:
data_o: Data bus for reading data from the RAM.
'''
mem = [Bus(len(data_i)) for _ in range(2**len(addr_i))]
@seq_logic(clk_i.posedge)
def plot_grids(dataset, model_names, out_dir=None):
if out_dir is not None:
os.mkdir(out_dir)
scores = []
for model in model_names:
scores.extend(run_experiments(dataset=dataset, model=model))
prefix = "<style type='text/css'> .shap_benchmark__select:focus { outline-width: 0 }</style>"
out = "" # background: rgb(30, 136, 229)
# out += "<div style='font-weight: regular; font-size: 24px; text-align: center; background: #f8f8f8; color: #000; padding: 20px;'>SHAP Benchmark</div>\n"
# out += "<div style='height: 1px; background: #ddd;'></div>\n"
#out += "<div style='height: 7px; background-image: linear-gradient(to right, rgb(30, 136, 229), rgb(255, 13, 87));'></div>"
out += "<div style='position: fixed; left: 0px; top: 0px; right: 0px; height: 230px; background: #fff;'>\n" # box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
out += "<div style='position: absolute; bottom: 0px; left: 0px; right: 0px;' align='center'><table style='border-width: 1px; margin-right: 100px'>\n"
for ind, model in enumerate(model_names):
row_keys, col_keys, data = make_grid(scores, dataset, model)
# print(data)
# print(colors.red_blue_solid(0.))
# print(colors.red_blue_solid(1.))
# return
for metric in col_keys:
if metric not in [
"local_accuracy", "runtime", "consistency_guarantees"
]:
plot_curve(dataset, model, metric)
buf = io.BytesIO()
pl.gcf().set_size_inches(1200.0 / 175, 800.0 / 175)
pl.savefig(buf, format='png', dpi=175)
if out_dir is not None:
pl.savefig("%s/plot_%s_%s_%s.pdf" %
(out_dir, dataset, model, metric),
format='pdf')
pl.close()
buf.seek(0)
data_uri = base64.b64encode(
buf.read()).decode('utf-8').replace('\n', '')
plot_id = "plot__" + dataset + "__" + model + "__" + metric
prefix += "<div onclick='document.getElementById(\"%s\").style.display = \"none\"' style='display: none; position: fixed; z-index: 10000; left: 0px; right: 0px; top: 0px; bottom: 0px; background: rgba(255,255,255,0.9);' id='%s'>" % (
plot_id, plot_id)
prefix += "<img width='600' height='400' style='margin-left: auto; margin-right: auto; margin-top: 230px; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);' src='data:image/png;base64,%s'>" % data_uri
prefix += "</div>"
model_title = getattr(models, dataset + "__" +
model).__doc__.split("\n")[0].strip()
if ind == 0:
out += "<tr><td style='background: #fff; width: 250px'></td></td>"
for j in range(data.shape[1]):
metric_title = getattr(
metrics, col_keys[j]).__doc__.split("\n")[0].strip()
out += "<td style='width: 40px; background: #fff; text-align: right;'><div style='margin-left: 10px; margin-bottom: -5px; white-space: nowrap; transform: rotate(-45deg); transform-origin: left top 0; width: 1.5em; margin-top: 8em'>" + metric_title + "</div></td>"
out += "</tr>\n"
out += "</table></div></div>\n"
out += "<table style='border-width: 1px; margin-right: 100px; margin-top: 230px;'>\n"
out += "<tr><td style='background: #fff'></td><td colspan='%d' style='background: #fff; font-weight: bold; text-align: center; margin-top: 10px;'>%s</td></tr>\n" % (
data.shape[1], model_title)
for i in range(data.shape[0]):
out += "<tr>"
# if i == 0:
# out += "<td rowspan='%d' style='background: #fff; text-align: center; white-space: nowrap; vertical-align: middle; '><div style='font-weight: bold; transform: rotate(-90deg); transform-origin: left top 0; width: 1.5em; margin-top: 8em'>%s</div></td>" % (data.shape[0], model_name)
method_title = getattr(methods,
row_keys[i]).__doc__.split("\n")[0].strip()
out += "<td style='background: #ffffff; text-align: right; width: 250px' title='shap.LinearExplainer(model)'>" + method_title + "</td>\n"
for j in range(data.shape[1]):
plot_id = "plot__" + dataset + "__" + model + "__" + col_keys[j]
out += "<td onclick='document.getElementById(\"%s\").style.display = \"block\"' style='padding: 0px; padding-left: 0px; padding-right: 0px; border-left: 0px solid #999; width: 42px; height: 34px; background-color: #fff'>" % plot_id
#out += "<div style='opacity: "+str(2*(max(1-data[i,j], data[i,j])-0.5))+"; background-color: rgb" + str(tuple(v*255 for v in colors.red_blue_solid(0. if data[i,j] < 0.5 else 1.)[:-1])) + "; height: "+str((30*max(1-data[i,j], data[i,j])))+"px; margin-left: auto; margin-right: auto; width:"+str((30*max(1-data[i,j], data[i,j])))+"px'></div>"
out += "<div style='opacity: " + str(
1
) + "; background-color: rgb" + str(
tuple(
int(v * 255)
for v in colors.red_blue_no_bounds(5 * (data[i, j] -
0.8))[:-1])
) + "; height: " + str(
(30 * data[i, j])
) + "px; margin-left: auto; margin-right: auto; width:" + str(
(30 * data[i, j])) + "px'></div>"
#out += "<div style='float: left; background-color: #eee; height: 10px; width: "+str((40*(1-data[i,j])))+"px'></div>"
out += "</td>\n"
out += "</tr>\n" #
out += "<tr><td colspan='%d' style='background: #fff'></td></tr>" % (
data.shape[1] + 1)
out += "</table>"
out += "<div style='position: fixed; left: 0px; top: 0px; right: 0px; text-align: left; padding: 20px; text-align: right'>\n"
out += "<div style='float: left; font-weight: regular; font-size: 24px; color: #000;'>SHAP Benchmark <span style='font-size: 14px; color: #777777;'>v" + __version__ + "</span></div>\n"
# select {
# margin: 50px;
# width: 150px;
# padding: 5px 35px 5px 5px;
# font-size: 16px;
# border: 1px solid #ccc;
# height: 34px;
# -webkit-appearance: none;
# -moz-appearance: none;
# appearance: none;
# background: url(http://www.stackoverflow.com/favicon.ico) 96% / 15% no-repeat #eee;
# }
#out += "<div style='display: inline-block; margin-right: 20px; font-weight: normal; text-decoration: none; font-size: 18px; color: #000;'>Dataset:</div>\n"
out += "<select id='shap_benchmark__select' onchange=\"document.location = '../' + this.value + '/index.html'\"dir='rtl' class='shap_benchmark__select' style='font-weight: normal; font-size: 20px; color: #000; padding: 10px; background: #fff; border: 1px solid #fff; -webkit-appearance: none; appearance: none;'>\n"
out += "<option value='corrgroups60' " + (
"selected" if dataset == "corrgroups60" else
"") + ">Correlated Groups 60 Dataset</option>\n"
out += "<option value='independentlinear60' " + (
"selected" if dataset == "independentlinear60" else
"") + ">Independent Linear 60 Dataset</option>\n"
#out += "<option>CRIC</option>\n"
out += "</select>\n"
#out += "<script> document.onload = function() { document.getElementById('shap_benchmark__select').value = '"+dataset+"'; }</script>"
#out += "<div style='display: inline-block; margin-left: 20px; font-weight: normal; text-decoration: none; font-size: 18px; color: #000;'>CRIC</div>\n"
out += "</div>\n"
# output the legend
out += "<table style='border-width: 0px; width: 100px; position: fixed; right: 50px; top: 100px;'>\n"
out += "<tr><td style='background: #fff; font-weight: normal; text-align: center'>Higher score</td></tr>\n"
legend_size = 21
for i in range(legend_size - 9):
out += "<tr>"
out += "<td style='padding: 0px; padding-left: 0px; padding-right: 0px; border-left: 0px solid #999; height: 34px; background-color: #fff'>"
val = (legend_size - i - 1) / (legend_size - 1)
out += "<div style='opacity: 1; background-color: rgb" + str(
tuple(
int(v * 255)
for v in colors.red_blue_no_bounds(5 * (val - 0.8)))[:-1]
) + "; height: " + str(
30 *
val) + "px; margin-left: auto; margin-right: auto; width:" + str(
30 * val) + "px'></div>"
out += "</td>"
out += "</tr>\n" #
out += "<tr><td style='background: #fff; font-weight: normal; text-align: center'>Lower score</td></tr>\n"
out += "</table>\n"
if out_dir is not None:
with open(out_dir + "/index.html", "w") as f:
f.write(
"<html><body style='margin: 0px; font-size: 16px; font-family: \"Myriad Pro\", Arial, sans-serif;'><center>"
)
f.write(prefix)
f.write(out)
f.write("</center></body></html>")
else:
return HTML(prefix + out)
def display(self):
self.update_rm(display_meta=False)
header = widgets.Label(
value="Loading Dashboard...",
layout=widgets.Layout(width='800px'),
)
display(header)
if self.enable_datatables:
init_datatable_mode()
tables = widgets.Output()
plots = widgets.Output()
images = widgets.Output()
share = widgets.Output()
latex = widgets.Output()
main_out = widgets.Output()
# Display tabs
tab = widgets.Tab(children=[tables, plots, images, latex, share])
tab.set_title(0, 'Tables')
tab.set_title(1, 'Plots')
tab.set_title(2, 'Images')
tab.set_title(3, 'Latex')
tab.set_title(4, 'Share')
with main_out:
display(tab)
tables.clear_output()
plots.clear_output()
images.clear_output()
latex.clear_output()
share.clear_output()
# show tabs
tables_tab(self, tables)
plots_tab(self, plots)
images_tab(self, images)
latex_tab(self, latex)
share_tab(self, share)
header.value = (
f'Dashboard loaded (ver: {haven.__version__}). ' +
f'{len(self.rm_original.exp_list_all)} experiments selected from "{self.rm_original.savedir_base}"'
)
display(main_out)
if self.wide_display:
display(
HTML("<style>.container { width:100% !important; }</style>"))
# This makes cell show full height display
style = """
<style>
.output_scroll {
height: unset !important;
border-radius: unset !important;
-webkit-box-shadow: unset !important;
box-shadow: unset !important;
}
</style>
"""
display(HTML(style))
def show_video(url):
data = f'<iframe width="560" height="315" src="{url}" frameborder="0" allowfullscreen></iframe>'
return display(HTML(data))
def _repr_javascript_(self):
from IPython.core.display import display, HTML
vega_spec = self._get_vega(True)
vega_html = '<html lang="en"> \
<head> \
<script src="https://cdnjs.cloudflare.com/ajax/libs/vega/3.0.8/vega.js"></script> \
<script src="https://cdnjs.cloudflare.com/ajax/libs/vega-embed/3.0.0-rc7/vega-embed.js"></script> \
<script src="https://cdnjs.cloudflare.com/ajax/libs/vega-tooltip/0.5.1/vega-tooltip.min.js"></script> \
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/vega-tooltip/0.5.1/vega-tooltip.min.css"> \
<style> \
.vega-actions > a{ \
color:white; \
text-decoration: none; \
font-family: "Arial"; \
cursor:pointer; \
padding:5px; \
background:#AAAAAA; \
border-radius:4px; \
padding-left:10px; \
padding-right:10px; \
margin-right:5px; \
} \
.vega-actions{ \
margin-top:20px; \
text-align:center \
}\
.vega-actions > a{ \
background:#999999;\
} \
</style> \
</head> \
<body> \
<div id="vis"> \
</div> \
<script> \
var vega_json = '+_json.dumps(_json.dumps(vega_spec)).replace("'", "'")+'; \
var vega_json_parsed = JSON.parse(vega_json); \
var toolTipOpts = { \
showAllFields: true \
}; \
if(vega_json_parsed["metadata"] != null){ \
if(vega_json_parsed["metadata"]["bubbleOpts"] != null){ \
toolTipOpts = vega_json_parsed["metadata"]["bubbleOpts"]; \
}; \
}; \
vegaEmbed("#vis", vega_json_parsed).then(function (result) { \
vegaTooltip.vega(result.view, toolTipOpts); \
}); \
</script> \
</body> \
</html>'
display(HTML('<html> \
<body> \
<iframe style="border:0;margin:0" width="'+str((vega_spec["width"] if "width" in vega_spec else 600)+200)+'" height="'+str(vega_spec["height"]+220)+'" srcdoc='+"'"+vega_html+"'"+' src="demo_iframe_srcdoc.htm"> \
<p>Your browser does not support iframes.</p> \
</iframe> \
</body> \
</html>'));
def in_show_video(name, vext=vext, loop=True, autoplay=True, controls=True, embed=False, figpath=figpath, **kwargs):
"""
Columns represent isometric projections of a cube. The left column displays
iso-surfaces of the spectral envelope by displaying enclosing volumes at 5
different energy values with respect to the peak amplitude of the Fourier spectrum.
The middle column shows an isometric view of the faces of the movie cube.
The first frame of the movie lies on the x-y plane, the x-t plane lies on the
top face and motion direction is seen as diagonal lines on this face (vertical
motion is similarly see in the y-t face). The third column displays the actual
movie as an animation.
Given a name, displays the figures corresponding to the Fourier spectra, the
stimulus cubes and movies within the notebook.
"""
import os
from IPython.core.display import display, Image, HTML
from base64 import b64encode
opts = ' '
if loop: opts += 'loop="1" '
if autoplay: opts += 'autoplay="1" '
if controls: opts += 'controls '
if embed:
try:
with open(os.path.join(figpath, name + ext), "rb") as image_file:
im1 = b64encode(image_file.read()).decode("utf-8")
with open(os.path.join(figpath, name + '_cube' + ext), "rb") as image_file:
im2 = b64encode(image_file.read()).decode("utf-8")
with open(os.path.join(figpath, name + vext), "rb") as video_file:
im3 = b64encode(video_file.read()).decode("utf-8")
s = """
<center><table border=none width=100% height=100%>
<tr>
<td width=33%%><center><img src="data:image/png;base64,{0}" width=100%/></td>
<td rowspan=2 colspan=2><center><video src="data:video/webm;base64,{1}" {2} type="video/{3}" width=100%/></td>
</tr>
<tr>
<td><center><img src="data:image/png;base64,{4}" width=100%/></td>
</tr>
</table></center>""".format(im1, im3, opts, vext[1:], im2)
# display(HTML(s))
except:
video = open(os.path.join(figpath, name + vext), "rb").read()
video_encoded = b64encode(video).decode("utf-8")
s = """
<center><table border=none width=100% height=100%>
<tr> <td width=100%><center><video {0} src="data:video/{1};base64,{2}" width=100%\>
</td></tr></table></center>""".format(opts, vext[1:], video_encoded)
# display(HTML(s))
else:
if os.path.isfile(os.path.join(figpath, name + ext)) and os.path.isfile(os.path.join(figpath, name + '_cube' + ext)):
if os.path.isfile(os.path.join(figpath, name + vext)):
s = """
<center><table border=none width=100% height=100%>
<tr>
<td width=33%%><center><img src="{0}" width=100%/></td>
<td rowspan=2 colspan=2><center><video src="{1}" {2} type="video/{3}" width=100%/></td>
</tr>
<tr>
<td><center><img src="{4}" width=100%/></td>
</tr>
</table></center>""".format(os.path.join(figpath, name + ext),
os.path.join(figpath, name + vext),
opts, vext[1:],
os.path.join(figpath, name + '_cube' + ext))
else:
s = """
<center><table border=none width=100% height=100%>
<tr>
<td width=50%%><center><img src="{0}" width=100%/></td>
<td><center><img src="{1}" width=100%/></td>
</tr>
</table></center>""".format(os.path.join(figpath, name + ext),
os.path.join(figpath, name + '_cube' + ext))
else:
s = """
<center><table border=none width=100% height=100%>
<tr> <td width=100%><center><video {0} src="{2}" type="video/{1}" width=100%\>
</td></tr></table></center>""".format(opts, vext[1:], os.path.join(figpath, name + vext))
html = HTML(s)
html.reload()
display(html)
# To import urllib in a Python 2 and 3 compatible way
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
from IPython.core.display import display, HTML
# OneDork
url_style = "https://raw.githubusercontent.com/dunovank/jupyter-themes/master/jupyterthemes/styles/compiled/onedork.css"
# Download CSS
style = urlopen(url_style).read()
# Same for override CSS
url_override = "https://raw.githubusercontent.com/tbmc/simple_dark_theme_jupyter_notebook/master/override_style.css"
style_override = urlopen(url_override).read()
# Apply CSS
display(
HTML("""
<style>
%s
%s
</style>
""" % (style, style_override)))
