def show(tokens_a, tokens_b, query_vectors, key_vectors, attn):
"""Displays attention visualization"""
attention_details = _get_attention_details(tokens_a, tokens_b, query_vectors, key_vectors, attn)
att_json = json.dumps(attention_details)
display.display(display.HTML(vis_html))
display.display(display.Javascript('window.attention = %s' % att_json))
display.display(display.Javascript(vis_js))
def show(text, attn):
"""Displays attention visualization"""
attention_details = _get_attention(text, attn)
att_json = json.dumps(attention_details)
display.display(display.HTML(vis_html))
display.display(display.Javascript('window.attention = %s' % att_json))
display.display(display.Javascript(vis_js))
def visualize_sprites(embedding,
atlas,
sprite_size_in_3D=None,
initial_camera_z=60.0):
""" Plot sprites from the atlas in 3D using embedding as coordinates
Args:
embedding ([list of tuples (x, y, z)]): The order has to follow the order
of sprites in atlas (left to right first, then top to bottom)
atlas ([dict]): dict with the following items:
{
"path": path to PNG file of atlas. Should be relative to /usr/local/share/jupyter
See https://stackoverflow.com/a/49487396/13344574
"shape": {"rows": num_rows_in_atlas, "cols": num_cols_in_atlas},
"num_sprites": number of sprites within the atlas,
"sprite_size": {"height": height of single sprite,
"width": width of single sprite}
}
sprite_size_in_3D and initial_camera_z are calibration parameters
for better visualization.
sprite_size_in_3D ([dict]): {"height", expected height of single sprite in 3D,
"width", expected width of single sprite in 3D}
initial_camera_z ([float]): initial z position of camera
"""
if sprite_size_in_3D is None:
sprite_size_in_3D = {'height': 4.0, "width": 4.0}
data = {
"embedding": embedding,
"atlas": atlas,
"sprite_size_in_3D": sprite_size_in_3D,
"initial_camera_z": initial_camera_z
}
dlight.load_js_libs()
utils_dir = osp.dirname(osp.realpath(__file__))
ipd.display(
ipd.Javascript(
filename=osp.join(utils_dir, "js", "sprite_visualizer.js")))
ipd.display(
ipd.HTML(
filename=osp.join(utils_dir, "js", "sprite_visualizer.css.html")))
container_id = "sprite-visualizer-container-" + str(randrange(1000))
ipd.display(ipd.HTML("<div id='{}'></div> ".format(container_id)))
ipd.display(
ipd.Javascript("""
require(['sprite_visualizer'], function(sprite_visualizer) {{
sprite_visualizer(document.getElementById("{}"), {});
}});
""".format(container_id, json.dumps(data))))
def display_token(self, viz_id, token_id, position):
raw_token = self.tokenizer.convert_ids_to_tokens([token_id])[0]
clean_token = self.tokenizer.decode(token_id)
# Strip prefixes because bert decode still has ## for partials even after decode()
clean_token = strip_tokenizer_prefix(self.model_config, clean_token)
token = {
# 'token': self.tokenizer.decode([token_id]),
'token': clean_token,
'is_partial': is_partial_token(self.model_config, raw_token),
'token_id': int(token_id),
'position': position,
'type': 'output'
}
js = f"""
// We don't really need these require scripts. But this is to avert
//this code from running before display_input_sequence which DOES require external files
requirejs(['basic', 'ecco'], function(basic, ecco){{
console.log('addToken viz_id', '{viz_id}');
window.ecco['{viz_id}'].addToken({json.dumps(token)})
window.ecco['{viz_id}'].redraw()
}})
"""
# print(js)
d.display(d.Javascript(js))
def serve_kernel_port_as_window(port, path='/', anchor_text=None):
"""Displays a link in the output to open a browser tab to a port on the kernel.
This allows viewing URLs hosted on the kernel in new browser tabs.
The URL will only be valid for the current user while the notebook is open in
Colab.
Args:
port: The kernel port to be exposed to the client.
path: The path to be navigated to.
anchor_text: Text content of the anchor link.
"""
if not anchor_text:
anchor_text = 'https://localhost:{port}{path}'.format(port=port,
path=path)
code = """(async (port, path, text, element) => {
if (!google.colab.kernel.accessAllowed) {
return;
}
element.appendChild(document.createTextNode(''));
const url = await google.colab.kernel.proxyPort(port);
const anchor = document.createElement('a');
anchor.href = url + path;
anchor.target = '_blank';
anchor.setAttribute('data-href', url + path);
anchor.textContent = text;
element.appendChild(anchor);
})""" + '({port}, {path}, {text}, window.element)'.format(
port=port, path=json.dumps(path), text=json.dumps(anchor_text))
display.display(display.Javascript(code))
def predict_token(self, inputs, topk=50, temperature=1.0):
output = self.model(**inputs)
scores = output[0][0][-1] / temperature
s = scores.detach().numpy()
sorted_predictions = s.argsort()[::-1]
sm = F.softmax(scores, dim=-1).detach().numpy()
tokens = [self.tokenizer.decode([t]) for t in sorted_predictions[:topk]]
probs = sm[sorted_predictions[:topk]]
prediction_data = []
for idx, (token, prob) in enumerate(zip(tokens, probs)):
# print(idx, token, prob)
prediction_data.append({'token': token,
'prob': str(prob),
'ranking': idx + 1,
'token_id': str(sorted_predictions[idx])
})
params = prediction_data
viz_id = 'viz_{}'.format(round(random.random() * 1000000))
d.display(d.HTML(filename=os.path.join(self._path, "html", "predict_token.html")))
js = """
requirejs(['predict_token'], function(predict_token){{
if (window.predict === undefined)
window.predict = {{}}
window.predict["{}"] = new predict_token.predictToken("{}", {})
}}
)
""".format(viz_id, viz_id, json.dumps(params))
d.display(d.Javascript(js))
def explorable(self, printJson: Optional[bool] = False):
tokens = []
for idx, token in enumerate(self.tokens[0]):
type = "input" if idx < self.n_input_tokens else 'output'
tokens.append({
'token': token,
'token_id': int(self.token_ids[0][idx]),
'type': type
})
data = {'tokens': tokens}
d.display(
d.HTML(filename=os.path.join(self._path, "html", "setup.html")))
js = f"""
requirejs(['basic', 'ecco'], function(basic, ecco){{
const viz_id = basic.init()
ecco.renderOutputSequence({{
parentDiv: viz_id,
data: {data},
tokenization_config: {json.dumps(self.config['tokenizer_config'])}
}})
}}, function (err) {{
console.log(err);
}})"""
d.display(d.Javascript(js))
if printJson:
print(data)
def record_audio(seconds: int = 3, normalize_db: float = 0.1):
# Use Javascript to record audio.
record_js_code = """
const sleep = time => new Promise(resolve => setTimeout(resolve, time))
const b2text = blob => new Promise(resolve => {
const reader = new FileReader()
reader.onloadend = e => resolve(e.srcElement.result)
reader.readAsDataURL(blob)
})
var record = time => new Promise(async resolve => {
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
recorder = new MediaRecorder(stream)
chunks = []
recorder.ondataavailable = e => chunks.push(e.data)
recorder.start()
await sleep(time)
recorder.onstop = async ()=>{
blob = new Blob(chunks)
text = await b2text(blob)
resolve(text)
}
recorder.stop()
})
"""
print('Starting recording for {} seconds...'.format(seconds))
_display.display(_display.Javascript(record_js_code))
audio_string = output.eval_js('record(%d)' % (seconds * 1000.0))
print('Finished recording!')
audio_bytes = base64.b64decode(audio_string.split(',')[1])
return audio_bytes_to_np(audio_bytes, normalize_db=normalize_db)
def _display_colab(port, height):
"""Display the LIT UI in colab.
Args:
port: The port the LIT server is running on.
height: The height of the LIT UI in pixels.
"""
shell = """
(async () => {
const url = new URL(
await google.colab.kernel.proxyPort(%PORT%, {'cache': true}));
const iframe = document.createElement('iframe');
iframe.src = url;
iframe.setAttribute('width', '100%');
iframe.setAttribute('height', '%HEIGHT%px');
iframe.setAttribute('frameborder', 0);
document.body.appendChild(iframe);
})();
"""
replacements = [
('%PORT%', '%d' % port),
('%HEIGHT%', '%d' % height),
]
for (k, v) in replacements:
shell = shell.replace(k, v)
script = display.Javascript(shell)
display.display(script)
def test_API_promise_to_have():
view = nv.demo()
# Structure
structure = nv.Structure()
structure.get_structure_string
nt.assert_true(hasattr(structure, 'id'))
nt.assert_true(hasattr(structure, 'ext'))
nt.assert_true(hasattr(structure, 'params'))
# Widget
nv.NGLWidget._set_coordinates
nv.NGLWidget._set_initial_structure
nv.NGLWidget.add_component
nv.NGLWidget.add_trajectory
nv.NGLWidget.coordinates_dict
nv.NGLWidget.set_representations
nv.NGLWidget.clear
nv.NGLWidget.center
nv._get_notebook_info()
# display
display.Javascript(jsutils.js_clean_error_output)
display.display(view.player.repr_widget)
view.player._display()
def display_input_sequence(self, input_ids):
tokens = []
for idx, token_id in enumerate(input_ids):
type = "input"
tokens.append({'token': self.tokenizer.decode([token_id]),
'position': idx,
'token_id': int(token_id),
'type': type})
data = {'tokens': tokens}
d.display(d.HTML(filename=os.path.join(self._path, "html", "setup.html")))
d.display(d.HTML(filename=os.path.join(self._path, "html", "basic.html")))
viz_id = f'viz_{round(random.random() * 1000000)}'
# html = f"""
# <div id='{viz_id}_output'></div>
# <script>
# """
js = f"""
requirejs( ['basic', 'ecco'], function(basic, ecco){{
basic.init('{viz_id}')
window.ecco['{viz_id}'] = ecco.renderOutputSequence('{viz_id}', {data})
}}, function (err) {{
console.log(err);
}})
"""
# print(js)
# d.display(d.HTML(html))
d.display(d.Javascript(js))
return viz_id
def explorable(self, printJson: Optional[bool] = False):
tokens = []
for idx, token in enumerate(self.tokens):
type = "input" if idx < self.n_input_tokens else 'output'
tokens.append({
'token': token,
'token_id': int(self.token_ids[idx]),
'type': type
})
data = {'tokens': tokens}
d.display(
d.HTML(filename=os.path.join(self._path, "html", "setup.html")))
d.display(
d.HTML(filename=os.path.join(self._path, "html", "basic.html")))
viz_id = 'viz_{}'.format(round(random.random() * 1000000))
js = """
requirejs(['basic', 'ecco'], function(basic, ecco){{
const viz_id = basic.init()
ecco.renderOutputSequence(viz_id, {})
}}, function (err) {{
console.log(err);
}})""".format(data)
d.display(d.Javascript(js))
if printJson:
print(data)
def wavejson_to_wavedrom(wavejson, width=None):
'''
Create WaveDrom display from WaveJSON data.
This code is from https://github.com/witchard/ipython-wavedrom.
'''
# Set the width of the waveform display.
style = ''
if width != None:
style = ' style="width: {w}px"'.format(w=str(int(width)))
# Generate the HTML from the JSON.
htmldata = '<div{style}><script type="WaveDrom">{json}</script></div>'.format(
style=style, json=json.dumps(wavejson))
DISP.display_html(DISP.HTML(htmldata))
# Trigger the WaveDrom Javascript that creates the graphical display.
DISP.display_javascript(
DISP.Javascript(
data='WaveDrom.ProcessAll();',
lib=[
'http://wavedrom.com/wavedrom.min.js',
'http://wavedrom.com/skins/default.js'
]))
# The following allows the display of WaveDROM in the HTML files generated by nbconvert.
# It's disabled because it makes Github's nbconvert freak out.
setup = '''
def show(self):
figPanelTemp = _templateEnv.get_template('html/figure.html')
figPanel = figPanelTemp.render(id=self.__ID)
HTML = display.HTML(data=figPanel)
display.display(HTML)
# JavaScript code
JScode = self.render()
# NOTE: r69 in Google CDN does not work creating Three.Points objects.
# No idea of why.
libs = [
'https://ajax.googleapis.com/ajax/libs/threejs/r75/three.min.js',
# 'http://threejs.org/build/three.min.js',
'http://threejs.org/examples/js/controls/OrbitControls.js',
# './js/OrbitControls.js',
# './js/three.js',
'./js/numjis_bundle.js',
'./js/sciwis_bundle.js'
]
JS = display.Javascript(data=JScode, lib=libs)
display.display(JS)
def record(duration=3):
print("recording ... ", end = " ")
display.display(display.Javascript(RECORD))
s = output.eval_js(f'record({duration*1000})')
print(" finished!")
b = b64decode(s.split(',')[1])
audio = AudioSegment.from_file(BytesIO(b))
return audio
def display_structure(topo_aug, pos, size=(600, 600)):
import IPython.display as disp
id_string = uuid.uuid4()
return disp.Javascript(
lib='/files/js/protein-viewer.js',
data='render_structure(element, "%s", %i, %i, %s, %s);' %
(id_string, size[0], size[1], topo_aug[0],
topo_aug[1](pos))), id_string
def attention(self, attention_values=None, layer=0, **kwargs):
position = self.n_input_tokens
# importance_id = position - self.n_input_tokens
importance_id = self.n_input_tokens - 1 # Sete first values to first output token
tokens = []
if attention_values:
attn = attention_values
else:
attn = self.attention_values[layer]
# normalize attention heads
attn = attn.sum(axis=1) / attn.shape[1]
for idx, token in enumerate(self.tokens):
# print(idx, attn.shape)
type = "input" if idx < self.n_input_tokens else 'output'
if idx < len(attn[0][importance_id]):
attention_value = attn[0][importance_id][idx].cpu().detach(
).numpy()
else:
attention_value = 0
tokens.append({
'token': token,
'token_id': int(self.token_ids[idx]),
'type': type,
'value':
str(attention_value), # because json complains of floats
'position': idx
})
data = {
'tokens':
tokens,
'attributions':
[att.tolist() for att in attn[0].cpu().detach().numpy()]
}
d.display(
d.HTML(filename=os.path.join(self._path, "html", "setup.html")))
d.display(
d.HTML(filename=os.path.join(self._path, "html", "basic.html")))
viz_id = 'viz_{}'.format(round(random.random() * 1000000))
js = """
requirejs(['basic', 'ecco'], function(basic, ecco){{
const viz_id = basic.init()
ecco.interactiveTokens(viz_id, {})
}}, function (err) {{
console.log(err);
}})""".format(data)
d.display(d.Javascript(js))
if 'printJson' in kwargs and kwargs['printJson']:
print(data)
def draw_synthetic_input(input_height, input_width, callback):
""" Draw synthetic input and call the provided callback
with the image array when image is changed.
Args:
input_height ([int]): height of the input image
input_width ([int]): width of the input image
callback ([func]): callback to call when drawn image is changed.
The only argument to the callback will be a 2D image_array
with grayscale values of type int within [0, 255]
"""
def draw_synthetic_input_callback(x):
x = json.loads(x) # parse json from string
callback(x)
return ipd.JSON({'result': "parsed synthetic input"})
# JS to PY communication.
# See https://colab.research.google.com/notebooks/snippets/advanced_outputs.ipynb#scrollTo=Ytn7tY-C9U0T
output_callback_name = "draw_synthetic_input_callback_" + str(
randrange(1000))
output.register_callback('notebook.' + output_callback_name,
draw_synthetic_input_callback)
dlight.load_js_libs()
utils_dir = osp.dirname(osp.realpath(__file__))
ipd.display(
ipd.Javascript(filename=osp.join(utils_dir, "js", "draw_image.js")))
ipd.display(
ipd.HTML(filename=osp.join(utils_dir, "js", "draw_image.css.html")))
data = {
"image_height": input_height,
"image_width": input_width,
"callback_name": output_callback_name
}
container_id = "draw-image-container-" + str(randrange(1000))
ipd.display(ipd.HTML("<div id='{}'></div> ".format(container_id)))
ipd.display(
ipd.Javascript("""
require(['draw_image'], function(draw_image) {{
draw_image(document.getElementById("{}"), {});
}});
""".format(container_id, json.dumps(data))))
def _history_with_cells_as_json(self):
"""Utility accessor to allow frontends an expression to fetch history.
Returns:
A Javascript display object with the execution history.
"""
# To be able to access the raw string as an expression we need to transfer
# the plain string rather than the quoted string representation. The
# Javascript disiplay wrapper is used for that.
return display.Javascript(json.dumps(self._input_hist_cells))
def __init__(self, images, labels=None, port=8889, **kwargs):
"""
Args:
images: a list of images
labels: a list of labels for each image
port: port number for connection
"""
self.id = uuid4().hex
self.addr = "ws://localhost:" + str(port)
# start application
if ImageViewer.app is None:
ImageViewer.app = Application(port=port)
ImageViewer.app.listen(port)
# add addr to window
display.display(
display.Javascript('window.addr = "{}"'.format(self.addr)))
# add id to window
display.display(display.Javascript('window.id = "{}"'.format(self.id)))
# create client
self.client = Client(self.addr, images, labels, **kwargs)
def record(seconds=1):
display(ipd.Javascript(RECORD))
print(f"Recording started for {seconds} seconds.")
s = colab_output.eval_js("record(%d)" % (seconds * 1000))
print("Recording ended.")
b = b64decode(s.split(",")[1])
fileformat = "wav"
filename = f"_audio.{fileformat}"
AudioSegment.from_file(BytesIO(b)).export(filename, format=fileformat)
return torchaudio.load(filename)
def ensure_setup_widget():
global is_setup
if is_setup:
return
with open(file_path.parent / 'yyz_widgets.js', encoding='utf8') as f:
js = display.Javascript(f.read())
display.display(js)
is_setup = True
def _display_colab(port, height, open_in_new_tab, ui_params: RenderConfig):
"""Display the LIT UI in colab.
Args:
port: The port the LIT server is running on.
height: The height of the LIT UI in pixels.
open_in_new_tab: Whether to show the UI in a new tab instead of in the
output cell.
ui_params: RenderConfig of options for the LIT UI.
"""
params = ui_params.get_query_str()
if open_in_new_tab:
shell = """
(async () => {
const proxyPort = await google.colab.kernel.proxyPort(
%PORT%, {'cache': true})
const url = new URL(proxyPort + '%PARAMS%')
const a = document.createElement('a');
a.href = "javascript:void(0);"
a.onclick = (e) => window.open(url, "_blank");
a.innerHTML = url;
document.body.appendChild(a);
window.open(url, "_blank");
})();
"""
else:
shell = """
(async () => {
const proxyPort = await google.colab.kernel.proxyPort(
%PORT%, {'cache': true})
const url = new URL(proxyPort + '%PARAMS%')
const iframe = document.createElement('iframe');
iframe.src = url;
iframe.setAttribute('width', '100%');
iframe.setAttribute('height', '%HEIGHT%px');
iframe.setAttribute('frameborder', 0);
document.body.appendChild(iframe);
})();
"""
replacements = [
('%PORT%', '%d' % port),
('%HEIGHT%', '%d' % height),
('%PARAMS%', '%s' % params),
]
for (k, v) in replacements:
shell = shell.replace(k, v)
script = display.Javascript(shell)
display.display(script)
def _run_js(js, bindings={}, show=True, debug=False):
js_fname = pkg_resources.resource_filename(
"jm", os.path.join("js", "{0}.js".format(js)))
with open(js_fname, "r") as f:
content = f.read()
content = pystache.render(content, bindings)
if debug:
print(content)
content = ipd.Javascript(content)
if show:
display(content)
return
return content
def load_js_libs():
# to inspect require config: require.s.contexts._.config
# ipd.display(ipd.Javascript("""
# require.config({
# baseUrl: "/nbextensions/dlight/lib", // see https://stackoverflow.com/a/49487396/13344574
# paths: {
# d3: 'd3.v5.7.min',
# THREE: 'three.min',
# }
# });
# """))
ipd.display(
ipd.Javascript("""
require.config({
paths: {
d3: 'https://d3js.org/d3.v5.min',
THREE: 'https://cdnjs.cloudflare.com/ajax/libs/three.js/88/three.min',
}
});
"""))
dlight_dir = osp.dirname(osp.realpath(__file__))
ipd.display(
ipd.Javascript(filename=osp.join(dlight_dir, "lib",
"three_trackball_controls.js")))
ipd.display(
ipd.Javascript("""
require(['THREE', 'trackballLoader'], function(THREE, trackballLoader) {
if (!THREE.hasOwnProperty("TrackballControls")) {
console.log("Loading trackball controls for THREE");
trackballLoader(THREE);
}
});
"""))
ipd.display(ipd.HTML(dlight.utils.css_style.global_style))
def test_displayobject_repr():
h = display.HTML("<br />")
assert repr(h) == "<IPython.core.display.HTML object>"
h._show_mem_addr = True
assert repr(h) == object.__repr__(h)
h._show_mem_addr = False
assert repr(h) == "<IPython.core.display.HTML object>"
j = display.Javascript("")
assert repr(j) == "<IPython.core.display.Javascript object>"
j._show_mem_addr = True
assert repr(j) == object.__repr__(j)
j._show_mem_addr = False
assert repr(j) == "<IPython.core.display.Javascript object>"
def _executed_cells_as_json(self):
"""Utility accessor to allow frontends an expression to fetch executed cels.
Returns:
A Javascript display object of a dict of the executed cell IDs to their
execution index.
"""
cells = dict()
for i, cell in enumerate(self._input_hist_cells):
cells[cell['cell_id']] = i
# To be able to access the raw string as an expression we need to transfer
# the plain string rather than the quoted string representation. The
# Javascript display wrapper is used for that.
return display.Javascript(json.dumps(cells))
def test_displayobject_repr():
h = display.HTML('<br />')
nt.assert_equal(repr(h), '<IPython.core.display.HTML object>')
h._show_mem_addr = True
nt.assert_equal(repr(h), object.__repr__(h))
h._show_mem_addr = False
nt.assert_equal(repr(h), '<IPython.core.display.HTML object>')
j = display.Javascript('')
nt.assert_equal(repr(j), '<IPython.core.display.Javascript object>')
j._show_mem_addr = True
nt.assert_equal(repr(j), object.__repr__(j))
j._show_mem_addr = False
nt.assert_equal(repr(j), '<IPython.core.display.Javascript object>')
def _show_elfinder_jupyter(url="/elfinder", height=600, width="100%"):
from IPython import display
code = """(async (url, width, height, element) => {
element.appendChild(document.createTextNode(''));
const iframe = document.createElement('iframe');
iframe.src = url;
iframe.height = height;
iframe.width = width;
iframe.style.border = 0;
element.appendChild(iframe);
})""" + "({url}, {width}, {height}, element[0])".format(
url=json.dumps(url),
width=json.dumps(width),
height=json.dumps(height))
display.display(display.Javascript(code))
def javascript(content=None, url=None, script_id=None):
"""Publishes javascript content into the output."""
if (content is None) == (url is None):
raise ValueError('exactly one of content and url should be none')
if url is not None:
# Note: display.javascript will try to download script from python
# which is very rarely useful.
html('<script src=%r></script>' % url)
return
if not script_id and 'sourceURL=' not in content:
script_id = 'js_' + hashlib.md5(
content.encode('utf8')).hexdigest()[:10]
if script_id:
content += '\n//# sourceURL=%s' % script_id
display.display(display.Javascript(content))
