def explorable(self, printJson: Optional[bool] = False): tokens = [] for idx, token in enumerate(self.tokens): type = "input" if idx < self.n_input_tokens else 'output' tokens.append({ 'token': token, 'token_id': int(self.token_ids[idx]), 'type': type }) data = {'tokens': tokens} d.display( d.HTML(filename=os.path.join(self._path, "html", "setup.html"))) d.display( d.HTML(filename=os.path.join(self._path, "html", "basic.html"))) viz_id = 'viz_{}'.format(round(random.random() * 1000000)) js = """ requirejs(['basic', 'ecco'], function(basic, ecco){{ const viz_id = basic.init() ecco.renderOutputSequence(viz_id, {}) }}, function (err) {{ console.log(err); }})""".format(data) d.display(d.Javascript(js)) if printJson: print(data)
def display_input_sequence(self, input_ids): tokens = [] for idx, token_id in enumerate(input_ids): type = "input" tokens.append({'token': self.tokenizer.decode([token_id]), 'position': idx, 'token_id': int(token_id), 'type': type}) data = {'tokens': tokens} d.display(d.HTML(filename=os.path.join(self._path, "html", "setup.html"))) d.display(d.HTML(filename=os.path.join(self._path, "html", "basic.html"))) viz_id = f'viz_{round(random.random() * 1000000)}' # html = f""" # <div id='{viz_id}_output'></div> # <script> # """ js = f""" requirejs( ['basic', 'ecco'], function(basic, ecco){{ basic.init('{viz_id}') window.ecco['{viz_id}'] = ecco.renderOutputSequence('{viz_id}', {data}) }}, function (err) {{ console.log(err); }}) """ # print(js) # d.display(d.HTML(html)) d.display(d.Javascript(js)) return viz_id
def __init__(self, config_builder, height=1000): """Constructor for colab notebook WitWidget. Args: config_builder: WitConfigBuilder object containing settings for WIT. height: Optional height in pixels for WIT to occupy. Defaults to 1000. """ self._ctor_complete = False self.id = WitWidget.index base.WitWidgetBase.__init__(self, config_builder) # Add this instance to the static instance list. WitWidget.widgets.append(self) # Display WIT Polymer element. display.display(display.HTML(self._get_element_html())) display.display( display.HTML(WIT_HTML.format(height=height, id=self.id))) # Increment the static instance WitWidget index counter WitWidget.index += 1 # Send the provided config and examples to JS output.eval_js("""configCallback('{config}')""".format( config=json.dumps(self.config))) output.eval_js("""updateExamplesCallback({examples})""".format( examples=json.dumps(self.examples))) self._generate_sprite() self._ctor_complete = True
def attention(self, attention_values=None, layer=0, **kwargs): position = self.n_input_tokens # importance_id = position - self.n_input_tokens importance_id = self.n_input_tokens - 1 # Sete first values to first output token tokens = [] if attention_values: attn = attention_values else: attn = self.attention_values[layer] # normalize attention heads attn = attn.sum(axis=1) / attn.shape[1] for idx, token in enumerate(self.tokens): # print(idx, attn.shape) type = "input" if idx < self.n_input_tokens else 'output' if idx < len(attn[0][importance_id]): attention_value = attn[0][importance_id][idx].cpu().detach( ).numpy() else: attention_value = 0 tokens.append({ 'token': token, 'token_id': int(self.token_ids[idx]), 'type': type, 'value': str(attention_value), # because json complains of floats 'position': idx }) data = { 'tokens': tokens, 'attributions': [att.tolist() for att in attn[0].cpu().detach().numpy()] } d.display( d.HTML(filename=os.path.join(self._path, "html", "setup.html"))) d.display( d.HTML(filename=os.path.join(self._path, "html", "basic.html"))) viz_id = 'viz_{}'.format(round(random.random() * 1000000)) js = """ requirejs(['basic', 'ecco'], function(basic, ecco){{ const viz_id = basic.init() ecco.interactiveTokens(viz_id, {}) }}, function (err) {{ console.log(err); }})""".format(data) d.display(d.Javascript(js)) if 'printJson' in kwargs and kwargs['printJson']: print(data)
def display(self): self.html_code = html_progress_bar(self.value, self.total, self.prefix, self.label, self.width) if self.parent is not None: # If this is a child bar, the parent will take care of the display. self.parent.display() return if self.output is None: self.output = disp.display(disp.HTML(self.html_code), display_id=True) else: self.output.update(disp.HTML(self.html_code))
def display(self): self.html_code = html_progress_bar(self.value, self.total, self.prefix, self.label, self.width) if self.inner_table is not None: self.html_code += text_to_html_table(self.inner_table) if self.child_bar is not None: self.html_code += self.child_bar.html_code if self.output is None: self.output = disp.display(disp.HTML(self.html_code), display_id=True) else: self.output.update(disp.HTML(self.html_code))
def __init__(self, config_builder, height=1000): """Constructor for colab notebook WitWidget. Args: config_builder: WitConfigBuilder object containing settings for WIT. height: Optional height in pixels for WIT to occupy. Defaults to 1000. """ tf.logging.set_verbosity(tf.logging.WARN) config = config_builder.build() copied_config = dict(config) self.estimator_and_spec = ( dict(config.get('estimator_and_spec')) if 'estimator_and_spec' in config else {}) self.compare_estimator_and_spec = ( dict(config.get('compare_estimator_and_spec')) if 'compare_estimator_and_spec' in config else {}) if 'estimator_and_spec' in copied_config: del copied_config['estimator_and_spec'] if 'compare_estimator_and_spec' in copied_config: del copied_config['compare_estimator_and_spec'] self.custom_predict_fn = ( config.get('custom_predict_fn') if 'custom_predict_fn' in config else None) self.compare_custom_predict_fn = ( config.get('compare_custom_predict_fn') if 'compare_custom_predict_fn' in config else None) if 'custom_predict_fn' in copied_config: del copied_config['custom_predict_fn'] if 'compare_custom_predict_fn' in copied_config: del copied_config['compare_custom_predict_fn'] self._set_examples(config['examples']) del copied_config['examples'] self.config = copied_config # Add this instance to the static instance list. WitWidget.widgets.append(self) # Display WIT Polymer element. display.display(display.HTML(self._get_element_html())) display.display(display.HTML( WIT_HTML.format( examples=json.dumps(self.examples), height=height, id=WitWidget.index))) # Increment the static instance WitWidget index counter WitWidget.index += 1 # Send the provided config and examples to JS output.eval_js("""configCallback('{config}')""".format( config=json.dumps(self.config))) output.eval_js('updateExamplesCallback()') self._generate_sprite()
def visualize_sprites(embedding, atlas, sprite_size_in_3D=None, initial_camera_z=60.0): """ Plot sprites from the atlas in 3D using embedding as coordinates Args: embedding ([list of tuples (x, y, z)]): The order has to follow the order of sprites in atlas (left to right first, then top to bottom) atlas ([dict]): dict with the following items: { "path": path to PNG file of atlas. Should be relative to /usr/local/share/jupyter See https://stackoverflow.com/a/49487396/13344574 "shape": {"rows": num_rows_in_atlas, "cols": num_cols_in_atlas}, "num_sprites": number of sprites within the atlas, "sprite_size": {"height": height of single sprite, "width": width of single sprite} } sprite_size_in_3D and initial_camera_z are calibration parameters for better visualization. sprite_size_in_3D ([dict]): {"height", expected height of single sprite in 3D, "width", expected width of single sprite in 3D} initial_camera_z ([float]): initial z position of camera """ if sprite_size_in_3D is None: sprite_size_in_3D = {'height': 4.0, "width": 4.0} data = { "embedding": embedding, "atlas": atlas, "sprite_size_in_3D": sprite_size_in_3D, "initial_camera_z": initial_camera_z } dlight.load_js_libs() utils_dir = osp.dirname(osp.realpath(__file__)) ipd.display( ipd.Javascript( filename=osp.join(utils_dir, "js", "sprite_visualizer.js"))) ipd.display( ipd.HTML( filename=osp.join(utils_dir, "js", "sprite_visualizer.css.html"))) container_id = "sprite-visualizer-container-" + str(randrange(1000)) ipd.display(ipd.HTML("<div id='{}'></div> ".format(container_id))) ipd.display( ipd.Javascript(""" require(['sprite_visualizer'], function(sprite_visualizer) {{ sprite_visualizer(document.getElementById("{}"), {}); }}); """.format(container_id, json.dumps(data))))
def render(self): """Render the widget to the display.""" # Display WIT Polymer element. display.display(display.HTML(self._get_element_html())) display.display( display.HTML(WIT_HTML.format(height=self.height, id=self.id))) # Send the provided config and examples to JS output.eval_js("""configCallback({config})""".format( config=json.dumps(self.config))) output.eval_js("""updateExamplesCallback({examples})""".format( examples=json.dumps(self.examples))) self._generate_sprite() self._rendering_complete = True
def printsql(sql_string): """Prints an SQL statement with syntax highlighting""" style_string = """ <style> {pygments_css} </style> """ dis.display( dis.HTML( style_string.format( pygments_css=HtmlFormatter().get_style_defs(".highlight")))) dis.display( dis.HTML(data=highlight(sql_string, SqlLexer(), HtmlFormatter())))
def solveLocal(problem): for r in range(1): problem.randomRestart() state = problem for i in range(100000): originalConflicts = state.numConflicts() v1, v2 = state.randomSwap() state.gradientDescent(v1, v2) if args.debug_ipython: from time import sleep from IPython import display state.lastMoves = [s1, s2] display.display(display.HTML(state.prettyprinthtml())) display.clear_output(True) sleep(0.5) if state.numConflicts() == 0: return state break if args.debug: os.system("clear") print state raw_input("Press Enter to continue...")
def solveCSP(problem): statesExplored = 0 frontier = [problem] while frontier: state = frontier.pop() statesExplored += 1 if state.complete(): print 'Number of explored: ' + str(statesExplored) return state else: successors = state.getAllSuccessors() if args.debug: if not successors: print "DEADEND BACKTRACKING \n" frontier.extend(successors) if args.debug: os.system("clear") print state raw_input("Press Enter to continue...") if args.debug_ipython: from time import sleep from IPython import display display.display(display.HTML(state.prettyprinthtml())) display.clear_output(True) sleep(0.5) return None
def scene_to_notebook(scene, height=500, **kwargs): """ Convert a scene to HTML containing embedded geometry and a three.js viewer that will display nicely in an IPython/Jupyter notebook. Parameters ------------- scene : trimesh.Scene Source geometry Returns ------------- html : IPython.display.HTML Object containing rendered scene """ # keep as soft dependency from IPython import display # convert scene to a full HTML page as_html = scene_to_html(scene=scene) # escape the quotes in the HTML srcdoc = as_html.replace('"', '"') # embed this puppy as the srcdoc attr of an IFframe # I tried this a dozen ways and this is the only one that works # display.IFrame/display.Javascript really, really don't work # note trailing space to avoid IPython's pointless hardcoded warning embedded = display.HTML( '<iframe srcdoc="{srcdoc}" ' 'width="100%" height="{height}px" ' 'style="border:none;"></iframe> '.format( srcdoc=srcdoc, height=height)) return embedded
def show_gif(fname): import base64 from IPython import display with open(fname, 'rb') as fd: b64 = base64.b64encode(fd.read()).decode('ascii') return display.HTML(f'<img src="data:image/gif;base64,{b64}" />')
def __init__(self, names, initial_size, width=900, height=480): # Call this once to raise an error early if necessary: self._colors = colors.colors(len(names)) self._start = time.perf_counter() self._initial_size = initial_size self._sources = collections.OrderedDict( [ ( name, { "hist": bm.ColumnDataSource( data={"top": [], "left": [], "right": []} ), "pdf": bm.ColumnDataSource(data={"x": [], "y": []}), "stddev": bm.ColumnDataSource( data={"base": [], "lower": [], "upper": []} ), "median": bm.ColumnDataSource(data={"x": [], "y": []}), }, ) for name in names ] ) self._width = width self._height = height self._plot = None self._elapsed_rendering_seconds = 0.0 self._describe_widget = ipdisplay.HTML("") self._display_id = str(uuid.uuid1())
def show(self, format=None): """show the data object content in Jupyter :param format: format to use (when there is no/wrong suffix), e.g. 'png' """ if not is_ipython: logger.warning( "Jupyter/IPython was not detected, .show() will only display inside Jupyter" ) return from IPython import display suffix = self.suffix.lower() if format: suffix = "." + format if suffix in [".jpg", ".png", ".gif"]: display.display(display.Image(self.get(), format=suffix[1:])) elif suffix in [".htm", ".html"]: display.display(display.HTML(self.get(encoding="utf-8"))) elif suffix in [".csv", ".pq", ".parquet"]: display.display(self.as_df()) elif suffix in [".yaml", ".txt", ".py"]: display.display(display.Pretty(self.get(encoding="utf-8"))) elif suffix == ".json": display.display(display.JSON(orjson.loads(self.get()))) elif suffix == ".md": display.display(display.Markdown(self.get(encoding="utf-8"))) else: logger.error(f"unsupported show() format {suffix} for {self.url}")
def explorable(self, printJson: Optional[bool] = False): tokens = [] for idx, token in enumerate(self.tokens[0]): type = "input" if idx < self.n_input_tokens else 'output' tokens.append({ 'token': token, 'token_id': int(self.token_ids[0][idx]), 'type': type }) data = {'tokens': tokens} d.display( d.HTML(filename=os.path.join(self._path, "html", "setup.html"))) js = f""" requirejs(['basic', 'ecco'], function(basic, ecco){{ const viz_id = basic.init() ecco.renderOutputSequence({{ parentDiv: viz_id, data: {data}, tokenization_config: {json.dumps(self.config['tokenizer_config'])} }}) }}, function (err) {{ console.log(err); }})""" d.display(d.Javascript(js)) if printJson: print(data)
def show_video(path_to_mp4: str) -> None: """show_video creates an HTML element to display the given mp4 video in IPython.""" mp4 = pathlib.Path(path_to_mp4) video_b64 = base64.b64encode(mp4.read_bytes()) html = HTML_TEMPLATE.format(alt=mp4, data=video_b64.decode("ascii")) ipydisplay.display(ipydisplay.HTML(data=html))
def progress(n, width=50): """Display progress bar for long running operations. :param n: total number of steps to completion :param width: width of the progress bar (only for the text version) >>> import arlpy >>> progress = arlpy.utils.progress(100) >>> for j in range(100): next(progress) """ if _notebook: import IPython.display as _ipyd p = _ipyd.ProgressBar(total=n) did = str(_uuid.uuid4()) _ipyd.display(p, display_id=did) for j in range(1, n): p.progress = j _ipyd.update_display(p, display_id=did) yield j _ipyd.update_display(_ipyd.HTML(''), display_id=did) yield None else: _sys.stdout.write('%s|\n' % ('-' * width)) _sys.stdout.flush() c = 0 for j in range(n): c1 = int(width * (j + 1) / n) if c1 > c: _sys.stdout.write('>' * (c1 - c)) c = c1 if c == width: _sys.stdout.write('\n') _sys.stdout.flush() yield j
def source_code_display_toggle_button(show_initially=True): ''' Add a button to toggle display of source code cells in a Jupyter notebook Args: show_initially (bool): Begin with source code visible ''' js_template = ''' <script> code_show=true/* Set to opposite of show_initially */; function code_toggle() { code_show = !code_show if (code_show){$('div.input').show();} else {$('div.input').hide();} } $( document ).ready(code_toggle); </script> <form action="javascript:code_toggle()"> <input type="submit" value="Toggle code display"> </form> ''' substitutions = { 'true/* Set to opposite of show_initially */': str(not show_initially).lower() } display.display( display.HTML( _replace_literals(template=js_template, substitutions=substitutions)))
def render_fairness_indicator(slicing_metrics, event_handlers=None) -> None: """Renders the fairness inidcator view in Colab. Colab requires custom visualization to be rendered in a sandbox so we cannot use Jupyter widget. Args: slicing_metrics: A dictionary containing data for visualization. event_handlers: The event handler callback. """ event_handler_code = util.maybe_generate_event_handler_code(event_handlers) display.display( display.HTML(""" <script src="/nbextensions/tfma_widget_js/vulcanized_tfma.js"> </script> <fairness-nb-container id="component"></fairness-nb-container> <script> const element = document.getElementById('component'); {event_handler_code} element.slicingMetrics = JSON.parse('{slicingMetrics}'); </script> """.format(event_handler_code=event_handler_code, slicingMetrics=json.dumps(slicing_metrics))))
def wavejson_to_wavedrom(wavejson, width=None): ''' Create WaveDrom display from WaveJSON data. This code is from https://github.com/witchard/ipython-wavedrom. ''' # Set the width of the waveform display. style = '' if width != None: style = ' style="width: {w}px"'.format(w=str(int(width))) # Generate the HTML from the JSON. htmldata = '<div{style}><script type="WaveDrom">{json}</script></div>'.format( style=style, json=json.dumps(wavejson)) DISP.display_html(DISP.HTML(htmldata)) # Trigger the WaveDrom Javascript that creates the graphical display. DISP.display_javascript( DISP.Javascript( data='WaveDrom.ProcessAll();', lib=[ 'http://wavedrom.com/wavedrom.min.js', 'http://wavedrom.com/skins/default.js' ])) # The following allows the display of WaveDROM in the HTML files generated by nbconvert. # It's disabled because it makes Github's nbconvert freak out. setup = '''
def _repr_html_(self): css = """.vaex-description pre { max-width : 450px; white-space : nowrap; overflow : hidden; text-overflow: ellipsis; } .table { display: table; border-bottom: 3px solid black; } .row { display: table-row; } .cell { display: table-cell; } .centered { text-align: center; font-weight: bold; } .vaex-description pre:hover { max-width : initial; white-space: pre; }""" from IPython import display style = "<style>%s</style>" % css display.display(display.HTML(style)) table1 = self._head_and_tail_table() table2 = self.geometry._head_and_tail_table() return '<div class="table"><div class="row"><div class="cell centered">Attributes</div></div><div class="row"><div class="cell">%s</div></div></div><div class="table"><div class="row"><div class="cell centered">Geometry</div></div><div class="row"><div class="cell">%s</div></div></div>' % ( table1, table2)
def render_component( component_name, data, config ): """Renders the specified component in Colab. Colab requires custom visualization to be rendered in a sandbox so we cannot use Jupyter widget. Args: component_name: The name of the component to render. data: A dictionary containing data for visualization. config: A dictionary containing the configuration. """ display.display( display.HTML(""" <link rel="import" href="/nbextensions/tfma_widget_js/vulcanized_template.html"> <{component_name} id="component"></{component_name}> <script> const element = document.getElementById('component'); element.config = JSON.parse('{config}'); element.data = JSON.parse('{data}'); </script> """.format( component_name=component_name, config=json.dumps(config), data=json.dumps(data))))
def predict_token(self, inputs, topk=50, temperature=1.0): output = self.model(**inputs) scores = output[0][0][-1] / temperature s = scores.detach().numpy() sorted_predictions = s.argsort()[::-1] sm = F.softmax(scores, dim=-1).detach().numpy() tokens = [self.tokenizer.decode([t]) for t in sorted_predictions[:topk]] probs = sm[sorted_predictions[:topk]] prediction_data = [] for idx, (token, prob) in enumerate(zip(tokens, probs)): # print(idx, token, prob) prediction_data.append({'token': token, 'prob': str(prob), 'ranking': idx + 1, 'token_id': str(sorted_predictions[idx]) }) params = prediction_data viz_id = 'viz_{}'.format(round(random.random() * 1000000)) d.display(d.HTML(filename=os.path.join(self._path, "html", "predict_token.html"))) js = """ requirejs(['predict_token'], function(predict_token){{ if (window.predict === undefined) window.predict = {{}} window.predict["{}"] = new predict_token.predictToken("{}", {}) }} ) """.format(viz_id, viz_id, json.dumps(params)) d.display(d.Javascript(js))
def inline_maps(map_list): """ Embeds the HTML source of the map_list directly into the IPython notebook. This method will not work if the map depends on any files (json data). Also this uses the HTML5 srcdoc attribute, which may not be supported in all browsers. map_list: 2-D array of maps. dimensions should be [nRows][nCols]. The method will throw a RuntimeError if not nRows: Number of rows nCols: Number of columns """ nRows = len(map_list) # nCols = max([len(row) for row in map_list]) hb = hgen.HTML() t = hb.table(width="100%") for r in range(nRows): row = t.tr for c in range(len(map_list[r])): currMap = map_list[r][c] currMap._build_map() row.td( '<iframe srcdoc="{srcdoc}" style="width: 100%; height: 510px; border: none"></iframe>' .format(srcdoc=currMap.HTML.replace('"', '"'))) return idisp.HTML( '<iframe srcdoc="{srcdoc}" style="width: 100%; height: {ht}px; border: none"></iframe>' .format(srcdoc=str(t).replace('"', '"'), ht=510 * nRows))
def update_html(a_slider_dict, state_flag=0): part1_html = generate_slider_element(a_slider_dict) new_html = part1_html + "\n" + (js_update_template % state_flag) #print (new_html) display.display(display.HTML(new_html)) return
def render(self): """Render the widget to the display.""" # Display WIT Polymer element. display.display(display.HTML(self._get_element_html())) display.display( display.HTML(WIT_HTML.format(height=self.height, id=self.id))) # Send the provided config and examples to JS. output.eval_js("""configCallback({config})""".format( config=json.dumps(self.config))) self.set_examples_in_progress = True self._set_examples_looper('updateExamplesCallback({data})') self.set_examples_in_progress = False self._generate_sprite() self._rendering_complete = True
def show(text, attn): """Displays attention visualization""" attention_details = _get_attention(text, attn) att_json = json.dumps(attention_details) display.display(display.HTML(vis_html)) display.display(display.Javascript('window.attention = %s' % att_json)) display.display(display.Javascript(vis_js))
def import_assets(self): assets_str = "" with open(os.path.join(DIST_DIR, "index.html"), "r") as f: assets_str = "".join([assets_str, f.read().format(self.id, "gallery")]) with open(os.path.join(DIST_DIR, "gallery.js"), "r") as f: assets_str = "".join([assets_str, "<script>{}</script>".format(f.read())]) display.display(display.HTML(assets_str))