def test_add_html_tab(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) _, new_nb = self._split_and_merge(nb, ['python3', 'python2']) new_nb = notebook.add_html_tab(new_nb, tabs=['python3', 'python2']) cells = new_nb.cells self.assertEqual(len(cells), 18) self.assertRegex(cells[1].source, 'mdl-js-tabs') self.assertRegex(cells[2].source, 'mdl-tabs__panel.*python2') self.assertRegex(cells[4].source, '</div>') self.assertRegex(cells[5].source, '</div>') self.assertRegex(cells[8].source, 'mdl-tabs__panel.*python3') self.assertRegex(cells[12].source, 'mdl-tabs__panel.*python2') nb = notebook.split_markdown_cell(notebook.read_markdown(_all_tab_cell)) _, new_nb = self._split_and_merge(nb, ['python3', 'python2', 'python4']) cells = new_nb.cells self.assertEqual(len(cells), 5) self.assertEqual(cells[4].metadata['tab'], ['python3', 'python2']) new_nb = notebook.add_html_tab(new_nb, tabs=['python3', 'python2', 'python4']) cells = new_nb.cells self.assertEqual(len(cells), 15) self.assertRegex(cells[3].source, 'mdl-js-tabs') self.assertRegex(cells[4].source, 'mdl-tabs__panel.*python3') self.assertRegex(cells[7].source, 'mdl-tabs__panel.*python2') self.assertRegex(cells[11].source, 'mdl-tabs__panel.*python4')
def slides(): parser = argparse.ArgumentParser( description='Generate slides from markdown files.') parser.add_argument('filename', nargs='+', help='the source markdown files') parser.add_argument('--tab', default=None, help='the tab') args = parser.parse_args(sys.argv[2:]) tab = args.tab cf = config.Config() sd = Slides(cf) for fn in args.filename: fns = glob.glob(fn) if not len(fns): logging.warning('Not found ' + fn) return for md in fns: with open(md, 'r') as f: nb = notebook.read_markdown(f.read()) if tab: nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, cf.default_tab) output_fn = str(pathlib.Path(md).with_suffix('')) + ( '_' + tab if tab else '_') + '_slides.ipynb' sd.generate(nb, output_fn)
def process_and_eval_notebook(input_fn, output_fn, run_cells, timeout=20 * 60, lang='python', tab=None, default_tab=None): with open(input_fn, 'r') as f: md = f.read() nb = notebook.read_markdown(md) if tab: # get the tab nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, default_tab) if not nb: logging.info(f"Skip to eval tab {tab} for {input_fn}") # write an emtpy file to track the dependencies open(output_fn, 'w') return # evaluate if run_cells: # change to the notebook directory to resolve the relpaths properly cwd = os.getcwd() os.chdir(os.path.join(cwd, os.path.dirname(output_fn))) notedown.run(nb, timeout) os.chdir(cwd) # write nb['metadata'].update({'language_info': {'name': lang}}) with open(output_fn, 'w') as f: f.write(nbformat.writes(nb))
def test_merge_tab_notebooks(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) _, new_nb = self._split_and_merge(nb, ['python3', 'python2']) self.assertEqual(len(nb.cells), len(new_nb.cells)) for cell, new_cell in zip(nb.cells, new_nb.cells): if new_cell.source != cell.source: self.assertTrue(new_cell.source in cell.source)
def test_get_tab_notebook(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) new_nb = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3') cells = new_nb.cells self.assertEqual(cells[0].cell_type, 'markdown') self.assertEqual(cells[1].cell_type, 'markdown') self.assertEqual(cells[1].metadata['tab'], ['python2']) self.assertEqual(cells[2].cell_type, 'markdown') self.assertEqual('tab' in cells[2].metadata, False) self.assertEqual(cells[3].metadata['tab'], ['python2']) self.assertEqual(cells[4].cell_type, 'code') self.assertEqual(cells[4].metadata['tab'], ['python2']) self.assertEqual(len(cells), 6) new_nb = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3') cells = new_nb.cells self.assertEqual(cells[3].metadata['tab'], ['python3']) self.assertEqual(len(cells), 5) nb = notebook.read_markdown(_multi_tab_cell) cells = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3').cells self.assertEqual(len(cells), 3) self.assertEqual(cells[1].metadata['tab'], ['python2']) cells = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3').cells self.assertEqual(len(cells), 3) self.assertEqual(cells[1].metadata['tab'], ['python3'])
def test_add_html_tab(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3') nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3') nb4 = notebook.merge_tab_notebooks([nb2, nb3]) new_nb = notebook.add_html_tab(nb4, default_tab='python3') writer = nbconvert.RSTExporter() body, _ = writer.from_notebook_node(new_nb)
def test_merge_tab_notebooks(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3') nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3') new_nb = notebook.merge_tab_notebooks([nb2, nb3]) self.assertEqual(len(nb.cells), len(new_nb.cells)) for cell, new_cell in zip(nb.cells, new_nb.cells): if new_cell.source != cell.source: self.assertTrue(new_cell.source in cell.source)
def test_split_markdown_cell(self): nb = notebook.read_markdown(_markdown_src) new_nb = notebook.split_markdown_cell(nb) cells = new_nb.cells self.assertEqual(len(cells), 8) self.assertEqual(cells[0].cell_type, 'markdown') self.assertEqual(cells[1].cell_type, 'markdown') self.assertEqual(cells[1].metadata['tab'], 'python2') self.assertEqual(cells[2].cell_type, 'markdown') self.assertEqual('tab' in cells[2].metadata, False) self.assertEqual(cells[3].metadata['tab'], 'python2') self.assertEqual(cells[4].metadata['tab'], 'python3') self.assertEqual(cells[5].cell_type, 'code') self.assertEqual(cells[6].cell_type, 'code')
def tabcheck(self): notebooks, _, _ = self._find_md_files() error = False for fn in notebooks: with open(fn, 'r') as f: nb = notebook.read_markdown(f.read()) nb = notebook.split_markdown_cell(nb) for c in nb.cells: tabs = notebook.get_cell_tab(c) for tab in tabs: if tab and tab not in self.config.tabs + ['all']: logging.error(f"Unknown tab \"{tab}\" in {fn}") logging.error(f"The cell is {c}") error = True if error: exit(-1)
def _process_and_eval_notebook(input_fn, output_fn, run_cells, config, timeout=20 * 60, lang='python'): with open(input_fn, 'r') as f: md = f.read() nb = notebook.read_markdown(md) tab = config.tab if tab: # get the tab nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, config.default_tab) if not nb: logging.info(f"Skip to eval tab {tab} for {input_fn}") # write an empty file to track the dependencies open(output_fn, 'w') return # replace alias if tab in config.library: nb = library.replace_alias(nb, config.library[tab]) # evaluate if run_cells: # change to the notebook directory to resolve the relpaths properly cwd = os.getcwd() os.chdir(os.path.join(cwd, os.path.dirname(output_fn))) notedown.run(nb, timeout) os.chdir(cwd) # change stderr output to stdout output for cell in nb.cells: if cell.cell_type == 'code' and 'outputs' in cell: outputs = [] for out in cell['outputs']: if ('data' in out and 'text/plain' in out['data'] and out['data']['text/plain'].startswith('HBox')): # that's tqdm progress bar cannot displayed properly. continue if 'name' in out and out['name'] == 'stderr': out['name'] = 'stdout' outputs.append(out) cell['outputs'] = outputs # write nb['metadata'].update({'language_info': {'name': lang}}) with open(output_fn, 'w') as f: f.write(nbformat.writes(nb))
def _process_and_eval_notebook(scheduler, input_fn, output_fn, run_cells, config, timeout=20 * 60, lang='python'): with open(input_fn, 'r') as f: md = f.read() nb = notebook.read_markdown(md) tab = config.tab if tab: # get the tab nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, config.default_tab) if not nb: logging.info(f"Skip to eval tab {tab} for {input_fn}") # write an empty file to track the dependencies open(output_fn, 'w') return # replace alias if tab in config.library: nb = library.replace_alias(nb, config.library[tab]) nb = library.format_code_nb(nb) if not run_cells: logging.info(f'Converting {input_fn} to {output_fn}') _job(nb, output_fn, run_cells, timeout, lang) else: # use at most 2 gpus to eval a notebook num_gpus = resource.get_notebook_gpus(nb, 2) scheduler.add(1, num_gpus, target=_job, args=(nb, output_fn, run_cells, timeout, lang), description=f'Evaluating {input_fn}')