def test_get_tab_notebook(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) new_nb = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3') cells = new_nb.cells self.assertEqual(cells[0].cell_type, 'markdown') self.assertEqual(cells[1].cell_type, 'markdown') self.assertEqual(cells[1].metadata['tab'], ['python2']) self.assertEqual(cells[2].cell_type, 'markdown') self.assertEqual('tab' in cells[2].metadata, False) self.assertEqual(cells[3].metadata['tab'], ['python2']) self.assertEqual(cells[4].cell_type, 'code') self.assertEqual(cells[4].metadata['tab'], ['python2']) self.assertEqual(len(cells), 6) new_nb = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3') cells = new_nb.cells self.assertEqual(cells[3].metadata['tab'], ['python3']) self.assertEqual(len(cells), 5) nb = notebook.read_markdown(_multi_tab_cell) cells = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3').cells self.assertEqual(len(cells), 3) self.assertEqual(cells[1].metadata['tab'], ['python2']) cells = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3').cells self.assertEqual(len(cells), 3) self.assertEqual(cells[1].metadata['tab'], ['python3'])
def test_add_html_tab(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3') nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3') nb4 = notebook.merge_tab_notebooks([nb2, nb3]) new_nb = notebook.add_html_tab(nb4, default_tab='python3') writer = nbconvert.RSTExporter() body, _ = writer.from_notebook_node(new_nb)
def test_merge_tab_notebooks(self): nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src)) nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3') nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3') new_nb = notebook.merge_tab_notebooks([nb2, nb3]) self.assertEqual(len(nb.cells), len(new_nb.cells)) for cell, new_cell in zip(nb.cells, new_nb.cells): if new_cell.source != cell.source: self.assertTrue(new_cell.source in cell.source)
def slides(): parser = argparse.ArgumentParser( description='Generate slides from markdown files.') parser.add_argument('filename', nargs='+', help='the source markdown files') parser.add_argument('--tab', default=None, help='the tab') args = parser.parse_args(sys.argv[2:]) tab = args.tab cf = config.Config() sd = Slides(cf) for fn in args.filename: fns = glob.glob(fn) if not len(fns): logging.warning('Not found ' + fn) return for md in fns: with open(md, 'r') as f: nb = notebook.read_markdown(f.read()) if tab: nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, cf.default_tab) output_fn = str(pathlib.Path(md).with_suffix('')) + ( '_' + tab if tab else '_') + '_slides.ipynb' sd.generate(nb, output_fn)
def process_and_eval_notebook(input_fn, output_fn, run_cells, timeout=20 * 60, lang='python', tab=None, default_tab=None): with open(input_fn, 'r') as f: md = f.read() nb = notebook.read_markdown(md) if tab: # get the tab nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, default_tab) if not nb: logging.info(f"Skip to eval tab {tab} for {input_fn}") # write an emtpy file to track the dependencies open(output_fn, 'w') return # evaluate if run_cells: # change to the notebook directory to resolve the relpaths properly cwd = os.getcwd() os.chdir(os.path.join(cwd, os.path.dirname(output_fn))) notedown.run(nb, timeout) os.chdir(cwd) # write nb['metadata'].update({'language_info': {'name': lang}}) with open(output_fn, 'w') as f: f.write(nbformat.writes(nb))
def _save_code(input_fn, output_fp, save_mark=None, tab=None, default_tab=None): """get the code blocks (import, class, def) that will be saved""" with open(input_fn, 'r') as f: nb = notebook.read_markdown(f.read()) if tab: nb = notebook.get_tab_notebook(nb, tab, default_tab) if not nb: return saved = [] for cell in nb.cells: if cell.cell_type == 'code': lines = cell.source.split('\n') for i, l in enumerate(lines): if ((save_mark and l.strip().startswith('#') and save_mark in l) or (tab and l.strip().endswith('@save') and '#' in l)): if l.strip().startswith('#'): block = [lines[i + 1]] else: block = lines[i:i + 2] # For code block only containing import statements (e.g., in # preface.md) if lines[i + 1].startswith('import') or lines[ i + 1].startswith('from'): for j in range(i + 2, len(lines)): block.append(lines[j]) # For code blocks containing def or class else: for j in range(i + 2, len(lines)): l = lines[j] if not l.startswith(' ') and len(l): break block.append(l) if len(block[-1]) == 0: del block[-1] saved.append(block) if saved: logging.info('Found %d blocks in %s', len(saved), input_fn) for block in saved: logging.info(' --- %s', block[0]) code = '# Defined in file: %s\n%s\n\n\n' % (input_fn, '\n'.join(block)) output_fp.write(code)
def _save_code(input_fn, save_mark='@save', tab=None, default_tab=None): """get the code blocks (import, class, def) that will be saved""" with open(input_fn, 'r', encoding='UTF-8') as f: nb = notebook.read_markdown(f.read()) if tab: nb = notebook.get_tab_notebook(nb, tab, default_tab) if not nb: return [] saved = [] for i, cell in enumerate(nb.cells): if cell.cell_type == 'code': block = _save_block(cell.source, save_mark) if block: label = _find_latest_label(nb.cells[:i - 1]) saved.append([block, label, input_fn]) return saved
def _process_and_eval_notebook(input_fn, output_fn, run_cells, config, timeout=20 * 60, lang='python'): with open(input_fn, 'r') as f: md = f.read() nb = notebook.read_markdown(md) tab = config.tab if tab: # get the tab nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, config.default_tab) if not nb: logging.info(f"Skip to eval tab {tab} for {input_fn}") # write an empty file to track the dependencies open(output_fn, 'w') return # replace alias if tab in config.library: nb = library.replace_alias(nb, config.library[tab]) # evaluate if run_cells: # change to the notebook directory to resolve the relpaths properly cwd = os.getcwd() os.chdir(os.path.join(cwd, os.path.dirname(output_fn))) notedown.run(nb, timeout) os.chdir(cwd) # change stderr output to stdout output for cell in nb.cells: if cell.cell_type == 'code' and 'outputs' in cell: outputs = [] for out in cell['outputs']: if ('data' in out and 'text/plain' in out['data'] and out['data']['text/plain'].startswith('HBox')): # that's tqdm progress bar cannot displayed properly. continue if 'name' in out and out['name'] == 'stderr': out['name'] = 'stdout' outputs.append(out) cell['outputs'] = outputs # write nb['metadata'].update({'language_info': {'name': lang}}) with open(output_fn, 'w') as f: f.write(nbformat.writes(nb))
def _save_code(input_fn, output_fp, save_mark='@save', tab=None, default_tab=None): """get the code blocks (import, class, def) that will be saved""" with open(input_fn, 'r', encoding='UTF-8') as f: nb = notebook.read_markdown(f.read()) if tab: nb = notebook.get_tab_notebook(nb, tab, default_tab) if not nb: return saved = [] for cell in nb.cells: if cell.cell_type == 'code': block = _save_block(cell.source, save_mark) if block: saved.append(block) if saved: logging.info('Found %d blocks in %s', len(saved), input_fn) for block in saved: code = '# Defined in file: %s\n%s\n\n\n' %(input_fn, block) output_fp.write(code)
def _process_and_eval_notebook(scheduler, input_fn, output_fn, run_cells, config, timeout=20 * 60, lang='python'): with open(input_fn, 'r') as f: md = f.read() nb = notebook.read_markdown(md) tab = config.tab if tab: # get the tab nb = notebook.split_markdown_cell(nb) nb = notebook.get_tab_notebook(nb, tab, config.default_tab) if not nb: logging.info(f"Skip to eval tab {tab} for {input_fn}") # write an empty file to track the dependencies open(output_fn, 'w') return # replace alias if tab in config.library: nb = library.replace_alias(nb, config.library[tab]) nb = library.format_code_nb(nb) if not run_cells: logging.info(f'Converting {input_fn} to {output_fn}') _job(nb, output_fn, run_cells, timeout, lang) else: # use at most 2 gpus to eval a notebook num_gpus = resource.get_notebook_gpus(nb, 2) scheduler.add(1, num_gpus, target=_job, args=(nb, output_fn, run_cells, timeout, lang), description=f'Evaluating {input_fn}')
def _split_and_merge(self, nb, tabs): split_nb = [notebook.get_tab_notebook(nb, tab, tabs[0]) for tab in tabs] merged_nb = notebook.merge_tab_notebooks(split_nb) return split_nb, merged_nb