def test_add_html_tab(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
_, new_nb = self._split_and_merge(nb, ['python3', 'python2'])
new_nb = notebook.add_html_tab(new_nb, tabs=['python3', 'python2'])
cells = new_nb.cells
self.assertEqual(len(cells), 18)
self.assertRegex(cells[1].source, 'mdl-js-tabs')
self.assertRegex(cells[2].source, 'mdl-tabs__panel.*python2')
self.assertRegex(cells[4].source, '</div>')
self.assertRegex(cells[5].source, '</div>')
self.assertRegex(cells[8].source, 'mdl-tabs__panel.*python3')
self.assertRegex(cells[12].source, 'mdl-tabs__panel.*python2')
nb = notebook.split_markdown_cell(notebook.read_markdown(_all_tab_cell))
_, new_nb = self._split_and_merge(nb, ['python3', 'python2', 'python4'])
cells = new_nb.cells
self.assertEqual(len(cells), 5)
self.assertEqual(cells[4].metadata['tab'], ['python3', 'python2'])
new_nb = notebook.add_html_tab(new_nb, tabs=['python3', 'python2', 'python4'])
cells = new_nb.cells
self.assertEqual(len(cells), 15)
self.assertRegex(cells[3].source, 'mdl-js-tabs')
self.assertRegex(cells[4].source, 'mdl-tabs__panel.*python3')
self.assertRegex(cells[7].source, 'mdl-tabs__panel.*python2')
self.assertRegex(cells[11].source, 'mdl-tabs__panel.*python4')
def slides():
parser = argparse.ArgumentParser(
description='Generate slides from markdown files.')
parser.add_argument('filename',
nargs='+',
help='the source markdown files')
parser.add_argument('--tab', default=None, help='the tab')
args = parser.parse_args(sys.argv[2:])
tab = args.tab
cf = config.Config()
sd = Slides(cf)
for fn in args.filename:
fns = glob.glob(fn)
if not len(fns):
logging.warning('Not found ' + fn)
return
for md in fns:
with open(md, 'r') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, cf.default_tab)
output_fn = str(pathlib.Path(md).with_suffix('')) + (
'_' + tab if tab else '_') + '_slides.ipynb'
sd.generate(nb, output_fn)
def process_and_eval_notebook(input_fn,
output_fn,
run_cells,
timeout=20 * 60,
lang='python',
tab=None,
default_tab=None):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an emtpy file to track the dependencies
open(output_fn, 'w')
return
# evaluate
if run_cells:
# change to the notebook directory to resolve the relpaths properly
cwd = os.getcwd()
os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
notedown.run(nb, timeout)
os.chdir(cwd)
# write
nb['metadata'].update({'language_info': {'name': lang}})
with open(output_fn, 'w') as f:
f.write(nbformat.writes(nb))
def test_merge_tab_notebooks(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
_, new_nb = self._split_and_merge(nb, ['python3', 'python2'])
self.assertEqual(len(nb.cells), len(new_nb.cells))
for cell, new_cell in zip(nb.cells, new_nb.cells):
if new_cell.source != cell.source:
self.assertTrue(new_cell.source in cell.source)
def test_get_tab_notebook(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
new_nb = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
cells = new_nb.cells
self.assertEqual(cells[0].cell_type, 'markdown')
self.assertEqual(cells[1].cell_type, 'markdown')
self.assertEqual(cells[1].metadata['tab'], ['python2'])
self.assertEqual(cells[2].cell_type, 'markdown')
self.assertEqual('tab' in cells[2].metadata, False)
self.assertEqual(cells[3].metadata['tab'], ['python2'])
self.assertEqual(cells[4].cell_type, 'code')
self.assertEqual(cells[4].metadata['tab'], ['python2'])
self.assertEqual(len(cells), 6)
new_nb = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
cells = new_nb.cells
self.assertEqual(cells[3].metadata['tab'], ['python3'])
self.assertEqual(len(cells), 5)
nb = notebook.read_markdown(_multi_tab_cell)
cells = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3').cells
self.assertEqual(len(cells), 3)
self.assertEqual(cells[1].metadata['tab'], ['python2'])
cells = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3').cells
self.assertEqual(len(cells), 3)
self.assertEqual(cells[1].metadata['tab'], ['python3'])
def test_add_html_tab(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
nb4 = notebook.merge_tab_notebooks([nb2, nb3])
new_nb = notebook.add_html_tab(nb4, default_tab='python3')
writer = nbconvert.RSTExporter()
body, _ = writer.from_notebook_node(new_nb)
def test_merge_tab_notebooks(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
new_nb = notebook.merge_tab_notebooks([nb2, nb3])
self.assertEqual(len(nb.cells), len(new_nb.cells))
for cell, new_cell in zip(nb.cells, new_nb.cells):
if new_cell.source != cell.source:
self.assertTrue(new_cell.source in cell.source)
def test_split_markdown_cell(self):
nb = notebook.read_markdown(_markdown_src)
new_nb = notebook.split_markdown_cell(nb)
cells = new_nb.cells
self.assertEqual(len(cells), 8)
self.assertEqual(cells[0].cell_type, 'markdown')
self.assertEqual(cells[1].cell_type, 'markdown')
self.assertEqual(cells[1].metadata['tab'], 'python2')
self.assertEqual(cells[2].cell_type, 'markdown')
self.assertEqual('tab' in cells[2].metadata, False)
self.assertEqual(cells[3].metadata['tab'], 'python2')
self.assertEqual(cells[4].metadata['tab'], 'python3')
self.assertEqual(cells[5].cell_type, 'code')
self.assertEqual(cells[6].cell_type, 'code')
def tabcheck(self):
notebooks, _, _ = self._find_md_files()
error = False
for fn in notebooks:
with open(fn, 'r') as f:
nb = notebook.read_markdown(f.read())
nb = notebook.split_markdown_cell(nb)
for c in nb.cells:
tabs = notebook.get_cell_tab(c)
for tab in tabs:
if tab and tab not in self.config.tabs + ['all']:
logging.error(f"Unknown tab \"{tab}\" in {fn}")
logging.error(f"The cell is {c}")
error = True
if error:
exit(-1)
def _process_and_eval_notebook(input_fn,
output_fn,
run_cells,
config,
timeout=20 * 60,
lang='python'):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
tab = config.tab
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, config.default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an empty file to track the dependencies
open(output_fn, 'w')
return
# replace alias
if tab in config.library:
nb = library.replace_alias(nb, config.library[tab])
# evaluate
if run_cells:
# change to the notebook directory to resolve the relpaths properly
cwd = os.getcwd()
os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
notedown.run(nb, timeout)
os.chdir(cwd)
# change stderr output to stdout output
for cell in nb.cells:
if cell.cell_type == 'code' and 'outputs' in cell:
outputs = []
for out in cell['outputs']:
if ('data' in out and 'text/plain' in out['data']
and out['data']['text/plain'].startswith('HBox')):
# that's tqdm progress bar cannot displayed properly.
continue
if 'name' in out and out['name'] == 'stderr':
out['name'] = 'stdout'
outputs.append(out)
cell['outputs'] = outputs
# write
nb['metadata'].update({'language_info': {'name': lang}})
with open(output_fn, 'w') as f:
f.write(nbformat.writes(nb))
def _process_and_eval_notebook(scheduler,
input_fn,
output_fn,
run_cells,
config,
timeout=20 * 60,
lang='python'):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
tab = config.tab
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, config.default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an empty file to track the dependencies
open(output_fn, 'w')
return
# replace alias
if tab in config.library:
nb = library.replace_alias(nb, config.library[tab])
nb = library.format_code_nb(nb)
if not run_cells:
logging.info(f'Converting {input_fn} to {output_fn}')
_job(nb, output_fn, run_cells, timeout, lang)
else:
# use at most 2 gpus to eval a notebook
num_gpus = resource.get_notebook_gpus(nb, 2)
scheduler.add(1,
num_gpus,
target=_job,
args=(nb, output_fn, run_cells, timeout, lang),
description=f'Evaluating {input_fn}')
