def test_add_html_tab(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
_, new_nb = self._split_and_merge(nb, ['python3', 'python2'])
new_nb = notebook.add_html_tab(new_nb, tabs=['python3', 'python2'])
cells = new_nb.cells
self.assertEqual(len(cells), 18)
self.assertRegex(cells[1].source, 'mdl-js-tabs')
self.assertRegex(cells[2].source, 'mdl-tabs__panel.*python2')
self.assertRegex(cells[4].source, '</div>')
self.assertRegex(cells[5].source, '</div>')
self.assertRegex(cells[8].source, 'mdl-tabs__panel.*python3')
self.assertRegex(cells[12].source, 'mdl-tabs__panel.*python2')
nb = notebook.split_markdown_cell(notebook.read_markdown(_all_tab_cell))
_, new_nb = self._split_and_merge(nb, ['python3', 'python2', 'python4'])
cells = new_nb.cells
self.assertEqual(len(cells), 5)
self.assertEqual(cells[4].metadata['tab'], ['python3', 'python2'])
new_nb = notebook.add_html_tab(new_nb, tabs=['python3', 'python2', 'python4'])
cells = new_nb.cells
self.assertEqual(len(cells), 15)
self.assertRegex(cells[3].source, 'mdl-js-tabs')
self.assertRegex(cells[4].source, 'mdl-tabs__panel.*python3')
self.assertRegex(cells[7].source, 'mdl-tabs__panel.*python2')
self.assertRegex(cells[11].source, 'mdl-tabs__panel.*python4')
def test_get_tab_notebook(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
new_nb = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
cells = new_nb.cells
self.assertEqual(cells[0].cell_type, 'markdown')
self.assertEqual(cells[1].cell_type, 'markdown')
self.assertEqual(cells[1].metadata['tab'], ['python2'])
self.assertEqual(cells[2].cell_type, 'markdown')
self.assertEqual('tab' in cells[2].metadata, False)
self.assertEqual(cells[3].metadata['tab'], ['python2'])
self.assertEqual(cells[4].cell_type, 'code')
self.assertEqual(cells[4].metadata['tab'], ['python2'])
self.assertEqual(len(cells), 6)
new_nb = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
cells = new_nb.cells
self.assertEqual(cells[3].metadata['tab'], ['python3'])
self.assertEqual(len(cells), 5)
nb = notebook.read_markdown(_multi_tab_cell)
cells = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3').cells
self.assertEqual(len(cells), 3)
self.assertEqual(cells[1].metadata['tab'], ['python2'])
cells = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3').cells
self.assertEqual(len(cells), 3)
self.assertEqual(cells[1].metadata['tab'], ['python3'])
def process_and_eval_notebook(input_fn,
output_fn,
run_cells,
timeout=20 * 60,
lang='python',
tab=None,
default_tab=None):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an emtpy file to track the dependencies
open(output_fn, 'w')
return
# evaluate
if run_cells:
# change to the notebook directory to resolve the relpaths properly
cwd = os.getcwd()
os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
notedown.run(nb, timeout)
os.chdir(cwd)
# write
nb['metadata'].update({'language_info': {'name': lang}})
with open(output_fn, 'w') as f:
f.write(nbformat.writes(nb))
def save_file(root_dir: str, nbfile: str):
nbfile = pathlib.Path(nbfile)
pyfile = root_dir / nbfile.with_suffix('.py')
with nbfile.open('r') as f:
nb = notebook.read_markdown(f.read())
saved = []
save_all = False
for cell in nb.cells:
if cell.cell_type == 'code':
src = cell.source.lstrip()
if re.search('# *@save_all', src):
save_all = True
if save_all or re.search('# *@save_cell', src):
saved.append(src)
else:
blk = _save_block(src, '@save')
if blk:
saved.append(blk)
if saved:
with pyfile.open('w') as f:
f.write(
f'# This file is generated from {str(nbfile)} automatically through:\n'
)
f.write('# d2lbook build lib\n')
f.write('# Don\'t edit it directly\n\n')
for blk in saved:
f.write(blk + '\n\n')
logging.info(f'Found {len(saved)} blocks in {str(nbfile)}')
def slides():
parser = argparse.ArgumentParser(
description='Generate slides from markdown files.')
parser.add_argument('filename',
nargs='+',
help='the source markdown files')
parser.add_argument('--tab', default=None, help='the tab')
args = parser.parse_args(sys.argv[2:])
tab = args.tab
cf = config.Config()
sd = Slides(cf)
for fn in args.filename:
fns = glob.glob(fn)
if not len(fns):
logging.warning('Not found ' + fn)
return
for md in fns:
with open(md, 'r') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, cf.default_tab)
output_fn = str(pathlib.Path(md).with_suffix('')) + (
'_' + tab if tab else '_') + '_slides.ipynb'
sd.generate(nb, output_fn)
def test_merge_tab_notebooks(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
_, new_nb = self._split_and_merge(nb, ['python3', 'python2'])
self.assertEqual(len(nb.cells), len(new_nb.cells))
for cell, new_cell in zip(nb.cells, new_nb.cells):
if new_cell.source != cell.source:
self.assertTrue(new_cell.source in cell.source)
def test_convert_notebook(self):
nb = notebook.read_markdown(_markdown_src)
body, _ = rst.convert_notebook(nb, {})
lines = body.split('\n')
for l in lines:
if l.startswith(':math:`x=1`'):
self.assertEqual(l, ':math:`x=1`, :numref:`sec_2`')
def test_add_html_tab(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
nb4 = notebook.merge_tab_notebooks([nb2, nb3])
new_nb = notebook.add_html_tab(nb4, default_tab='python3')
writer = nbconvert.RSTExporter()
body, _ = writer.from_notebook_node(new_nb)
def test_merge_tab_notebooks(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
new_nb = notebook.merge_tab_notebooks([nb2, nb3])
self.assertEqual(len(nb.cells), len(new_nb.cells))
for cell, new_cell in zip(nb.cells, new_nb.cells):
if new_cell.source != cell.source:
self.assertTrue(new_cell.source in cell.source)
def test_split_markdown_cell(self):
nb = notebook.read_markdown(_markdown_src)
new_nb = notebook.split_markdown_cell(nb)
cells = new_nb.cells
self.assertEqual(len(cells), 8)
self.assertEqual(cells[0].cell_type, 'markdown')
self.assertEqual(cells[1].cell_type, 'markdown')
self.assertEqual(cells[1].metadata['tab'], 'python2')
self.assertEqual(cells[2].cell_type, 'markdown')
self.assertEqual('tab' in cells[2].metadata, False)
self.assertEqual(cells[3].metadata['tab'], 'python2')
self.assertEqual(cells[4].metadata['tab'], 'python3')
self.assertEqual(cells[5].cell_type, 'code')
self.assertEqual(cells[6].cell_type, 'code')
def _save_code(input_fn,
output_fp,
save_mark=None,
tab=None,
default_tab=None):
"""get the code blocks (import, class, def) that will be saved"""
with open(input_fn, 'r') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
return
saved = []
for cell in nb.cells:
if cell.cell_type == 'code':
lines = cell.source.split('\n')
for i, l in enumerate(lines):
if ((save_mark and l.strip().startswith('#')
and save_mark in l)
or (tab and l.strip().endswith('@save') and '#' in l)):
if l.strip().startswith('#'):
block = [lines[i + 1]]
else:
block = lines[i:i + 2]
# For code block only containing import statements (e.g., in
# preface.md)
if lines[i + 1].startswith('import') or lines[
i + 1].startswith('from'):
for j in range(i + 2, len(lines)):
block.append(lines[j])
# For code blocks containing def or class
else:
for j in range(i + 2, len(lines)):
l = lines[j]
if not l.startswith(' ') and len(l):
break
block.append(l)
if len(block[-1]) == 0:
del block[-1]
saved.append(block)
if saved:
logging.info('Found %d blocks in %s', len(saved), input_fn)
for block in saved:
logging.info(' --- %s', block[0])
code = '# Defined in file: %s\n%s\n\n\n' % (input_fn,
'\n'.join(block))
output_fp.write(code)
def _save_code(input_fn, save_mark='@save', tab=None, default_tab=None):
"""get the code blocks (import, class, def) that will be saved"""
with open(input_fn, 'r', encoding='UTF-8') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
return []
saved = []
for i, cell in enumerate(nb.cells):
if cell.cell_type == 'code':
block = _save_block(cell.source, save_mark)
if block:
label = _find_latest_label(nb.cells[:i - 1])
saved.append([block, label, input_fn])
return saved
def tabcheck(self):
notebooks, _, _ = self._find_md_files()
error = False
for fn in notebooks:
with open(fn, 'r') as f:
nb = notebook.read_markdown(f.read())
nb = notebook.split_markdown_cell(nb)
for c in nb.cells:
tabs = notebook.get_cell_tab(c)
for tab in tabs:
if tab and tab not in self.config.tabs + ['all']:
logging.error(f"Unknown tab \"{tab}\" in {fn}")
logging.error(f"The cell is {c}")
error = True
if error:
exit(-1)
def _process_and_eval_notebook(input_fn,
output_fn,
run_cells,
config,
timeout=20 * 60,
lang='python'):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
tab = config.tab
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, config.default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an empty file to track the dependencies
open(output_fn, 'w')
return
# replace alias
if tab in config.library:
nb = library.replace_alias(nb, config.library[tab])
# evaluate
if run_cells:
# change to the notebook directory to resolve the relpaths properly
cwd = os.getcwd()
os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
notedown.run(nb, timeout)
os.chdir(cwd)
# change stderr output to stdout output
for cell in nb.cells:
if cell.cell_type == 'code' and 'outputs' in cell:
outputs = []
for out in cell['outputs']:
if ('data' in out and 'text/plain' in out['data']
and out['data']['text/plain'].startswith('HBox')):
# that's tqdm progress bar cannot displayed properly.
continue
if 'name' in out and out['name'] == 'stderr':
out['name'] = 'stdout'
outputs.append(out)
cell['outputs'] = outputs
# write
nb['metadata'].update({'language_info': {'name': lang}})
with open(output_fn, 'w') as f:
f.write(nbformat.writes(nb))
def _save_code(input_fn, output_fp, save_mark='@save', tab=None, default_tab=None):
"""get the code blocks (import, class, def) that will be saved"""
with open(input_fn, 'r', encoding='UTF-8') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
return
saved = []
for cell in nb.cells:
if cell.cell_type == 'code':
block = _save_block(cell.source, save_mark)
if block: saved.append(block)
if saved:
logging.info('Found %d blocks in %s', len(saved), input_fn)
for block in saved:
code = '# Defined in file: %s\n%s\n\n\n' %(input_fn, block)
output_fp.write(code)
def _process_and_eval_notebook(scheduler,
input_fn,
output_fn,
run_cells,
config,
timeout=20 * 60,
lang='python'):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
tab = config.tab
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, config.default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an empty file to track the dependencies
open(output_fn, 'w')
return
# replace alias
if tab in config.library:
nb = library.replace_alias(nb, config.library[tab])
nb = library.format_code_nb(nb)
if not run_cells:
logging.info(f'Converting {input_fn} to {output_fn}')
_job(nb, output_fn, run_cells, timeout, lang)
else:
# use at most 2 gpus to eval a notebook
num_gpus = resource.get_notebook_gpus(nb, 2)
scheduler.add(1,
num_gpus,
target=_job,
args=(nb, output_fn, run_cells, timeout, lang),
description=f'Evaluating {input_fn}')
def test_convert_notebook(self):
nb = notebook.read_markdown(_markdown_src)
body, _ = rst.convert_notebook(nb, {})
def test_generate_slides(self):
nb = notebook.read_markdown(_md)
nb = slides._generate_slides(nb)
common.print_list(nb.cells)
self.assertEqual(len(nb.cells), 6)
def test_remove_slide_marks(self):
nb = notebook.read_markdown(_md)
nb = slides.remove_slide_marks(nb)
common.print_list(nb.cells)
