def test_get_tab_notebook(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
new_nb = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
cells = new_nb.cells
self.assertEqual(cells[0].cell_type, 'markdown')
self.assertEqual(cells[1].cell_type, 'markdown')
self.assertEqual(cells[1].metadata['tab'], ['python2'])
self.assertEqual(cells[2].cell_type, 'markdown')
self.assertEqual('tab' in cells[2].metadata, False)
self.assertEqual(cells[3].metadata['tab'], ['python2'])
self.assertEqual(cells[4].cell_type, 'code')
self.assertEqual(cells[4].metadata['tab'], ['python2'])
self.assertEqual(len(cells), 6)
new_nb = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
cells = new_nb.cells
self.assertEqual(cells[3].metadata['tab'], ['python3'])
self.assertEqual(len(cells), 5)
nb = notebook.read_markdown(_multi_tab_cell)
cells = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3').cells
self.assertEqual(len(cells), 3)
self.assertEqual(cells[1].metadata['tab'], ['python2'])
cells = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3').cells
self.assertEqual(len(cells), 3)
self.assertEqual(cells[1].metadata['tab'], ['python3'])
def test_add_html_tab(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
nb4 = notebook.merge_tab_notebooks([nb2, nb3])
new_nb = notebook.add_html_tab(nb4, default_tab='python3')
writer = nbconvert.RSTExporter()
body, _ = writer.from_notebook_node(new_nb)
def test_merge_tab_notebooks(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
new_nb = notebook.merge_tab_notebooks([nb2, nb3])
self.assertEqual(len(nb.cells), len(new_nb.cells))
for cell, new_cell in zip(nb.cells, new_nb.cells):
if new_cell.source != cell.source:
self.assertTrue(new_cell.source in cell.source)
def slides():
parser = argparse.ArgumentParser(
description='Generate slides from markdown files.')
parser.add_argument('filename',
nargs='+',
help='the source markdown files')
parser.add_argument('--tab', default=None, help='the tab')
args = parser.parse_args(sys.argv[2:])
tab = args.tab
cf = config.Config()
sd = Slides(cf)
for fn in args.filename:
fns = glob.glob(fn)
if not len(fns):
logging.warning('Not found ' + fn)
return
for md in fns:
with open(md, 'r') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, cf.default_tab)
output_fn = str(pathlib.Path(md).with_suffix('')) + (
'_' + tab if tab else '_') + '_slides.ipynb'
sd.generate(nb, output_fn)
def process_and_eval_notebook(input_fn,
output_fn,
run_cells,
timeout=20 * 60,
lang='python',
tab=None,
default_tab=None):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an emtpy file to track the dependencies
open(output_fn, 'w')
return
# evaluate
if run_cells:
# change to the notebook directory to resolve the relpaths properly
cwd = os.getcwd()
os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
notedown.run(nb, timeout)
os.chdir(cwd)
# write
nb['metadata'].update({'language_info': {'name': lang}})
with open(output_fn, 'w') as f:
f.write(nbformat.writes(nb))
def _save_code(input_fn,
output_fp,
save_mark=None,
tab=None,
default_tab=None):
"""get the code blocks (import, class, def) that will be saved"""
with open(input_fn, 'r') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
return
saved = []
for cell in nb.cells:
if cell.cell_type == 'code':
lines = cell.source.split('\n')
for i, l in enumerate(lines):
if ((save_mark and l.strip().startswith('#')
and save_mark in l)
or (tab and l.strip().endswith('@save') and '#' in l)):
if l.strip().startswith('#'):
block = [lines[i + 1]]
else:
block = lines[i:i + 2]
# For code block only containing import statements (e.g., in
# preface.md)
if lines[i + 1].startswith('import') or lines[
i + 1].startswith('from'):
for j in range(i + 2, len(lines)):
block.append(lines[j])
# For code blocks containing def or class
else:
for j in range(i + 2, len(lines)):
l = lines[j]
if not l.startswith(' ') and len(l):
break
block.append(l)
if len(block[-1]) == 0:
del block[-1]
saved.append(block)
if saved:
logging.info('Found %d blocks in %s', len(saved), input_fn)
for block in saved:
logging.info(' --- %s', block[0])
code = '# Defined in file: %s\n%s\n\n\n' % (input_fn,
'\n'.join(block))
output_fp.write(code)
def _save_code(input_fn, save_mark='@save', tab=None, default_tab=None):
"""get the code blocks (import, class, def) that will be saved"""
with open(input_fn, 'r', encoding='UTF-8') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
return []
saved = []
for i, cell in enumerate(nb.cells):
if cell.cell_type == 'code':
block = _save_block(cell.source, save_mark)
if block:
label = _find_latest_label(nb.cells[:i - 1])
saved.append([block, label, input_fn])
return saved
def _process_and_eval_notebook(input_fn,
output_fn,
run_cells,
config,
timeout=20 * 60,
lang='python'):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
tab = config.tab
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, config.default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an empty file to track the dependencies
open(output_fn, 'w')
return
# replace alias
if tab in config.library:
nb = library.replace_alias(nb, config.library[tab])
# evaluate
if run_cells:
# change to the notebook directory to resolve the relpaths properly
cwd = os.getcwd()
os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
notedown.run(nb, timeout)
os.chdir(cwd)
# change stderr output to stdout output
for cell in nb.cells:
if cell.cell_type == 'code' and 'outputs' in cell:
outputs = []
for out in cell['outputs']:
if ('data' in out and 'text/plain' in out['data']
and out['data']['text/plain'].startswith('HBox')):
# that's tqdm progress bar cannot displayed properly.
continue
if 'name' in out and out['name'] == 'stderr':
out['name'] = 'stdout'
outputs.append(out)
cell['outputs'] = outputs
# write
nb['metadata'].update({'language_info': {'name': lang}})
with open(output_fn, 'w') as f:
f.write(nbformat.writes(nb))
def _save_code(input_fn, output_fp, save_mark='@save', tab=None, default_tab=None):
"""get the code blocks (import, class, def) that will be saved"""
with open(input_fn, 'r', encoding='UTF-8') as f:
nb = notebook.read_markdown(f.read())
if tab:
nb = notebook.get_tab_notebook(nb, tab, default_tab)
if not nb:
return
saved = []
for cell in nb.cells:
if cell.cell_type == 'code':
block = _save_block(cell.source, save_mark)
if block: saved.append(block)
if saved:
logging.info('Found %d blocks in %s', len(saved), input_fn)
for block in saved:
code = '# Defined in file: %s\n%s\n\n\n' %(input_fn, block)
output_fp.write(code)
def _process_and_eval_notebook(scheduler,
input_fn,
output_fn,
run_cells,
config,
timeout=20 * 60,
lang='python'):
with open(input_fn, 'r') as f:
md = f.read()
nb = notebook.read_markdown(md)
tab = config.tab
if tab:
# get the tab
nb = notebook.split_markdown_cell(nb)
nb = notebook.get_tab_notebook(nb, tab, config.default_tab)
if not nb:
logging.info(f"Skip to eval tab {tab} for {input_fn}")
# write an empty file to track the dependencies
open(output_fn, 'w')
return
# replace alias
if tab in config.library:
nb = library.replace_alias(nb, config.library[tab])
nb = library.format_code_nb(nb)
if not run_cells:
logging.info(f'Converting {input_fn} to {output_fn}')
_job(nb, output_fn, run_cells, timeout, lang)
else:
# use at most 2 gpus to eval a notebook
num_gpus = resource.get_notebook_gpus(nb, 2)
scheduler.add(1,
num_gpus,
target=_job,
args=(nb, output_fn, run_cells, timeout, lang),
description=f'Evaluating {input_fn}')
def _split_and_merge(self, nb, tabs):
split_nb = [notebook.get_tab_notebook(nb, tab, tabs[0]) for tab in tabs]
merged_nb = notebook.merge_tab_notebooks(split_nb)
return split_nb, merged_nb
