def preprocess(self, nb, resources):
from datetime import datetime
# Well, now we are adding prolog in the epilog ...
nb.cells = [
from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'': True, 'prolog': True},
'execution_count': None,
'source': ("#{}\n".format(datetime.now())) + "%load_ext metapack_jupyter.magic"
})
] + nb.cells
assert self.pkg_dir
if len(self.dataframes) > 0:
nb.cells.append(from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'mt_dataframes': True, 'epilog': True},
'execution_count': None,
'source': '\n'.join("%mt_materialize {} '{}' ".format(df, self.pkg_dir) for df in self.dataframes)
}))
nb.cells.append(from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'mt_materialize': True, 'epilog': True},
'execution_count': None,
'source': dedent("""
%mt_materialize_all '{pkg_dir}'
""".format(pkg_dir=self.pkg_dir))
}))
nb.cells.append(from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'mt_final_metatab': True, 'epilog': True},
'execution_count': None,
'source': dedent("""
%mt_show_metatab
""".format(pkg_dir=self.pkg_dir))
}))
nb.cells.append(from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'mt_show_libdirs': True, 'epilog': True},
'execution_count': None,
'source': dedent("""
%mt_show_libdirs
""".format(pkg_dir=self.pkg_dir))
}))
return nb, resources
def run(self, notebook, parameters=None, cell_timeout=600):
from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
from nbformat.notebooknode import from_dict
ep = ExecutePreprocessor(timeout=cell_timeout, kernel_name="python3")
if parameters:
parameter_node = from_dict(
{
"cell_type": "code",
"metadata": {"tags": ["parameters"]},
"source": parameters,
}
)
for i, cell in enumerate(notebook.cells):
if cell.metadata:
if cell.metadata.tags:
if "default-parameters" in cell.metadata.tags:
break
else:
i = 0
notebook.cells.insert(i, parameter_node)
errors = False
try:
out = ep.preprocess(notebook)
except CellExecutionError:
errors = True
finally:
return notebook, errors
def _markdown_cell(self, source):
"""Create a markdown cell with source content."""
return notebooknode.from_dict({
"cell_type": "markdown",
"metadata": {},
"source": source
})
def from_notebook_node(self, nb, copy=False):
if copy:
nb = notebooknode.from_dict(nb.copy())
self._update_prev_next_history(nb)
self._generate_notebook_meme(nb)
self._generate_cell_meme(nb)
self._update_prev_next_meme(nb)
return nb
def preprocess(self, nb, resources):
nb.cells = [from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {},
'execution_count': None,
'source': dedent("""
from metatab.jupyter.script import get_ipython
""")
})] + nb.cells
return nb, resources
def preprocess(self, nb, resources):
source = '\n'.join('{}={}'.format(k, repr(v)) for k, v in self.env.items()
if k and isinstance(v, (int, float, str)))
nb.cells = [
from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'': True, 'prolog': True},
'execution_count': None,
'source': source
}),
# Mark this notebook as being run in a build, which can change the behaviour of some magics
from_dict({
'cell_type': 'code',
'outputs': [],
'metadata': {'': True, 'prolog': True},
'execution_count': None,
'source': "METAPACK_BUILDING=True"
}),
] + nb.cells
return nb, resources
def __init__(self, ipynb: TextIO):
_notebook = nbformat.read(ipynb, as_version=4)
value = from_dict(_notebook)
# self is a dict. If you look at the MRO, NotebookNode is a dict.
# Think of the below as two dicts (`self` and `value`) merging
# (where `self` happens to be an empty dict).
dict.__init__(self, value)
self.__dict__["ipynb"] = os.path.abspath(ipynb)
self.__dict__["name"] = os.path.basename(ipynb)
self.__dict__["env"] = dict()
self.__dict__["tests"] = self.extract_codes()
self.__dict__["result"] = None
self.__dict__["stack"] = None
def copy_nb_from_template(
path_source,
name_source,
path_target,
name_target,
dic_replace: dict,
):
path_in = os.path.join(path_source, name_source)
path_out = os.path.join(path_target, name_target)
nb_in = jupytext.read(path_in)
st_in = json.dumps(nb_in)
# %%
st_out: str = st_in
for l, v in dic_replace.items():
st_out = st_out.replace(l, v)
new_out = json.loads(st_out)
new_out = nb.from_dict(new_out)
# %%
jupytext.write(new_out, path_out)
def from_notebook_node(self, nb, copy=False):
if copy:
nb = notebooknode.from_dict(nb.copy())
MemeGenerator().from_notebook_node(nb)
orig_nb = deepcopy(nb)
self._update_prev_next_history(nb)
self._update_notebook_meme(nb)
self._update_cell_meme(nb)
self._update_prev_next_cell_meme(nb)
self._update_root_cells(nb)
self._update_root_cells_history(orig_nb, nb)
if self.clear_server_signature:
self.log.debug('Clear server signature metadata')
if 'lc_server_signature' in nb.metadata['lc_notebook_meme']:
del nb.metadata['lc_notebook_meme']['lc_server_signature']
return nb
def run(self, notebook, parameters=None, cell_timeout=600):
from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
from nbformat.notebooknode import from_dict
ep = ExecutePreprocessor(timeout=cell_timeout, kernel_name="python3")
if parameters:
parameter_node = from_dict(
{
"cell_type": "code",
"metadata": {"tags": ["parameters"]},
"source": parameters,
}
)
for i, cell in enumerate(notebook.cells):
if cell.metadata:
if cell.metadata.tags:
if "default-parameters" in cell.metadata.tags:
break
else:
i = 0
notebook.cells.insert(i, parameter_node)
error = None
try:
out = ep.preprocess(notebook)
except CellExecutionError as e:
try:
lines = [line.strip()
for line in str(e.traceback).split('\n') if line]
error = lines[-1]
except:
error = 'Exception: unknown error.'
finally:
return notebook, error
def test_file_manager(instance_path, renga_client, monkeypatch,
deployer_responses, storage_responses, notebook):
"""Test file manager."""
client = renga_client
client.buckets.create('bucket1')
monkeypatch.setenv('RENGA_ENDPOINT', client.api.endpoint)
monkeypatch.setenv('RENGA_ACCESS_TOKEN', client.api.token['access_token'])
monkeypatch.setenv('RENGA_CONTEXT_ID', 'abcd')
contents_manager = RengaStorageManager()
contents_manager._save_notebook('current_context/inputs/notebook',
from_dict(notebook))
notebook_model = contents_manager.get('current_context/inputs/notebook')
assert notebook_model['content']['cells'][0]['source'] == notebook[
'cells'][0]['source']
new_path = '1.ipynb'
notebook_model['path'] = 'current_context/inputs/' + new_path
contents_manager.update(notebook_model, 'current_context/inputs/notebook')
assert client.buckets[1234].files[9876].filename == new_path
notebook_model = contents_manager.get('buckets/1234/9876')
assert notebook_model['name'] == new_path
new_path = '2.ipynb'
notebook_model['path'] = 'buckets/1234/' + new_path
contents_manager.update(notebook_model, 'buckets/1234/9876')
assert client.buckets[1234].files[9876].filename == new_path
# Move to different bucket:
new_path = '3.ipynb'
notebook_model['path'] = 'buckets/2345/' + new_path
with pytest.raises(RuntimeError):
contents_manager.update(notebook_model, 'buckets/1234/9876')
notebook = notebooknode.from_dict({
'cells': [
{
"cell_type": 'raw',
"metadata": {},
"source": 'Text in a raw cell, should be ignored.'
},
{
"cell_type": 'markdown',
"metadata": {},
"source": (
'This line should be ignored.\n'
'# This is the title\n'
'Date: 2018-06-10\n'
'This line should be ignored.\n'
'Tags: Test, Front Matter, Markdown\n'
)
},
{
"cell_type": 'code',
"metadata": {},
"outputs": [],
"execution_count": 1,
"source": 'Text in a code cell, should be ignored.'
},
{
"cell_type": 'markdown',
"metadata": {},
"source": (
'This line should be ignored.\n'
'Categories: Front Matter, Converter\n'
'This line should be ignored.\n'
'This text should be ignored.<!--eofm-->This text should be visible.\n'
'This line should be visible.\n'
)
},
{
"cell_type": 'markdown',
"metadata": {},
"source": (
'Some text with an inline equality $escaped\_0 = lower_0$.\n'
'And a display equality:\n'
'$$escaped\_1 = subscript_1.$$'
)
},
{
"cell_type": 'markdown',
"metadata": {},
"source": 'Some text with an .\nAnd more text.'
},
],
'metadata': {},
'nbformat': 4,
'nbformat_minor': 2
})
def test_replace_latex_enclosing_dollars(preprocessor):
result = preprocessor._replace_latex_enclosing_dollars(source)
assert result == dollars_processed
def test_fix_latex_escaped_underscores(preprocessor):
result = preprocessor._fix_latex_escaped_underscores(dollars_processed)
assert result == fully_processed
raw_cell, code_cell, markdown_cell, expected_markdown_cell = [
notebooknode.from_dict({
"cell_type": cell_type,
"metadata": {},
"source": source
}) for cell_type, source in [('raw', source), (
'code', source), ('markdown', source), ('markdown', fully_processed)]
]
@pytest.mark.parametrize("input_cell, expected_cell", [
(raw_cell, raw_cell),
(code_cell, code_cell),
(markdown_cell, expected_markdown_cell),
])
def test_preprocess_cell(preprocessor, input_cell, expected_cell):
assert preprocessor.preprocess_cell(input_cell, None,
None) == (expected_cell, None)
import pytest
from nbformat import notebooknode
from nb2hugo.preprocessors import WrapHtmlPreprocessor
from copy import deepcopy
@pytest.fixture
def preprocessor():
return WrapHtmlPreprocessor()
cell_without_outputs = notebooknode.from_dict({})
cell_with_empty_outputs = notebooknode.from_dict({"outputs": []})
cell_without_data_in_outputs = notebooknode.from_dict(
{"outputs": [{
"name": "stdout"
}]})
cell_without_html_in_outputs = notebooknode.from_dict(
{"outputs": [{
"data": {
"text/plain": "text"
}
}]})
cell_with_text_html_in_outputs = notebooknode.from_dict(
{"outputs": [{
"data": {
"text/html": "html"
}
}]})
transformed_cell_with_text_html_in_outputs = notebooknode.from_dict({
"outputs": [{
notebook = notebooknode.from_dict({
'cells': [
{
"cell_type": 'raw',
"metadata": {},
"source": 'Text in a raw cell, should be ignored.'
},
{
"cell_type": 'markdown',
"metadata": {},
"source": (
'This line should be ignored.\n'
'# This is the title\n'
'Date: 2018-06-10\n'
'This line should be ignored.\n'
'Tags: Test, Front Matter, Markdown\n'
)
},
{
"cell_type": 'code',
"metadata": {},
"outputs": [],
"execution_count": 1,
"source": 'Text in a code cell, should be ignored.'
},
{
"cell_type": 'markdown',
"metadata": {},
"source": (
'This line should be ignored.\n'
'Categories: Front Matter, Converter\n'
'This line should be ignored.\n'
'This text should be ignored.<!--eofm-->This text should be visible.\n'
'This line should be visible.\n'
)
}
],
'metadata': {},
'nbformat': 4,
'nbformat_minor': 2
})
