def test_save(self):
        resp = self.api.read('a.ipynb', 'foo')
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        nb.cells.append(new_markdown_cell(u'Created by test ³'))

        nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb, 'type': 'notebook'}
        resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))

        nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
        with io.open(nbfile, 'r', encoding='utf-8') as f:
            newnb = read(f, as_version=4)
        self.assertEqual(newnb.cells[0].source,
                         u'Created by test ³')
        nbcontent = self.api.read('a.ipynb', 'foo').json()['content']
        newnb = from_dict(nbcontent)
        self.assertEqual(newnb.cells[0].source,
                         u'Created by test ³')

        # Save and rename
        nbmodel= {'name': 'a2.ipynb', 'path':'foo/bar', 'content': nb, 'type': 'notebook'}
        resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
        saved = resp.json()
        self.assertEqual(saved['name'], 'a2.ipynb')
        self.assertEqual(saved['path'], 'foo/bar')
        assert os.path.isfile(pjoin(self.notebook_dir.name,'foo','bar','a2.ipynb'))
        assert not os.path.isfile(pjoin(self.notebook_dir.name, 'foo', 'a.ipynb'))
        with assert_http_error(404):
            self.api.read('a.ipynb', 'foo')
    def test_very_long_cells(self):
        """
        Torture test that long cells do not cause issues
        """
        lorem_ipsum_text = textwrap.dedent("""\
          Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
          dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
          nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
          suscipit interdum at, ultrices sit amet orci. Mauris facilisis
          imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
          consequat malesuada neque sit amet pulvinar. Curabitur pretium
          ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
          volutpat, eu adipiscing purus pulvinar. Maecenas consequat
          luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
          erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
          at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
          Quisque at sapien non nibh feugiat tempus ac ultricies purus.
           """)
        lorem_ipsum_text = lorem_ipsum_text.replace("\n"," ") + "\n\n"
        large_lorem_ipsum_text = "".join([lorem_ipsum_text]*3000)

        notebook_name = "lorem_ipsum_long.ipynb"
        nb = v4.new_notebook(
            cells=[
                    v4.new_markdown_cell(source=large_lorem_ipsum_text)
            ]
        )

        with TemporaryDirectory() as td:
            nbfile = os.path.join(td, notebook_name)
            with open(nbfile, 'w') as f:
                write(nb, f, 4)

            (output, resources) = LatexExporter(template_file='article').from_filename(nbfile)
            assert len(output) > 0
Beispiel #3
0
    def test_checkpoints_follow_file(self):

        # Read initial file state
        orig = self.api.read('foo/a.ipynb')

        # Create a checkpoint of initial state
        r = self.api.new_checkpoint('foo/a.ipynb')
        cp1 = r.json()

        # Modify file and save
        nbcontent = json.loads(orig.text)['content']
        nb = from_dict(nbcontent)
        hcell = new_markdown_cell('Created by test')
        nb.cells.append(hcell)
        nbmodel = {'content': nb, 'type': 'notebook'}
        self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        # Rename the file.
        self.api.rename('foo/a.ipynb', 'foo/z.ipynb')

        # Looking for checkpoints in the old location should yield no results.
        self.assertEqual(self.api.get_checkpoints('foo/a.ipynb').json(), [])

        # Looking for checkpoints in the new location should work.
        cps = self.api.get_checkpoints('foo/z.ipynb').json()
        self.assertEqual(cps, [cp1])

        # Delete the file.  The checkpoint should be deleted as well.
        self.api.delete('foo/z.ipynb')
        cps = self.api.get_checkpoints('foo/z.ipynb').json()
        self.assertEqual(cps, [])
Beispiel #4
0
    def test_very_long_cells(self):
        """
        Torture test that long cells do not cause issues
        """
        lorem_ipsum_text = textwrap.dedent("""\
          Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
          dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
          nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
          suscipit interdum at, ultrices sit amet orci. Mauris facilisis
          imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
          consequat malesuada neque sit amet pulvinar. Curabitur pretium
          ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
          volutpat, eu adipiscing purus pulvinar. Maecenas consequat
          luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
          erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
          at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
          Quisque at sapien non nibh feugiat tempus ac ultricies purus.
           """)
        lorem_ipsum_text = lorem_ipsum_text.replace("\n", " ") + "\n\n"
        large_lorem_ipsum_text = "".join([lorem_ipsum_text] * 3000)

        notebook_name = "lorem_ipsum_long.ipynb"
        nb = v4.new_notebook(
            cells=[v4.new_markdown_cell(source=large_lorem_ipsum_text)])

        with TemporaryDirectory() as td:
            nbfile = os.path.join(td, notebook_name)
            with open(nbfile, 'w') as f:
                write(nb, f, 4)

            (output, resources) = LatexExporter(
                template_file='article').from_filename(nbfile)
            assert len(output) > 0
Beispiel #5
0
    def test_checkpoints_follow_file(self):

        # Read initial file state
        orig = self.api.read('foo/a.ipynb')

        # Create a checkpoint of initial state
        r = self.api.new_checkpoint('foo/a.ipynb')
        cp1 = r.json()

        # Modify file and save
        nbcontent = json.loads(orig.text)['content']
        nb = from_dict(nbcontent)
        hcell = new_markdown_cell('Created by test')
        nb.cells.append(hcell)
        nbmodel = {'content': nb, 'type': 'notebook'}
        self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        # Rename the file.
        self.api.rename('foo/a.ipynb', 'foo/z.ipynb')

        # Looking for checkpoints in the old location should yield no results.
        self.assertEqual(self.api.get_checkpoints('foo/a.ipynb').json(), [])

        # Looking for checkpoints in the new location should work.
        cps = self.api.get_checkpoints('foo/z.ipynb').json()
        self.assertEqual(cps, [cp1])

        # Delete the file.  The checkpoint should be deleted as well.
        self.api.delete('foo/z.ipynb')
        cps = self.api.get_checkpoints('foo/z.ipynb').json()
        self.assertEqual(cps, [])
    def build_notebook(self):
        """Build a reveal slides notebook in memory for use with tests.
        Overrides base in PreprocessorTestsBase"""

        outputs = [nbformat.new_output(output_type="stream", name="stdout", text="a")]

        slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
        subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}

        cells=[nbformat.new_code_cell(source="", execution_count=1, outputs=outputs),
               nbformat.new_markdown_cell(source="", metadata=slide_metadata),
               nbformat.new_code_cell(source="", execution_count=2, outputs=outputs),
               nbformat.new_markdown_cell(source="", metadata=slide_metadata),
               nbformat.new_markdown_cell(source="", metadata=subslide_metadata)]

        return nbformat.new_notebook(cells=cells)
    def setUp(self):
        nbdir = self.notebook_dir.name

        if not os.path.isdir(pjoin(nbdir, 'foo')):
            os.mkdir(pjoin(nbdir, 'foo'))

        nb = new_notebook()

        nb.cells.append(new_markdown_cell(u'Created by test ³'))
        cc1 = new_code_cell(source=u'print(2*6)')
        cc1.outputs.append(new_output(output_type="stream", text=u'12'))
        cc1.outputs.append(
            new_output(
                output_type="execute_result",
                data={'image/png': png_green_pixel},
                execution_count=1,
            ))
        nb.cells.append(cc1)

        with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'),
                     'w',
                     encoding='utf-8') as f:
            write(nb, f, version=4)

        self.nbconvert_api = NbconvertAPI(self.base_url())
Beispiel #8
0
def lines_to_notebook(lines, name=None):
    """
    Convert the lines of an m file into an IPython notebook

    Parameters
    ----------
    lines : list
        A list of strings. Each element is a line in the m file

    Returns
    -------
    notebook : an IPython NotebookNode class instance, containing the
    information required to create a file


    """
    source = []
    md = np.empty(len(lines), dtype=object)
    new_cell = np.empty(len(lines), dtype=object)
    for idx, l in enumerate(lines):
        new_cell[idx], md[idx], this_source = format_line(l)
        # Transitions between markdown and code and vice-versa merit a new
        # cell, even if no newline, or "%%" is found. Make sure not to do this
        # check for the very first line!
        if idx > 1 and not new_cell[idx]:
            if md[idx] != md[idx - 1]:
                new_cell[idx] = True

        source.append(this_source)
    # This defines the breaking points between cells:
    new_cell_idx = np.hstack([np.where(new_cell)[0], -1])

    # Listify the sources:
    cell_source = [
        source[new_cell_idx[i]:new_cell_idx[i + 1]]
        for i in range(len(new_cell_idx) - 1)
    ]
    cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx) - 1)]
    cells = []

    # Append the notebook with loading matlab magic extension
    notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
                    + "pymat.load_ipython_extension(ip)"
    cells.append(nbformat.new_code_cell(notebook_head))  #, language='python'))

    for cell_idx, cell_s in enumerate(cell_source):
        if cell_md[cell_idx]:
            cells.append(nbformat.new_markdown_cell(cell_s))
        else:
            cell_s.insert(0, '%%matlab\n')
            cells.append(
                nbformat.new_code_cell(cell_s))  #, language='matlab'))

    #ws = nbformat.new_worksheet(cells=cells)
    notebook = nbformat.new_notebook(cells=cells)
    return notebook
    def add_markdown_cell(self, path):
        # Load and update
        model = self.contents.get(path=path)
        model['content'].cells.append(
            new_markdown_cell('Created by test: ' + path)
        )

        # Save and checkpoint again.
        self.contents.save(model, path=path)
        return model
def lines_to_notebook(lines, name=None):
    """
    Convert the lines of an m file into an IPython notebook

    Parameters
    ----------
    lines : list
        A list of strings. Each element is a line in the m file

    Returns
    -------
    notebook : an IPython NotebookNode class instance, containing the
    information required to create a file


    """
    source = []
    md = np.empty(len(lines), dtype=object)
    new_cell = np.empty(len(lines), dtype=object)
    for idx, l in enumerate(lines):
        new_cell[idx], md[idx], this_source = format_line(l)
        # Transitions between markdown and code and vice-versa merit a new
        # cell, even if no newline, or "%%" is found. Make sure not to do this
        # check for the very first line!
        if idx>1 and not new_cell[idx]:
            if md[idx] != md[idx-1]:
                new_cell[idx] = True

        source.append(this_source)
    # This defines the breaking points between cells:
    new_cell_idx = np.hstack([np.where(new_cell)[0], -1])

    # Listify the sources:
    cell_source = [source[new_cell_idx[i]:new_cell_idx[i+1]]
                   for i in range(len(new_cell_idx)-1)]
    cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx)-1)]
    cells = []

    # Append the notebook with loading matlab magic extension
    notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
                    + "pymat.load_ipython_extension(ip)"
    cells.append(nbformat.new_code_cell(notebook_head))#, language='python'))

    for cell_idx, cell_s in enumerate(cell_source):
        if cell_md[cell_idx]:
            cells.append(nbformat.new_markdown_cell(cell_s))
        else:
            cell_s.insert(0, '%%matlab\n')
            cells.append(nbformat.new_code_cell(cell_s))#, language='matlab'))

    #ws = nbformat.new_worksheet(cells=cells)
    notebook = nbformat.new_notebook(cells=cells)
    return notebook
Beispiel #11
0
    def test_save(self):
        resp = self.api.read('foo/a.ipynb')
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        nb.cells.append(new_markdown_cell(u'Created by test ³'))

        nbmodel = {'content': nb, 'type': 'notebook'}
        resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        newnb = from_dict(nbcontent)
        self.assertEqual(newnb.cells[0].source, u'Created by test ³')
Beispiel #12
0
    def test_save(self):
        resp = self.api.read('foo/a.ipynb')
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        nb.cells.append(new_markdown_cell(u'Created by test ³'))

        nbmodel= {'content': nb, 'type': 'notebook'}
        resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        newnb = from_dict(nbcontent)
        self.assertEqual(newnb.cells[0].source,
                         u'Created by test ³')
Beispiel #13
0
  def flush_cell(self):
    if len(self._code_lines) > 0 and len(self._text_lines) > 0:
      raise Exception('only text or only code can be flushed')
    if len(self._code_lines) == 0 and len(self._text_lines) == 0:
      raise Exception('nothing to flush')

    if len(self._code_lines) > 0:
      self.cells.append(nbf.new_code_cell('\n'.join(self._code_lines)))
    else:    
      self.cells.append(nbf.new_markdown_cell('\n'.join(self._text_lines)))    

    self._code_lines = []
    self._text_lines = []
Beispiel #14
0
    def _create_solution_cell(source, cell_type):
        if cell_type == "markdown":
            cell = new_markdown_cell(source=source)
        elif cell_type == "code":
            cell = new_code_cell(source=source)
        else:
            raise ValueError("invalid cell type: {}".format(cell_type))

        cell.metadata.nbgrader = {}
        cell.metadata.nbgrader["solution"] = True
        cell.metadata.nbgrader["checksum"] = compute_checksum(cell)

        return cell
Beispiel #15
0
def create_locked_cell(source, cell_type, grade_id):
    if cell_type == "markdown":
        cell = new_markdown_cell(source=source)
    elif cell_type == "code":
        cell = new_code_cell(source=source)
    else:
        raise ValueError("invalid cell type: {}".format(cell_type))

    cell.metadata.nbgrader = {}
    cell.metadata.nbgrader["locked"] = True
    cell.metadata.nbgrader["grade_id"] = grade_id

    return cell
Beispiel #16
0
def create_locked_cell(source, cell_type, grade_id):
    if cell_type == "markdown":
        cell = new_markdown_cell(source=source)
    elif cell_type == "code":
        cell = new_code_cell(source=source)
    else:
        raise ValueError("invalid cell type: {}".format(cell_type))

    cell.metadata.nbgrader = {}
    cell.metadata.nbgrader["locked"] = True
    cell.metadata.nbgrader["grade_id"] = grade_id

    return cell
Beispiel #17
0
    def test_run_nb(self):
        """Test %run notebook.ipynb"""
        from IPython.nbformat import v4, writes
        nb = v4.new_notebook(cells=[
            v4.new_markdown_cell("The Ultimate Question of Everything"),
            v4.new_code_cell("answer=42")
        ])
        src = writes(nb, version=4)
        self.mktmp(src, ext='.ipynb')

        _ip.magic("run %s" % self.fname)

        nt.assert_equal(_ip.user_ns['answer'], 42)
Beispiel #18
0
  def flush_cell(self):
    if len(self._code_lines) > 0 and len(self._text_lines) > 0:
      raise Exception('only text or only code can be flushed')
    if len(self._code_lines) == 0 and len(self._text_lines) == 0:
      raise Exception('nothing to flush')

    if len(self._code_lines) > 0:
      self.cells.append(nbf.new_code_cell('\n'.join(self._code_lines)))
    else:    
      self.cells.append(nbf.new_markdown_cell('\n'.join(self._text_lines)))    

    self._code_lines = []
    self._text_lines = []
    def test_checkpoint_all(self):
        """
        Test that checkpoint_all correctly makes a checkpoint for all files.
        """
        paths = populate(self.contents)
        original_content_minus_trust = {
            # Remove metadata that we expect to have dropped
            path: strip_transient(self.contents.get(path)['content'])
            for path in paths
        }

        original_cps = {}
        for path in paths:
            # Create a checkpoint, then update the file.
            original_cps[path] = self.contents.create_checkpoint(path)
            self.add_markdown_cell(path)

        # Verify that we still have the old version checkpointed.
        cp_content = {
            path: self.checkpoints.get_notebook_checkpoint(
                cp['id'],
                path,
            )['content']
            for path, cp in iteritems(original_cps)
        }
        self.assertEqual(original_content_minus_trust, cp_content)

        new_cps = checkpoint_all(
            self.checkpoints.db_url,
            self.td.name,
            self.checkpoints.user_id,
        )

        new_cp_content = {
            path: self.checkpoints.get_notebook_checkpoint(
                cp['id'],
                path,
            )['content']
            for path, cp in iteritems(new_cps)
        }
        for path, new_content in iteritems(new_cp_content):
            old_content = original_content_minus_trust[_norm_unicode(path)]
            self.assertEqual(
                new_content['cells'][:-1],
                old_content['cells'],
            )
            self.assertEqual(
                new_content['cells'][-1],
                new_markdown_cell('Created by test: ' + _norm_unicode(path)),
            )
Beispiel #20
0
def create_grade_and_solution_cell(source, cell_type, grade_id, points):
    if cell_type == "markdown":
        cell = new_markdown_cell(source=source)
    elif cell_type == "code":
        cell = new_code_cell(source=source)
    else:
        raise ValueError("invalid cell type: {}".format(cell_type))

    cell.metadata.nbgrader = {}
    cell.metadata.nbgrader["solution"] = True
    cell.metadata.nbgrader["grade"] = True
    cell.metadata.nbgrader["grade_id"] = grade_id
    cell.metadata.nbgrader["points"] = points

    return cell
Beispiel #21
0
def create_grade_and_solution_cell(source, cell_type, grade_id, points):
    if cell_type == "markdown":
        cell = new_markdown_cell(source=source)
    elif cell_type == "code":
        cell = new_code_cell(source=source)
    else:
        raise ValueError("invalid cell type: {}".format(cell_type))

    cell.metadata.nbgrader = {}
    cell.metadata.nbgrader["solution"] = True
    cell.metadata.nbgrader["grade"] = True
    cell.metadata.nbgrader["grade_id"] = grade_id
    cell.metadata.nbgrader["points"] = points

    return cell
Beispiel #22
0
 def test_run_nb(self):
     """Test %run notebook.ipynb"""
     from IPython.nbformat import v4, writes
     nb = v4.new_notebook(
        cells=[
             v4.new_markdown_cell("The Ultimate Question of Everything"),
             v4.new_code_cell("answer=42")
         ]
     )
     src = writes(nb, version=4)
     self.mktmp(src, ext='.ipynb')
     
     _ip.magic("run %s" % self.fname)
     
     nt.assert_equal(_ip.user_ns['answer'], 42)
    def build_notebook(self):
        """Build a reveal slides notebook in memory for use with tests.
        Overrides base in PreprocessorTestsBase"""

        outputs = [
            nbformat.new_output(output_type="stream", name="stdout", text="a")
        ]

        slide_metadata = {'slideshow': {'slide_type': 'slide'}}
        subslide_metadata = {'slideshow': {'slide_type': 'subslide'}}

        cells = [
            nbformat.new_code_cell(source="",
                                   execution_count=1,
                                   outputs=outputs),
            nbformat.new_markdown_cell(source="", metadata=slide_metadata),
            nbformat.new_code_cell(source="",
                                   execution_count=2,
                                   outputs=outputs),
            nbformat.new_markdown_cell(source="", metadata=slide_metadata),
            nbformat.new_markdown_cell(source="", metadata=subslide_metadata)
        ]

        return nbformat.new_notebook(cells=cells)
Beispiel #24
0
def write(cells, nb_version=4):
    """Turn cells list into valid IPython notebook code."""
    # Use IPython.nbformat functionality for writing the notebook
    if nb_version == 3:
        from IPython.nbformat.v3 import (
            new_code_cell,
            new_text_cell,
            new_worksheet,
            new_notebook,
            new_metadata,
            new_author,
        )

        nb = new_worksheet()

    elif nb_version == 4:
        from IPython.nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook

        nb_cells = []

    for cell_tp, language, block in cells:
        if cell_tp == "markdown":
            if nb_version == 3:
                nb.cells.append(new_text_cell(u"markdown", source=block))
            elif nb_version == 4:
                nb_cells.append(new_markdown_cell(source=block))
        elif cell_tp == "codecell":
            if nb_version == 3:
                nb.cells.append(new_code_cell(input=block))
            elif nb_version == 4:
                nb_cells.append(new_code_cell(source=block))

    if nb_version == 3:
        nb = new_notebook(worksheets=[nb], metadata=new_metadata())
        # Let us make v4 notebook here by upgrading
        from IPython.nbformat.v4 import upgrade

        nb = upgrade(nb)
        import IPython.nbformat.v4.nbjson as nbjson

        # Convert nb to json format
        filestr = nbjson.writes(nb)
    elif nb_version == 4:
        nb = new_notebook(cells=nb_cells)
        from IPython.nbformat import writes

        filestr = writes(nb, version=4)
    return filestr
Beispiel #25
0
    def test_save(self):
        resp = self.api.read('foo/a.ipynb')
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        nb.cells.append(new_markdown_cell(u'Created by test ³'))

        nbmodel = {'content': nb, 'type': 'notebook'}
        resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
        with io.open(nbfile, 'r', encoding='utf-8') as f:
            newnb = read(f, as_version=4)
        self.assertEqual(newnb.cells[0].source, u'Created by test ³')
        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        newnb = from_dict(nbcontent)
        self.assertEqual(newnb.cells[0].source, u'Created by test ³')
    def test_save(self):
        resp = self.api.read("foo/a.ipynb")
        nbcontent = json.loads(resp.text)["content"]
        nb = from_dict(nbcontent)
        nb.cells.append(new_markdown_cell(u"Created by test ³"))

        nbmodel = {"content": nb, "type": "notebook"}
        resp = self.api.save("foo/a.ipynb", body=json.dumps(nbmodel))

        nbfile = pjoin(self.notebook_dir.name, "foo", "a.ipynb")
        with io.open(nbfile, "r", encoding="utf-8") as f:
            newnb = read(f, as_version=4)
        self.assertEqual(newnb.cells[0].source, u"Created by test ³")
        nbcontent = self.api.read("foo/a.ipynb").json()["content"]
        newnb = from_dict(nbcontent)
        self.assertEqual(newnb.cells[0].source, u"Created by test ³")
Beispiel #27
0
def write(cells):
    """Turn cells list into valid IPython notebook code."""
    # Use IPython.nbformat functionality for writing the notebook
    from IPython.nbformat.v4 import (
        new_code_cell, new_markdown_cell, new_notebook)
    nb_cells = []

    for cell_tp, language, block in cells:
        if cell_tp == 'markdown':
            nb_cells.append(new_markdown_cell(source=block))
        elif cell_tp == 'codecell':
            nb_cells.append(new_code_cell(source=block))

    nb = new_notebook(cells=nb_cells)
    from IPython.nbformat import writes
    filestr = writes(nb, version=4)
    return filestr
Beispiel #28
0
def write(cells):
    """Turn cells list into valid IPython notebook code."""
    # Use IPython.nbformat functionality for writing the notebook
    from IPython.nbformat.v4 import (
        new_code_cell, new_markdown_cell, new_notebook)
    nb_cells = []

    for cell_tp, language, block in cells:
        if cell_tp == 'markdown':
            nb_cells.append(new_markdown_cell(source=block))
        elif cell_tp == 'codecell':
            nb_cells.append(new_code_cell(source=block))

    nb = new_notebook(cells=nb_cells)
    from IPython.nbformat import writes
    filestr = writes(nb, version=4)
    return filestr
Beispiel #29
0
    def test_save(self):
        resp = self.api.read('foo/a.ipynb')
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        nb.cells.append(new_markdown_cell(u'Created by test ³'))

        nbmodel= {'content': nb, 'type': 'notebook'}
        resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
        with io.open(nbfile, 'r', encoding='utf-8') as f:
            newnb = read(f, as_version=4)
        self.assertEqual(newnb.cells[0].source,
                         u'Created by test ³')
        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        newnb = from_dict(nbcontent)
        self.assertEqual(newnb.cells[0].source,
                         u'Created by test ³')
Beispiel #30
0
    def build_notebook(self):
        """Build a notebook in memory for use with preprocessor tests"""

        outputs = [
            nbformat.new_output("stream", name="stdout", text="a"),
            nbformat.new_output("display_data", data={'text/plain': 'b'}),
            nbformat.new_output("stream", name="stdout", text="c"),
            nbformat.new_output("stream", name="stdout", text="d"),
            nbformat.new_output("stream", name="stderr", text="e"),
            nbformat.new_output("stream", name="stderr", text="f"),
            nbformat.new_output("display_data", data={'image/png': 'Zw=='}), # g
            nbformat.new_output("display_data", data={'application/pdf': 'aA=='}), # h
        ]
        
        cells=[nbformat.new_code_cell(source="$ e $", execution_count=1, outputs=outputs),
               nbformat.new_markdown_cell(source="$ e $")]

        return nbformat.new_notebook(cells=cells)
Beispiel #31
0
    def test_contents_manager(self):
        "make sure ContentsManager returns right files (ipynb, bin, txt)."

        nbdir = self.notebook_dir.name
        base = self.base_url()

        nb = new_notebook(
            cells=[
                new_markdown_cell(u'Created by test ³'),
                new_code_cell("print(2*6)", outputs=[
                    new_output("stream", text="12"),
                ])
            ]
        )

        with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w', 
            encoding='utf-8') as f:
            write(nb, f, version=4)

        with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
            f.write(b'\xff' + os.urandom(5))
            f.close()

        with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
            f.write(u'foobar')
            f.close()

        r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
        self.assertEqual(r.status_code, 200)
        self.assertIn('print(2*6)', r.text)
        json.loads(r.text)

        r = requests.get(url_path_join(base, 'files', 'test.bin'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'application/octet-stream')
        self.assertEqual(r.content[:1], b'\xff')
        self.assertEqual(len(r.content), 6)

        r = requests.get(url_path_join(base, 'files', 'test.txt'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'text/plain')
        self.assertEqual(r.text, 'foobar')
Beispiel #32
0
    def test_contents_manager(self):
        "make sure ContentsManager returns right files (ipynb, bin, txt)."

        nbdir = self.notebook_dir.name
        base = self.base_url()

        nb = new_notebook(
            cells=[
                new_markdown_cell(u'Created by test ³'),
                new_code_cell("print(2*6)", outputs=[
                    new_output("stream", text="12"),
                ])
            ]
        )

        with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
                     encoding='utf-8') as f:
            write(nb, f, version=4)

        with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
            f.write(b'\xff' + os.urandom(5))
            f.close()

        with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
            f.write(u'foobar')
            f.close()

        r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
        self.assertEqual(r.status_code, 200)
        self.assertIn('print(2*6)', r.text)
        json.loads(r.text)

        r = requests.get(url_path_join(base, 'files', 'test.bin'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'application/octet-stream')
        self.assertEqual(r.content[:1], b'\xff')
        self.assertEqual(len(r.content), 6)

        r = requests.get(url_path_join(base, 'files', 'test.txt'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'text/plain')
        self.assertEqual(r.text, 'foobar')
    def setUp(self):
        nbdir = self.notebook_dir.name

        if not os.path.isdir(pjoin(nbdir, "foo")):
            os.mkdir(pjoin(nbdir, "foo"))

        nb = new_notebook()

        nb.cells.append(new_markdown_cell(u"Created by test ³"))
        cc1 = new_code_cell(source=u"print(2*6)")
        cc1.outputs.append(new_output(output_type="stream", text=u"12"))
        cc1.outputs.append(
            new_output(output_type="execute_result", data={"image/png": png_green_pixel}, execution_count=1)
        )
        nb.cells.append(cc1)

        with io.open(pjoin(nbdir, "foo", "testnb.ipynb"), "w", encoding="utf-8") as f:
            write(nb, f, version=4)

        self.nbconvert_api = NbconvertAPI(self.base_url())
    def setUp(self):
        nbdir = self.notebook_dir.name

        if not os.path.isdir(pjoin(nbdir, 'foo')):
            os.mkdir(pjoin(nbdir, 'foo'))

        nb = new_notebook()

        nb.cells.append(new_markdown_cell(u'Created by test ³'))
        cc1 = new_code_cell(source=u'print(2*6)')
        cc1.outputs.append(new_output(output_type="stream", text=u'12'))
        cc1.outputs.append(new_output(output_type="execute_result",
                                      data={'image/png': png_green_pixel},
                                      execution_count=1,
                                      ))
        nb.cells.append(cc1)

        with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
                     encoding='utf-8') as f:
            write(nb, f, version=4)

        self.nbconvert_api = NbconvertAPI(self.base_url())
Beispiel #35
0
def write(cells, nb_version=4):
    """Turn cells list into valid IPython notebook code."""
    # Use IPython.nbformat functionality for writing the notebook
    if nb_version == 3:
        from IPython.nbformat.v3 import (new_code_cell, new_text_cell,
                                         new_worksheet, new_notebook,
                                         new_metadata, new_author)
        nb = new_worksheet()

    elif nb_version == 4:
        from IPython.nbformat.v4 import (new_code_cell, new_markdown_cell,
                                         new_notebook)
        nb_cells = []

    for cell_tp, language, block in cells:
        if cell_tp == 'markdown':
            if nb_version == 3:
                nb.cells.append(new_text_cell(u'markdown', source=block))
            elif nb_version == 4:
                nb_cells.append(new_markdown_cell(source=block))
        elif cell_tp == 'codecell':
            if nb_version == 3:
                nb.cells.append(new_code_cell(input=block))
            elif nb_version == 4:
                nb_cells.append(new_code_cell(source=block))

    if nb_version == 3:
        nb = new_notebook(worksheets=[nb], metadata=new_metadata())
        # Let us make v4 notebook here by upgrading
        from IPython.nbformat.v4 import upgrade
        nb = upgrade(nb)
        import IPython.nbformat.v4.nbjson as nbjson
        # Convert nb to json format
        filestr = nbjson.writes(nb)
    elif nb_version == 4:
        nb = new_notebook(cells=nb_cells)
        from IPython.nbformat import writes
        filestr = writes(nb, version=4)
    return filestr
    def test_checkpoints(self):
        resp = self.api.read("foo/a.ipynb")
        r = self.api.new_checkpoint("foo/a.ipynb")
        self.assertEqual(r.status_code, 201)
        cp1 = r.json()
        self.assertEqual(set(cp1), {"id", "last_modified"})
        self.assertEqual(r.headers["Location"].split("/")[-1], cp1["id"])

        # Modify it
        nbcontent = json.loads(resp.text)["content"]
        nb = from_dict(nbcontent)
        hcell = new_markdown_cell("Created by test")
        nb.cells.append(hcell)
        # Save
        nbmodel = {"content": nb, "type": "notebook"}
        resp = self.api.save("foo/a.ipynb", body=json.dumps(nbmodel))

        # List checkpoints
        cps = self.api.get_checkpoints("foo/a.ipynb").json()
        self.assertEqual(cps, [cp1])

        nbcontent = self.api.read("foo/a.ipynb").json()["content"]
        nb = from_dict(nbcontent)
        self.assertEqual(nb.cells[0].source, "Created by test")

        # Restore cp1
        r = self.api.restore_checkpoint("foo/a.ipynb", cp1["id"])
        self.assertEqual(r.status_code, 204)
        nbcontent = self.api.read("foo/a.ipynb").json()["content"]
        nb = from_dict(nbcontent)
        self.assertEqual(nb.cells, [])

        # Delete cp1
        r = self.api.delete_checkpoint("foo/a.ipynb", cp1["id"])
        self.assertEqual(r.status_code, 204)
        cps = self.api.get_checkpoints("foo/a.ipynb").json()
        self.assertEqual(cps, [])
Beispiel #37
0
    def test_checkpoints(self):
        resp = self.api.read('foo/a.ipynb')
        r = self.api.new_checkpoint('foo/a.ipynb')
        self.assertEqual(r.status_code, 201)
        cp1 = r.json()
        self.assertEqual(set(cp1), {'id', 'last_modified'})
        self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id'])

        # Modify it
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        hcell = new_markdown_cell('Created by test')
        nb.cells.append(hcell)
        # Save
        nbmodel = {'content': nb, 'type': 'notebook'}
        resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        # List checkpoints
        cps = self.api.get_checkpoints('foo/a.ipynb').json()
        self.assertEqual(cps, [cp1])

        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        nb = from_dict(nbcontent)
        self.assertEqual(nb.cells[0].source, 'Created by test')

        # Restore cp1
        r = self.api.restore_checkpoint('foo/a.ipynb', cp1['id'])
        self.assertEqual(r.status_code, 204)
        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        nb = from_dict(nbcontent)
        self.assertEqual(nb.cells, [])

        # Delete cp1
        r = self.api.delete_checkpoint('foo/a.ipynb', cp1['id'])
        self.assertEqual(r.status_code, 204)
        cps = self.api.get_checkpoints('foo/a.ipynb').json()
        self.assertEqual(cps, [])
Beispiel #38
0
    def test_checkpoints(self):
        resp = self.api.read('foo/a.ipynb')
        r = self.api.new_checkpoint('foo/a.ipynb')
        self.assertEqual(r.status_code, 201)
        cp1 = r.json()
        self.assertEqual(set(cp1), {'id', 'last_modified'})
        self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id'])

        # Modify it
        nbcontent = json.loads(resp.text)['content']
        nb = from_dict(nbcontent)
        hcell = new_markdown_cell('Created by test')
        nb.cells.append(hcell)
        # Save
        nbmodel= {'content': nb, 'type': 'notebook'}
        resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))

        # List checkpoints
        cps = self.api.get_checkpoints('foo/a.ipynb').json()
        self.assertEqual(cps, [cp1])

        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        nb = from_dict(nbcontent)
        self.assertEqual(nb.cells[0].source, 'Created by test')

        # Restore cp1
        r = self.api.restore_checkpoint('foo/a.ipynb', cp1['id'])
        self.assertEqual(r.status_code, 204)
        nbcontent = self.api.read('foo/a.ipynb').json()['content']
        nb = from_dict(nbcontent)
        self.assertEqual(nb.cells, [])

        # Delete cp1
        r = self.api.delete_checkpoint('foo/a.ipynb', cp1['id'])
        self.assertEqual(r.status_code, 204)
        cps = self.api.get_checkpoints('foo/a.ipynb').json()
        self.assertEqual(cps, [])
Beispiel #39
0
    def build_notebook(self):
        """Build a notebook in memory for use with preprocessor tests"""

        outputs = [
            nbformat.new_output("stream", name="stdout", text="a"),
            nbformat.new_output("display_data", data={'text/plain': 'b'}),
            nbformat.new_output("stream", name="stdout", text="c"),
            nbformat.new_output("stream", name="stdout", text="d"),
            nbformat.new_output("stream", name="stderr", text="e"),
            nbformat.new_output("stream", name="stderr", text="f"),
            nbformat.new_output("display_data", data={'image/png':
                                                      'Zw=='}),  # g
            nbformat.new_output("display_data",
                                data={'application/pdf': 'aA=='}),  # h
        ]

        cells = [
            nbformat.new_code_cell(source="$ e $",
                                   execution_count=1,
                                   outputs=outputs),
            nbformat.new_markdown_cell(source="$ e $")
        ]

        return nbformat.new_notebook(cells=cells)
    def get(self, user, filename):
        ## filename can have a path on it
        next = "/hub/%s/public/%s" % (user, filename)
        filesystem_path = "/home/%s/Public/%s" % (user, filename)
        if os.path.isfile(filesystem_path): # download, raw, or view notebook
            command = "view"
            if len(self.get_arguments("view")) > 0:
                command = "view"
            elif len(self.get_arguments("download")) > 0:
                command = "download" 
            elif len(self.get_arguments("copy")) > 0:
                command = "copy" 
            elif len(self.get_arguments("pdf")) > 0:
                command = "pdf" 
            elif len(self.get_arguments("raw")) > 0:
                command = "raw" 
            # else:  view
            if filename.endswith(".ipynb"):
                if command in ["view", "pdf"]:
                    # first, make a notebook html:
                    #with open("/home/%s/Public/%s" % (user, filename)) as fp:
                    #    notebook_content = fp.read()
                    if command == "view":
                        exporter = HTMLExporter(template_file='full')
                    else:
                        exporter = PDFExporter(latex_count=1)
                    
                    nb_json = nbformat.read("/home/%s/Public/%s" % (user, filename), as_version=4)
                    #if command == "pdf":
                    #    # If pdf, remove heading numbering:
                    #    for cell in nb_json["worksheets"][0]["cells"]:
                    #        if cell["cell_type"] == "heading":
                    #            cell["source"] = re.sub("^([0-9]+\.?)+\s", "", cell["source"])
                    # where to get css, images?
                    if command == "pdf":
                        self.set_header('Content-Type', "application/pdf")
                        base_filename = os.path.basename(filename)
                        self.set_header('Content-Disposition', 'attachment; filename="%s"' % base_filename)
                    else: # render as HTML
                        # add header/footer:
                        path = "/hub/%s/public" % user
                        parts = [(path, path)]
                        for part in filename.split("/")[:-1]:
                            path += "/" + part
                            parts.append((path, part))
                        breadcrumbs = " / ".join(map(lambda pair: '<a href="%s" target="_blank">%s</a>' % pair, parts))
                        env = {
                            "breadcrumbs": breadcrumbs,
                            "url": path + "/" + filename + "?download"
                        }
                        cell = new_markdown_cell(source="""
<table width="100%" style="border: none;">
<tr style="border: none;">
  <td style="border: none;" width="100px">
    <img src="https://serendip.brynmawr.edu/oneworld/files/styles/thumbnail/public/pictures/SerendipStudioAvatar.png?itok=48Z_omRv"/> 
  </td>
  <td style="border: none;" width="50%">
    <h2><a href="https://serendip.brynmawr.edu/oneworld/tides/explore">TIDES: Teaching to Increase Diversity and Equity in STEM</a></h2>
  </td>
  <td style="border: none;">
        <a href="http://jupyter.physics.brynmawr.edu/hub/dblank/public/Jupyter%20Help.ipynb" title="Help">
            <span class='fa fa-info-circle fa-2x menu-icon'></span>
            <span class='menu-text'>Help</span>
        </a>
  </td>
  <td style="border: none;">
        <a href="{url}" title="Download Notebook" download>
            <span class='fa fa-download fa-2x menu-icon'></span>
            <span class='menu-text'>Download Notebook</span>
        </a>
  </td>
</tr>
<tr style="border: none;">
  <td colspan="4" style="border: none;">
      <b>Public notebooks:</b> {breadcrumbs}
  </td>
</tr>
</table>
<hr style="background-color: #534f9a; height: 5px; border: 0; ">
""".format(**env))
                        nb_json["cells"].insert(0, cell)
                    (body, resources) = exporter.from_notebook_node(nb_json)
                    self.write(body)
                elif command == "download": # download notebook json
                    self.download(user, filename, "text/plain")
                elif command == "copy": # copy notebook json, if logged in
                    if self.get_current_user_name():
                        self.copy_file(user, filename, self.get_current_user_name())
                    else:
                        self.write("Please <a href=\"/hub/login?next=%s\">login</a> to allow copy." % next)
                else: # raw, just get file contents
                    with open("/home/%s/Public/%s" % (user, filename), "rb") as fp:
                        self.write(fp.read())
            else: # some other kind of file
                # FIXME: how to get all of custom stuff?
                if command == "copy":
                    if self.get_current_user_name():
                        self.copy_file(user, filename, self.get_current_user_name())
                    else:
                        self.write("Please <a href=\"/hub/login?next=%s\">login</a> to allow copy." % next)
                else: # whatever, just get or download it
                    base_filename = os.path.basename(filename)
                    base, ext = os.path.splitext(base_filename)
                    app_log.info("extension is: %s" % ext)
                    if base_filename == "custom.css":
                        file_path = "/home/%s/.ipython/profile_default/static/custom/custom.css" % user
                        self.set_header('Content-Type', "text/css")
                        with open(file_path, "rb") as fp:
                            self.write(fp.read())
                    elif ext in [".txt", ".html", ".js", ".css", ".pdf", ".gif", ".jpeg", ".jpg", ".png"]: # show in browser
                        app_log.info("mime: %s" % str(mimetypes.guess_type(filename)[0]))
                        self.set_header('Content-Type', mimetypes.guess_type(filename)[0])
                        with open("/home/%s/Public/%s" % (user, filename), "rb") as fp:
                            self.write(fp.read())
                    else:
                        self.download(user, filename)
        else: # not a file; directory listing
            # filename can have a full path, and might be empty
            url_path = "/hub/%s/public" % user
            # could be: images, images/, images/subdir, images/subdir/
            if not filename.endswith("/") and filename.strip() != "":
                filename += "/"
            files = glob.glob("/home/%s/Public/%s*" % (user, filename))
            self.write("<h1>Jupyter Project at Bryn Mawr College</h1>\n")
            self.write("[<a href=\"/hub/login\">Home</a>] ")
            if self.get_current_user_name():
                self.write("[<a href=\"/user/%(current_user)s/tree\">%(current_user)s</a>] " % {"current_user": self.get_current_user_name()})
            self.write("<p/>\n")
            self.write("<p>Public files for <b>/hub/%s/public/%s</b>:</p>\n" % (user, filename))
            self.write("<ol>\n")
            for absolute_filename in sorted(files):
                if os.path.isdir(absolute_filename): 
                    dir_path = absolute_filename.split("/")
                    dir_name = dir_path[-1]
                    public_path = "/".join(dir_path[dir_path.index("Public") + 1:])
                    self.write("<li><a href=\"%(url_path)s/%(public_path)s\">%(dir_name)s</a></li>\n" % {"url_path": url_path, 
                                                                                                      "dir_name": dir_name,
                                                                                                      "public_path": public_path})
                else:
                    file_path, filename = absolute_filename.rsplit("/", 1)
                    dir_path = absolute_filename.split("/")
                    public_path = "/".join(dir_path[dir_path.index("Public") + 1:])
                    variables = {"user": user, "filename": filename, "url_path": url_path, "next": next,
                                 "public_path": public_path}
                    if filename.endswith(".ipynb"):
                        if self.get_current_user_name():
                            self.write(("<li><a href=\"%(url_path)s/%(public_path)s\">%(filename)s</a> "+
                                        "(<a href=\"%(url_path)s/%(public_path)s?raw\">raw</a>, " +
                                        "<a href=\"%(url_path)s/%(public_path)s?download\">download</a>, " +
                                        "<a href=\"%(url_path)s/%(public_path)s?copy\">copy</a>" +
                                        ((", <a href=\"/user/%s/notebooks/Public/%s\">edit</a>" % (user, filename)) if self.get_current_user_name() == user else ", edit") +
                                        ")</li>\n") % variables)
                        else:
                            self.write(("<li><a href=\"%(url_path)s/%(public_path)s\">%(filename)s</a> "+
                                        "(<a href=\"%(url_path)s/%(public_path)s?raw\">raw</a>, " +
                                        "<a href=\"%(url_path)s/%(public_path)s?download\">download</a>, " +
                                        "copy, edit) " +
                                        "[<a href=\"/hub/login?next=%(next)s\">login</a> to copy]</li>\n") % variables)
                    else:
                        # some other kind of file (eg, .zip, .css):
                        self.write("<li><a href=\"%(url_path)s/%(public_path)s\">%(filename)s</a></li>\n" % variables)
            self.write("</ol>\n")
            self.write("<hr>\n")
            self.write("<p><i>Please see <a href=\"/hub/dblank/public/Jupyter Help.ipynb\">Jupyter Help</a> for more information about this server.</i></p>\n")
Beispiel #41
0
import base64, os, re, sys
import IPython.nbformat.v4 as nbf

filename = os.path.splitext(sys.argv[1])[0]

try:
    title, description = open("{}.txt".format(filename), encoding="utf-8").read().split('\n\n', 1)
except IOError:
    title, description = filename, ""
description = description.replace("...", "").replace("'''", "**").replace("''", "*")
bendpattern = re.compile("^!+", re.MULTILINE)
bendcode = '<img src="http://pyx.sourceforge.net/bend.png" align="left">'
description = re.sub(bendpattern, lambda m: bendcode*(m.end()-m.start()), description)
code = open("{}.py".format(filename), encoding="utf-8").read()
code = re.sub('\.writeEPSfile\(("[a-z]+")?\)\n.*writePDFfile\(("[a-z]+")?\)\n.*writeSVGfile\(("[a-z]+")?\)\n', "", code)

nb = nbf.new_notebook()
cells = []
cells.append(nbf.new_markdown_cell(source="# " + title))
cells.append(nbf.new_code_cell(source=code, execution_count=1,
                               outputs=[nbf.new_output(output_type=u'execute_result', execution_count=1,
                                                       data={'image/png': base64.encodebytes(open("{}.png".format(filename), "rb").read()).decode("ascii"),
                                                             'image/svg+xml': open("{}.svg".format(filename), "r", encoding="utf-8").read()})]))
cells.append(nbf.new_markdown_cell(source=description))
nb = nbf.new_notebook(cells=cells, metadata={'language': 'python'})
open("{}.ipynb".format(filename), "w").write(nbf.writes(nb))
Beispiel #42
0
def create_text_cell():
    source = "this is the answer!\n"
    cell = new_markdown_cell(source=source)
    return cell
Beispiel #43
0
def ipynb_code(filestr, code_blocks, code_block_types, tex_blocks, format):
    """
    # We expand all newcommands now
    from html import embed_newcommands
    newcommands = embed_newcommands(filestr)
    if newcommands:
        filestr = newcommands + filestr
    """
    # Fix pandoc citations to normal internal links: [[key]](#key)
    filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr)

    # filestr becomes json list after this function so we must typeset
    # envirs here. All envirs are typeset as pandoc_quote.
    from common import _CODE_BLOCK, _MATH_BLOCK
    envir_format = option('ipynb_admon=', 'paragraph')
    # Remove all !bpop-!epop environments (they cause only problens and
    # have no use)
    for envir in 'pop', 'slidecell':
        filestr = re.sub('^<!-- !b%s .*\n' % envir,
                         '',
                         filestr,
                         flags=re.MULTILINE)
        filestr = re.sub('^<!-- !e%s .*\n' % envir,
                         '',
                         filestr,
                         flags=re.MULTILINE)
    filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n',
                     '',
                     filestr,
                     flags=re.DOTALL | re.MULTILINE)
    filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE)
    from doconce import doconce_envirs
    envirs = doconce_envirs()[8:-2]
    for envir in envirs:
        pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir)
        if envir_format in ('quote', 'paragraph', 'hrule'):

            def subst(m):
                title = m.group(1).strip()
                # Text size specified in parenthesis?
                m2 = re.search('^\s*\((.+?)\)', title)

                if title == '' and envir not in ('block', 'quote'):
                    title = envir.capitalize() + '.'
                elif title.lower() == 'none':
                    title == ''
                elif m2:
                    text_size = m2.group(1).lower()
                    title = title.replace('(%s)' % text_size, '').strip()
                elif title and title[-1] not in ('.', ':', '!', '?'):
                    # Make sure the title ends with puncuation
                    title += '.'
                # Recall that this formatting is called very late
                # so native format must be used!
                if title:
                    title = '**' + title + '**\n'
                    # Could also consider subsubsection formatting
                block = m.group(2)

                # Always use quote typesetting for quotes
                if envir_format == 'quote' or envir == 'quote':
                    # Make Markdown quote of the block: lines start with >
                    lines = []
                    for line in block.splitlines():
                        # Just quote plain text
                        if not (_MATH_BLOCK in line or _CODE_BLOCK in line
                                or line.startswith('FIGURE:')
                                or line.startswith('MOVIE:')
                                or line.startswith('|')):
                            lines.append('> ' + line)
                        else:
                            lines.append('\n' + line + '\n')
                    block = '\n'.join(lines) + '\n\n'

                    # Add quote and a blank line after title
                    if title:
                        title = '> ' + title + '>\n'
                else:
                    # Add a blank line after title
                    if title:
                        title += '\n'

                if envir_format == 'hrule':
                    # Native ------ does not work, use <hr/>
                    #text = '\n\n----------\n' + title + '----------\n' + \
                    #       block + '\n----------\n\n'
                    text = '\n\n<hr/>\n' + title + \
                           block + '\n<hr/>\n\n'
                else:
                    text = title + block + '\n\n'
                return text
        else:
            print '*** error: --ipynb_admon=%s is not supported' % envir_format
        filestr = re.sub(pattern,
                         subst,
                         filestr,
                         flags=re.DOTALL | re.MULTILINE)

    # Fix pyshell and ipy interactive sessions: remove prompt and output.
    # or split in multiple cells such that output comes out at the end of a cell
    # Fix sys environments and use run prog.py so programs can be run in cell
    # Insert %matplotlib inline in the first block using matplotlib
    # Only typeset Python code as blocks, otherwise !bc environmens
    # become plain indented Markdown.
    from doconce import dofile_basename
    from sets import Set
    ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename
    src_paths = Set()
    mpl_inline = False

    split_pyshell = option('ipynb_split_pyshell=', 'on')
    if split_pyshell is None:
        split_pyshell = False
    elif split_pyshell in ('no', 'False', 'off'):
        split_pyshell = False
    else:
        split_pyshell = True

    ipynb_code_tp = [None] * len(code_blocks)
    for i in range(len(code_blocks)):
        # Check if continuation lines are in the code block, because
        # doconce.py inserts a blank after the backslash
        if '\\ \n' in code_blocks[i]:
            code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n')

        if not mpl_inline and (
            re.search(r'import +matplotlib', code_blocks[i]) or \
            re.search(r'from +matplotlib', code_blocks[i]) or \
            re.search(r'import +scitools', code_blocks[i]) or \
            re.search(r'from +scitools', code_blocks[i])):
            code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i]
            mpl_inline = True

        tp = code_block_types[i]
        if tp.endswith('-t'):
            # Standard Markdown code with pandoc/github extension
            language = tp[:-2]
            language_spec = language2pandoc.get(language, '')
            #code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n'
            code_blocks[i] = "```%s\n" % language_spec + \
                             indent_lines(code_blocks[i].strip(), format) + \
                             "```"
            ipynb_code_tp[i] = 'markdown'
        elif tp.startswith('pyshell') or tp.startswith('ipy'):
            lines = code_blocks[i].splitlines()
            last_cell_end = -1
            if split_pyshell:
                new_code_blocks = []
                # Split for each output an put in separate cell
                for j in range(len(lines)):
                    if lines[j].startswith('>>>') or lines[j].startswith(
                            '... '):
                        lines[j] = lines[j][4:]
                    elif lines[j].startswith('In ['):  # IPython
                        lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
                    elif lines[j].startswith('   ...: '):  # IPython
                        lines[j] = lines[j][8:]
                    else:
                        # output (no prefix or Out)
                        lines[j] = ''
                        new_code_blocks.append('\n'.join(lines[last_cell_end +
                                                               1:j + 1]))
                        last_cell_end = j
                code_blocks[i] = new_code_blocks
                ipynb_code_tp[i] = 'cell'
            else:
                # Remove prompt and output lines; leave code executable in cell
                for j in range(len(lines)):
                    if lines[j].startswith('>>> ') or lines[j].startswith(
                            '... '):
                        lines[j] = lines[j][4:]
                    elif lines[j].startswith('In ['):
                        lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
                    else:
                        # output
                        lines[j] = ''

                for j in range(lines.count('')):
                    lines.remove('')
                code_blocks[i] = '\n'.join(lines)
                ipynb_code_tp[i] = 'cell'

        elif tp.startswith('sys'):
            # Do we find execution of python file? If so, copy the file
            # to separate subdir and make a run file command in a cell.
            # Otherwise, it is just a plain verbatim Markdown block.
            found_unix_lines = False
            lines = code_blocks[i].splitlines()
            for j in range(len(lines)):
                m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)',
                              lines[j])
                if m:
                    name = m.group(2).strip()
                    if os.path.isfile(name):
                        src_paths.add(os.path.dirname(name))
                        lines[j] = '%%run "%s"' % fullpath
                else:
                    found_unix_lines = True
            src_paths = list(src_paths)
            if src_paths and not found_unix_lines:
                # This is a sys block with run commands only
                code_blocks[i] = '\n'.join(lines)
                ipynb_code_tp[i] = 'cell'
            else:
                # Standard Markdown code
                code_blocks[i] = '\n'.join(lines)
                code_blocks[i] = indent_lines(code_blocks[i], format)
                ipynb_code_tp[i] = 'markdown'
        elif tp.endswith('hid'):
            ipynb_code_tp[i] = 'cell_hidden'
        elif tp.startswith('py'):
            ipynb_code_tp[i] = 'cell'
        else:
            # Should support other languages as well, but not for now
            code_blocks[i] = indent_lines(code_blocks[i], format)
            ipynb_code_tp[i] = 'markdown'

    # figure_files and movie_files are global variables and contain
    # all figures and movies referred to
    src_paths = list(src_paths)
    if figure_files:
        src_paths += figure_files
    if movie_files:
        src_paths += movie_files

    if src_paths:
        # Make tar file with all the source dirs with files
        # that need to be executed
        os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths)))
        print 'collected all required additional files in', ipynb_tarfile, 'which must be distributed with the notebook'
    elif os.path.isfile(ipynb_tarfile):
        os.remove(ipynb_tarfile)

    # Parse document into markdown text, code blocks, and tex blocks.
    # Store in nested list notebook_blocks.
    notebook_blocks = [[]]
    authors = ''
    for line in filestr.splitlines():
        if line.startswith('authors = [new_author(name='):  # old author method
            authors = line[10:]
        elif _CODE_BLOCK in line:
            code_block_tp = line.split()[-1]
            if code_block_tp in (
                    'pyhid', ) or not code_block_tp.endswith('hid'):
                notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
                notebook_blocks.append(line)
            # else: hidden block to be dropped (may include more languages
            # with time in the above tuple)
        elif _MATH_BLOCK in line:
            notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
            notebook_blocks.append(line)
        else:
            if not isinstance(notebook_blocks[-1], list):
                notebook_blocks.append([])
            notebook_blocks[-1].append(line)
    if isinstance(notebook_blocks[-1], list):
        notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()

    # Add block type info
    pattern = r'(\d+) +%s'
    for i in range(len(notebook_blocks)):
        if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]):
            m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i])
            idx = int(m.group(1))
            if ipynb_code_tp[idx] == 'cell':
                notebook_blocks[i] = ['cell', notebook_blocks[i]]
            elif ipynb_code_tp[idx] == 'cell_hidden':
                notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]]
            else:
                notebook_blocks[i] = ['text', notebook_blocks[i]]
        elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]):
            notebook_blocks[i] = ['math', notebook_blocks[i]]
        else:
            notebook_blocks[i] = ['text', notebook_blocks[i]]

    # Go through tex_blocks and wrap in $$
    # (doconce.py runs align2equations so there are no align/align*
    # environments in tex blocks)
    label2tag = {}
    tag_counter = 1
    for i in range(len(tex_blocks)):
        # Extract labels and add tags
        labels = re.findall(r'label\{(.+?)\}', tex_blocks[i])
        for label in labels:
            label2tag[label] = tag_counter
            # Insert tag to get labeled equation
            tex_blocks[i] = tex_blocks[i].replace(
                'label{%s}' % label,
                'label{%s} \\tag{%s}' % (label, tag_counter))
            tag_counter += 1

        # Remove \[ and \] or \begin/end{equation*} in single equations
        tex_blocks[i] = tex_blocks[i].replace(r'\[', '')
        tex_blocks[i] = tex_blocks[i].replace(r'\]', '')
        tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '')
        tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '')
        # Check for illegal environments
        m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i])
        if m:
            envir = m.group(1)
            if envir not in ('equation', 'equation*', 'align*', 'align',
                             'array'):
                print """\
*** warning: latex envir \\begin{%s} does not work well in Markdown.
    Stick to \\[ ... \\], equation, equation*, align, or align*
    environments in math environments.
""" % envir
        eq_type = 'heading'  # or '$$'
        eq_type = '$$'
        # Markdown: add $$ on each side of the equation
        if eq_type == '$$':
            # Make sure there are no newline after equation
            tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$'
        # Here: use heading (###) and simple formula (remove newline
        # in math expressions to keep everything within a heading) as
        # the equation then looks bigger
        elif eq_type == 'heading':
            tex_blocks[i] = '### $ ' + '  '.join(
                tex_blocks[i].splitlines()) + ' $'

        # Add labels for the eqs above the block (for reference)
        if labels:
            #label_tp = '<a name="%s"></a>'
            label_tp = '<div id="%s"></div>'
            tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \
                            ' '.join([label_tp % label
                                      for label in labels]) + '\n\n' + \
                                      tex_blocks[i]

    # blocks is now a list of text chunks in markdown and math/code line
    # instructions. Insert code and tex blocks
    for i in range(len(notebook_blocks)):
        if _CODE_BLOCK in notebook_blocks[i][
                1] or _MATH_BLOCK in notebook_blocks[i][1]:
            words = notebook_blocks[i][1].split()
            # start of notebook_blocks[i]: number block-indicator code-type
            n = int(words[0])
            if _CODE_BLOCK in notebook_blocks[i][1]:
                notebook_blocks[i][1] = code_blocks[n]  # can be list!
            if _MATH_BLOCK in notebook_blocks[i][1]:
                notebook_blocks[i][1] = tex_blocks[n]

    # Make IPython structures

    nb_version = int(option('ipynb_version=', '3'))
    if nb_version == 3:
        from IPython.nbformat.v3 import (new_code_cell, new_text_cell,
                                         new_worksheet, new_notebook,
                                         new_metadata, new_author)
        nb = new_worksheet()
    elif nb_version == 4:
        from IPython.nbformat.v4 import (new_code_cell, new_markdown_cell,
                                         new_notebook)
        cells = []

    mdstr = []  # plain md format of the notebook
    prompt_number = 1
    for block_tp, block in notebook_blocks:
        if (block_tp == 'text' or block_tp == 'math') and block != '':
            # Pure comments between math/code and math/code come
            # out as empty blocks, should detect that situation
            # (challenging - can have multiple lines of comments,
            # or begin and end comment lines with important things between)
            if nb_version == 3:
                nb.cells.append(new_text_cell(u'markdown', source=block))
            elif nb_version == 4:
                cells.append(new_markdown_cell(source=block))
            mdstr.append(('markdown', block))
        elif block_tp == 'cell' and block != '' and block != []:
            if isinstance(block, list):
                for block_ in block:
                    block_ = block_.rstrip()
                    if block_ != '':
                        if nb_version == 3:
                            nb.cells.append(
                                new_code_cell(input=block_,
                                              prompt_number=prompt_number,
                                              collapsed=False))
                        elif nb_version == 4:
                            cells.append(
                                new_code_cell(source=block_,
                                              execution_count=prompt_number))
                        prompt_number += 1
                        mdstr.append(('codecell', block_))
            else:
                block = block.rstrip()
                if block != '':
                    if nb_version == 3:
                        nb.cells.append(
                            new_code_cell(input=block,
                                          prompt_number=prompt_number,
                                          collapsed=False))
                    elif nb_version == 4:
                        cells.append(
                            new_code_cell(source=block,
                                          execution_count=prompt_number))
                    prompt_number += 1
                    mdstr.append(('codecell', block))
        elif block_tp == 'cell_hidden' and block != '':
            block = block.rstrip()
            if nb_version == 3:
                nb.cells.append(
                    new_code_cell(input=block,
                                  prompt_number=prompt_number,
                                  collapsed=True))
            elif nb_version == 4:
                cells.append(
                    new_code_cell(source=block, execution_count=prompt_number))
            prompt_number += 1
            mdstr.append(('codecell', block))
    """
    # Dump the notebook cells in a simple ASCII format
    # (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file)
    f = open(dofile_basename + '.md-ipynb', 'w')
    for cell_tp, block in mdstr:
        if cell_tp == 'markdown':
            f.write('\n-----\n\n')
        elif cell_tp == 'codecell':
            f.write('\n-----py\n\n')
        f.write(block)
    f.close()
    """

    if nb_version == 3:
        # Catch the title as the first heading
        m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE)
        title = m.group(1).strip() if m else ''
        # md below is not used for anything
        if authors:
            authors = eval(authors)
            md = new_metadata(name=title, authors=authors)
        else:
            md = new_metadata(name=title)
        nb = new_notebook(worksheets=[nb], metadata=new_metadata())
        # Let us make v4 notebook here by upgrading
        from IPython.nbformat.v4 import upgrade
        nb = upgrade(nb)
        import IPython.nbformat.v4.nbjson as nbjson

        # Convert nb to json format
        filestr = nbjson.writes(nb)
    elif nb_version == 4:
        nb = new_notebook(cells=cells)
        from IPython.nbformat import writes
        filestr = writes(nb, version=4)

    # Check that there are no empty cells:
    if '"input": []' in filestr:
        print '*** error: empty cells in notebook - report bug in DocOnce'
        _abort()
    # must do the replacements here at the very end when json is written out
    # \eqref and labels will not work, but labels (only in math) do no harm
    filestr = re.sub(r'([^\\])label\{',
                     r'\g<1>\\\\label{',
                     filestr,
                     flags=re.MULTILINE)

    # \\eqref{} just gives (???) link at this stage - future versions
    # will probably support labels
    #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr)
    # Now we use explicit references to tags
    def subst(m):
        label = m.group(1)
        try:
            return r'[(%s)](#%s)' % (label2tag[label], label)
        except KeyError as e:
            print '*** error: label "%s" is not defined' % str(e)

    filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr)
    """
    # MathJax reference to tag (recall that the equations have both label
    # and tag (know that tag only works well in HTML, but this mjx-eqn-no
    # label does not work in ipynb)
    filestr = re.sub(r'\(ref\{(.+?)\}\)',
                     lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr)
    """
    #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr)
    '''
    # Final fixes: replace all text between cells by markdown code cells
    # Note: the patterns are overlapping so a plain re.sub will not work,
    # here we run through all blocks found and subsitute the first remaining
    # one, one by one.
    pattern = r'   \},\n(.+?)\{\n    "cell_type":'
    begin_pattern = r'^(.+?)\{\n    "cell_type":'
    remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL)
    remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL)
    import string
    for block in remaining_block_begin + remaining_blocks:
        filestr = string.replace(filestr, block, json_markdown(block) + '   ',
                                 maxreplace=1)
    filestr_end = re.sub(r'   \{\n    "cell_type": .+?\n   \},\n', '', filestr,
                         flags=re.DOTALL)
    filestr = filestr.replace(filestr_end, json_markdown(filestr_end))
    filestr = """{
 "metadata": {
  "name": "SOME NAME"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
""" + filestr.rstrip() + '\n'+ \
    json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """
   ],
   "metadata": {}
  }
 ]
}"""
    '''
    return filestr
Beispiel #44
0
except IOError:
    title, description = filename, ""
description = description.replace("...", "").replace("'''",
                                                     "**").replace("''", "*")
bendpattern = re.compile("^!+", re.MULTILINE)
bendcode = '<img src="http://pyx.sourceforge.net/bend.png" align="left">'
description = re.sub(bendpattern, lambda m: bendcode * (m.end() - m.start()),
                     description)
code = open("{}.py".format(filename), encoding="utf-8").read()
code = re.sub(
    '\.writeEPSfile\(("[a-z]+")?\)\n.*writePDFfile\(("[a-z]+")?\)\n.*writeSVGfile\(("[a-z]+")?\)\n',
    "", code)

nb = nbf.new_notebook()
cells = []
cells.append(nbf.new_markdown_cell(source="# " + title))
cells.append(
    nbf.new_code_cell(source=code,
                      execution_count=1,
                      outputs=[
                          nbf.new_output(
                              output_type=u'execute_result',
                              execution_count=1,
                              data={
                                  'image/png':
                                  base64.encodebytes(
                                      open("{}.png".format(filename),
                                           "rb").read()).decode("ascii"),
                                  'image/svg+xml':
                                  open("{}.svg".format(filename),
                                       "r",
    def post(self, convert=False):
        # Check if this is a convert operation
        if convert:
            # Get JSON payload
            json_data = tornado.escape.json_decode(self.request.body)
            # Go through all the assignments / bundles in the project payload
            for x in json_data['bundles']:
                # Create a notebook version4 object
                nb = nbfv4.new_notebook()
                # Create the task description
                task_description = '## Task \n' +  x['description'] + '\n'\
                                    '___ \n' \
                                    '#### '+ x['owner'] + '\n' \
                                    '___ \n'
                # Create the general description string
                common_description ='#### Temporary folder \n' \
                                    'Set your working dir to following folder '+ json_data['gid']+'. Upload your csv/data files into '\
                                    'this directoy to use them.<br/>'\
                                    '`ftp://pycard.ifi.uzh.ch/data/'+json_data['gid']+'`'\
                                    '<br/><br/>'\
                                    'Use with R Kernel <br/>' \
                                    '`setwd("./'+ json_data['gid']+'")` <br/><br/>' \
                                    'Use with Python Kernel <br/> ' \
                                    '`import os` <br/>' \
                                    '`os.chdir("./'+ json_data['gid']+'")` \n' \
                                    '___ \n' \
                                    '#### Notes board \n' \
                                    'In order to avoid conflicts between notebooks and have a clean transition from one step to another, use the shared notes file ' \
                                    'shared.txt . The contents of the file will be loaded and made in every notebook, so it is a good place to register variable names used in the different steps, or to provide feedback after each iteration.  <br/><br/>'
                # Add the task_description as a markdown cell
                heading = nbfv4.new_markdown_cell(task_description)
                # Set the task_description as read-only
                heading['metadata'].run_control = dict()
                heading['metadata'].run_control['read_only'] = True
                # Append cell to notebook
                nb['cells'].append(heading)

                # Add the common description cell as a markdown cell
                common = nbfv4.new_markdown_cell(common_description)
                # Set the common description cell as read only
                common['metadata'].run_control = dict()
                common['metadata']['common'] = True
                common['metadata'].run_control['read_only'] = True
                # Add the cell to the notebook
                nb['cells'].append(common)

                # Create a markdown cell for the note board, set the variable_cell metadata to true
                variablesh = nbfv4.new_markdown_cell()
                variablesh['metadata']['variable_cell'] = True
                nb['cells'].append(variablesh)

                # Set the notebook kernel in metadata
                nb['metadata']['language'] = json_data['kernel']
                # Set cell toolbar to Side Comments in metadata
                nb['metadata']['celltoolbar'] = "Side Comments"
                # Set project ID in metadata
                nb['metadata']['pgid'] = json_data['gid']
                # Set id of notes board for this project (shared_notes.txt file)
                nb['metadata']['variablesid'] = json_data['variablesid']
                # Set Google ID for this notebook
                nb['metadata']['id'] = x['gid']
                # Set the worker assigned to this task
                nb['metadata']['bundle-owner'] = x['owner']

                # Go through all the actions in the assignment
                for a in x['actions']:
                    # Create action description text
                    text = '#### This is the description of the actions that need to be implemented.' \
                    '\n' \
                    '### ' + a['name'] + '\n' \
                    'Description:   ' + a['description'] + '<br>' \
                    'Input:         ' + a['input'] + '<br>' \
                    'Output:        ' + a['output']

                    code = "# Enter implementation here."
                    # Add the description cell as a markdown cell, set it read-only
                    desc = nbfv4.new_markdown_cell(text)
                    desc['metadata'].run_control = dict()
                    desc['metadata'].run_control['read_only'] = True
                    nb['cells'].append(desc)

                    # Create a cell for logging the work
                    log_text = '# Log and description \n' \
                               'Please record here all information needed to reproduce and understand your work: \n' \
                               '- Algorithms used\n' \
                               '- Things tried but discarded\n' \
                               '- Explanation **why** you have solved the problem the way you solved it.\n'

                    # Add the description cell as a markdown cell, set it read-only
                    log_desc = nbfv4.new_markdown_cell(log_text)
                    nb['cells'].append(log_desc)

                    # Create the cell code for this action
                    code_cell = nbfv4.new_code_cell(code)
                    code_cell['metadata'].side_comments = dict()

                    # Create the cell code for this action, set the section id needed
                    # for the SideComments extension as metadata

                    code_cell['metadata'].side_comments['id'] = ''.join(
                        random.SystemRandom().choice(string.ascii_uppercase +
                                                     string.digits)
                        for _ in range(20))
                    logger.info(code_cell['metadata'])
                    # Add cell to notebook
                    nb['cells'].append(code_cell)
                # Add the contents of the created notebook a the bundle {notebook} property in the project payload
                x['notebook'] = nbf.writes(nb, version=4)

            json_data['variables'] = html2text.html2text(
                json_data['variables'])

            # Send the answer back to client
            self.write(json.dumps(json_data, cls=Encoder))
            self.set_status(201)

        else:
            # Get the project payload from the client
            json_data = tornado.escape.json_decode(self.request.body)
            # Add the project to the database
            ret = db.addProject(json_data)
            if ret['ok'] == 1.0:
                # If project was added, create temporary work folder for the project
                # which can be accessed through ftp
                if not os.path.exists('/ftp/ipython/data/' + json_data['gid']):
                    os.makedirs('/ftp/ipython/data/' + json_data['gid'])
                    os.chmod('/ftp/ipython/data/' + json_data['gid'], 0o755)
                self.set_status(201)
    def post(self, convert=False):
        # Check if this is a convert operation
        if convert:
            # Get JSON payload
            json_data = tornado.escape.json_decode(self.request.body)
            # Go through all the assignments / bundles in the project payload
            for x in json_data['bundles']:
                # Create a notebook version4 object
                nb = nbfv4.new_notebook()
                # Create the task description
                task_description = '## Task \n' +  x['description'] + '\n'\
                                    '___ \n' \
                                    '#### '+ x['owner'] + '\n' \
                                    '___ \n'
                # Create the general description string
                common_description ='#### Temporary folder \n' \
                                    'Set your working dir to following folder '+ json_data['gid']+'. Upload your csv/data files into '\
                                    'this directoy to use them.<br/>'\
                                    '`ftp://pycard.ifi.uzh.ch/data/'+json_data['gid']+'`'\
                                    '<br/><br/>'\
                                    'Use with R Kernel <br/>' \
                                    '`setwd("./'+ json_data['gid']+'")` <br/><br/>' \
                                    'Use with Python Kernel <br/> ' \
                                    '`import os` <br/>' \
                                    '`os.chdir("./'+ json_data['gid']+'")` \n' \
                                    '___ \n' \
                                    '#### Notes board \n' \
                                    'In order to avoid conflicts between notebooks and have a clean transition from one step to another, use the shared notes file ' \
                                    'shared.txt . The contents of the file will be loaded and made in every notebook, so it is a good place to register variable names used in the different steps, or to provide feedback after each iteration.  <br/><br/>'
                # Add the task_description as a markdown cell
                heading = nbfv4.new_markdown_cell(task_description)
                # Set the task_description as read-only
                heading['metadata'].run_control = dict()
                heading['metadata'].run_control['read_only'] = True
                # Append cell to notebook
                nb['cells'].append(heading)

                # Add the common description cell as a markdown cell
                common = nbfv4.new_markdown_cell(common_description)
                # Set the common description cell as read only
                common['metadata'].run_control = dict()
                common['metadata']['common'] = True
                common['metadata'].run_control['read_only'] = True
                # Add the cell to the notebook
                nb['cells'].append(common)

                # Create a markdown cell for the note board, set the variable_cell metadata to true
                variablesh = nbfv4.new_markdown_cell()
                variablesh['metadata']['variable_cell'] = True
                nb['cells'].append(variablesh)

                # Set the notebook kernel in metadata
                nb['metadata']['language'] = json_data['kernel']
                # Set cell toolbar to Side Comments in metadata
                nb['metadata']['celltoolbar'] = "Side Comments"
                # Set project ID in metadata
                nb['metadata']['pgid'] = json_data['gid']
                # Set id of notes board for this project (shared_notes.txt file)
                nb['metadata']['variablesid'] = json_data['variablesid']
                # Set Google ID for this notebook
                nb['metadata']['id'] = x['gid']

                # Go through all the actions in the assignment
                for a in x['actions']:
                    # Create action description text
                    text = '#### This is the description of the actions that need to be implemented.' \
                    '\n' \
                    '### ' + a['name'] + '\n' \
                    'Description:   ' + a['description'] + '<br>' \
                    'Input:         ' + a['input'] + '<br>' \
                    'Output:        ' + a['output']

                    code = "# Enter implementation here."
                    # Add the description cell as a markdown cell, set it read-only
                    desc = nbfv4.new_markdown_cell(text)
                    desc['metadata'].run_control = dict()
                    desc['metadata'].run_control['read_only'] = True
                    nb['cells'].append(desc)

                    # Create the cell code for this action
                    code_cell = nbfv4.new_code_cell(code)
                    code_cell['metadata'].side_comments = dict()

                    # Create the cell code for this action, set the section id needed
                    # for the SideComments extension as metadata

                    code_cell['metadata'].side_comments['id'] = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(20))
                    logger.info(code_cell['metadata'])
                    # Add cell to notebook
                    nb['cells'].append(code_cell)
                # Add the contents of the created notebook a the bundle {notebook} property in the project payload
                x['notebook'] = nbf.writes(nb,version=4)


            json_data['variables'] = html2text.html2text(json_data['variables'])

            # Send the answer back to client
            self.write(json.dumps(json_data, cls=Encoder))
            self.set_status(201)

        else:
            # Get the project payload from the client
            json_data = tornado.escape.json_decode(self.request.body)
            # Add the project to the database
            ret = db.addProject(json_data)
            if ret['ok'] == 1.0:
                # If project was added, create temporary work folder for the project
                # which can be accessed through ftp
                if not os.path.exists('/ftp/ipython/data/' + json_data['gid']):
                    os.makedirs('/ftp/ipython/data/' + json_data['gid'])
                    os.chmod('/ftp/ipython/data/' + json_data['gid'], 0o755)
                self.set_status(201)
Beispiel #47
0
 def _create_text_cell():
     source = "this is the answer!\n"
     cell = new_markdown_cell(source=source)
     return cell
Beispiel #48
0
def ipynb_code(filestr, code_blocks, code_block_types,
               tex_blocks, format):
    """
    # We expand all newcommands now
    from html import embed_newcommands
    newcommands = embed_newcommands(filestr)
    if newcommands:
        filestr = newcommands + filestr
    """
    # Fix pandoc citations to normal internal links: [[key]](#key)
    filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr)

    # filestr becomes json list after this function so we must typeset
    # envirs here. All envirs are typeset as pandoc_quote.
    from common import _CODE_BLOCK, _MATH_BLOCK
    envir_format = option('ipynb_admon=', 'paragraph')
    # Remove all !bpop-!epop environments (they cause only problens and
    # have no use)
    for envir in 'pop', 'slidecell':
        filestr = re.sub('^<!-- !b%s .*\n' % envir, '', filestr,
                         flags=re.MULTILINE)
        filestr = re.sub('^<!-- !e%s .*\n' % envir, '', filestr,
                         flags=re.MULTILINE)
    filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n', '', filestr,
                     flags=re.DOTALL|re.MULTILINE)
    filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE)
    from doconce import doconce_envirs
    envirs = doconce_envirs()[8:-2]
    for envir in envirs:
        pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir)
        if envir_format in ('quote', 'paragraph', 'hrule'):
            def subst(m):
                title = m.group(1).strip()
                # Text size specified in parenthesis?
                m2 = re.search('^\s*\((.+?)\)', title)

                if title == '' and envir not in ('block', 'quote'):
                    title = envir.capitalize() + '.'
                elif title.lower() == 'none':
                    title == ''
                elif m2:
                    text_size = m2.group(1).lower()
                    title = title.replace('(%s)' % text_size, '').strip()
                elif title and title[-1] not in ('.', ':', '!', '?'):
                    # Make sure the title ends with puncuation
                    title += '.'
                # Recall that this formatting is called very late
                # so native format must be used!
                if title:
                    title = '**' + title + '**\n'
                    # Could also consider subsubsection formatting
                block = m.group(2)

                # Always use quote typesetting for quotes
                if envir_format == 'quote' or envir == 'quote':
                    # Make Markdown quote of the block: lines start with >
                    lines = []
                    for line in block.splitlines():
                        # Just quote plain text
                        if not (_MATH_BLOCK in line or
                                _CODE_BLOCK in line or
                                line.startswith('FIGURE:') or
                                line.startswith('MOVIE:') or
                                line.startswith('|')):
                            lines.append('> ' + line)
                        else:
                            lines.append('\n' + line + '\n')
                    block = '\n'.join(lines) + '\n\n'

                    # Add quote and a blank line after title
                    if title:
                        title = '> ' + title + '>\n'
                else:
                    # Add a blank line after title
                    if title:
                        title += '\n'

                if envir_format == 'hrule':
                    # Native ------ does not work, use <hr/>
                    #text = '\n\n----------\n' + title + '----------\n' + \
                    #       block + '\n----------\n\n'
                    text = '\n\n<hr/>\n' + title + \
                           block + '\n<hr/>\n\n'
                else:
                    text = title + block + '\n\n'
                return text
        else:
            print '*** error: --ipynb_admon=%s is not supported'  % envir_format
        filestr = re.sub(pattern, subst, filestr,
                         flags=re.DOTALL | re.MULTILINE)

    # Fix pyshell and ipy interactive sessions: remove prompt and output.
    # or split in multiple cells such that output comes out at the end of a cell
    # Fix sys environments and use run prog.py so programs can be run in cell
    # Insert %matplotlib inline in the first block using matplotlib
    # Only typeset Python code as blocks, otherwise !bc environmens
    # become plain indented Markdown.
    from doconce import dofile_basename
    from sets import Set
    ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename
    src_paths = Set()
    mpl_inline = False

    split_pyshell = option('ipynb_split_pyshell=', 'on')
    if split_pyshell is None:
        split_pyshell = False
    elif split_pyshell in ('no', 'False', 'off'):
        split_pyshell = False
    else:
        split_pyshell = True

    ipynb_code_tp = [None]*len(code_blocks)
    for i in range(len(code_blocks)):
        # Check if continuation lines are in the code block, because
        # doconce.py inserts a blank after the backslash
        if '\\ \n' in code_blocks[i]:
            code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n')

        if not mpl_inline and (
            re.search(r'import +matplotlib', code_blocks[i]) or \
            re.search(r'from +matplotlib', code_blocks[i]) or \
            re.search(r'import +scitools', code_blocks[i]) or \
            re.search(r'from +scitools', code_blocks[i])):
            code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i]
            mpl_inline = True

        tp = code_block_types[i]
        if tp.endswith('-t'):
            # Standard Markdown code with pandoc/github extension
            language = tp[:-2]
            language_spec = language2pandoc.get(language, '')
            #code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n'
            code_blocks[i] = "```%s\n" % language_spec + \
                             indent_lines(code_blocks[i].strip(), format) + \
                             "```"
            ipynb_code_tp[i] = 'markdown'
        elif tp.startswith('pyshell') or tp.startswith('ipy'):
            lines = code_blocks[i].splitlines()
            last_cell_end = -1
            if split_pyshell:
                new_code_blocks = []
                # Split for each output an put in separate cell
                for j in range(len(lines)):
                    if lines[j].startswith('>>>') or lines[j].startswith('... '):
                        lines[j] = lines[j][4:]
                    elif lines[j].startswith('In ['):  # IPython
                        lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
                    elif lines[j].startswith('   ...: '): # IPython
                        lines[j] = lines[j][8:]
                    else:
                        # output (no prefix or Out)
                        lines[j] = ''
                        new_code_blocks.append(
                            '\n'.join(lines[last_cell_end+1:j+1]))
                        last_cell_end = j
                code_blocks[i] = new_code_blocks
                ipynb_code_tp[i] = 'cell'
            else:
                # Remove prompt and output lines; leave code executable in cell
                for j in range(len(lines)):
                    if lines[j].startswith('>>> ') or lines[j].startswith('... '):
                        lines[j] = lines[j][4:]
                    elif lines[j].startswith('In ['):
                        lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
                    else:
                        # output
                        lines[j] = ''

                for j in range(lines.count('')):
                    lines.remove('')
                code_blocks[i] = '\n'.join(lines)
                ipynb_code_tp[i] = 'cell'

        elif tp.startswith('sys'):
            # Do we find execution of python file? If so, copy the file
            # to separate subdir and make a run file command in a cell.
            # Otherwise, it is just a plain verbatim Markdown block.
            found_unix_lines = False
            lines = code_blocks[i].splitlines()
            for j in range(len(lines)):
                m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)',
                              lines[j])
                if m:
                    name = m.group(2).strip()
                    if os.path.isfile(name):
                        src_paths.add(os.path.dirname(name))
                        lines[j] = '%%run "%s"' % fullpath
                else:
                    found_unix_lines = True
            src_paths = list(src_paths)
            if src_paths and not found_unix_lines:
                # This is a sys block with run commands only
                code_blocks[i] = '\n'.join(lines)
                ipynb_code_tp[i] = 'cell'
            else:
                # Standard Markdown code
                code_blocks[i] = '\n'.join(lines)
                code_blocks[i] = indent_lines(code_blocks[i], format)
                ipynb_code_tp[i] = 'markdown'
        elif tp.endswith('hid'):
            ipynb_code_tp[i] = 'cell_hidden'
        elif tp.startswith('py'):
            ipynb_code_tp[i] = 'cell'
        else:
            # Should support other languages as well, but not for now
            code_blocks[i] = indent_lines(code_blocks[i], format)
            ipynb_code_tp[i] = 'markdown'

    # figure_files and movie_files are global variables and contain
    # all figures and movies referred to
    src_paths = list(src_paths)
    if figure_files:
        src_paths += figure_files
    if movie_files:
        src_paths += movie_files

    if src_paths:
        # Make tar file with all the source dirs with files
        # that need to be executed
        os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths)))
        print 'collected all required additional files in', ipynb_tarfile, 'which must be distributed with the notebook'
    elif os.path.isfile(ipynb_tarfile):
        os.remove(ipynb_tarfile)


    # Parse document into markdown text, code blocks, and tex blocks.
    # Store in nested list notebook_blocks.
    notebook_blocks = [[]]
    authors = ''
    for line in filestr.splitlines():
        if line.startswith('authors = [new_author(name='):  # old author method
            authors = line[10:]
        elif _CODE_BLOCK in line:
            code_block_tp = line.split()[-1]
            if code_block_tp in ('pyhid',) or not code_block_tp.endswith('hid'):
                notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
                notebook_blocks.append(line)
            # else: hidden block to be dropped (may include more languages
            # with time in the above tuple)
        elif _MATH_BLOCK in line:
            notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
            notebook_blocks.append(line)
        else:
            if not isinstance(notebook_blocks[-1], list):
                notebook_blocks.append([])
            notebook_blocks[-1].append(line)
    if isinstance(notebook_blocks[-1], list):
        notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()


    # Add block type info
    pattern = r'(\d+) +%s'
    for i in range(len(notebook_blocks)):
        if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]):
            m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i])
            idx = int(m.group(1))
            if ipynb_code_tp[idx] == 'cell':
                notebook_blocks[i] = ['cell', notebook_blocks[i]]
            elif ipynb_code_tp[idx] == 'cell_hidden':
                notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]]
            else:
                notebook_blocks[i] = ['text', notebook_blocks[i]]
        elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]):
            notebook_blocks[i] = ['math', notebook_blocks[i]]
        else:
            notebook_blocks[i] = ['text', notebook_blocks[i]]

    # Go through tex_blocks and wrap in $$
    # (doconce.py runs align2equations so there are no align/align*
    # environments in tex blocks)
    label2tag = {}
    tag_counter = 1
    for i in range(len(tex_blocks)):
        # Extract labels and add tags
        labels = re.findall(r'label\{(.+?)\}', tex_blocks[i])
        for label in labels:
            label2tag[label] = tag_counter
            # Insert tag to get labeled equation
            tex_blocks[i] = tex_blocks[i].replace(
                'label{%s}' % label, 'label{%s} \\tag{%s}' % (label, tag_counter))
            tag_counter += 1

        # Remove \[ and \] or \begin/end{equation*} in single equations
        tex_blocks[i] = tex_blocks[i].replace(r'\[', '')
        tex_blocks[i] = tex_blocks[i].replace(r'\]', '')
        tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '')
        tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '')
        # Check for illegal environments
        m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i])
        if m:
            envir = m.group(1)
            if envir not in ('equation', 'equation*', 'align*', 'align',
                             'array'):
                print """\
*** warning: latex envir \\begin{%s} does not work well in Markdown.
    Stick to \\[ ... \\], equation, equation*, align, or align*
    environments in math environments.
""" % envir
        eq_type = 'heading'  # or '$$'
        eq_type = '$$'
        # Markdown: add $$ on each side of the equation
        if eq_type == '$$':
            # Make sure there are no newline after equation
            tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$'
        # Here: use heading (###) and simple formula (remove newline
        # in math expressions to keep everything within a heading) as
        # the equation then looks bigger
        elif eq_type == 'heading':
            tex_blocks[i] = '### $ ' + '  '.join(tex_blocks[i].splitlines()) + ' $'

        # Add labels for the eqs above the block (for reference)
        if labels:
            #label_tp = '<a name="%s"></a>'
            label_tp = '<div id="%s"></div>'
            tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \
                            ' '.join([label_tp % label
                                      for label in labels]) + '\n\n' + \
                                      tex_blocks[i]

    # blocks is now a list of text chunks in markdown and math/code line
    # instructions. Insert code and tex blocks
    for i in range(len(notebook_blocks)):
        if _CODE_BLOCK in notebook_blocks[i][1] or _MATH_BLOCK in notebook_blocks[i][1]:
            words = notebook_blocks[i][1].split()
            # start of notebook_blocks[i]: number block-indicator code-type
            n = int(words[0])
            if _CODE_BLOCK in notebook_blocks[i][1]:
                notebook_blocks[i][1] = code_blocks[n]  # can be list!
            if _MATH_BLOCK in notebook_blocks[i][1]:
                notebook_blocks[i][1] = tex_blocks[n]

    # Make IPython structures

    nb_version = int(option('ipynb_version=', '3'))
    if nb_version == 3:
        from IPython.nbformat.v3 import (
            new_code_cell, new_text_cell, new_worksheet,
            new_notebook, new_metadata, new_author)
        nb = new_worksheet()
    elif nb_version == 4:
        from IPython.nbformat.v4 import (
            new_code_cell, new_markdown_cell, new_notebook)
        cells = []

    mdstr = []  # plain md format of the notebook
    prompt_number = 1
    for block_tp, block in notebook_blocks:
        if (block_tp == 'text' or block_tp == 'math') and block != '':
            # Pure comments between math/code and math/code come
            # out as empty blocks, should detect that situation
            # (challenging - can have multiple lines of comments,
            # or begin and end comment lines with important things between)
            if nb_version == 3:
                nb.cells.append(new_text_cell(u'markdown', source=block))
            elif nb_version == 4:
                cells.append(new_markdown_cell(source=block))
            mdstr.append(('markdown', block))
        elif block_tp == 'cell' and block != '' and block != []:
            if isinstance(block, list):
                for block_ in block:
                    block_ = block_.rstrip()
                    if block_ != '':
                        if nb_version == 3:
                            nb.cells.append(new_code_cell(
                                input=block_,
                                prompt_number=prompt_number,
                                collapsed=False))
                        elif nb_version == 4:
                            cells.append(new_code_cell(
                                source=block_,
                                execution_count=prompt_number))
                        prompt_number += 1
                        mdstr.append(('codecell', block_))
            else:
                block = block.rstrip()
                if block != '':
                    if nb_version == 3:
                        nb.cells.append(new_code_cell(
                            input=block,
                            prompt_number=prompt_number,
                            collapsed=False))
                    elif nb_version == 4:
                        cells.append(new_code_cell(
                            source=block,
                            execution_count=prompt_number))
                    prompt_number += 1
                    mdstr.append(('codecell', block))
        elif block_tp == 'cell_hidden' and block != '':
            block = block.rstrip()
            if nb_version == 3:
                nb.cells.append(new_code_cell(
                    input=block, prompt_number=prompt_number, collapsed=True))
            elif nb_version == 4:
                cells.append(new_code_cell(
                    source=block, execution_count=prompt_number))
            prompt_number += 1
            mdstr.append(('codecell', block))

    """
    # Dump the notebook cells in a simple ASCII format
    # (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file)
    f = open(dofile_basename + '.md-ipynb', 'w')
    for cell_tp, block in mdstr:
        if cell_tp == 'markdown':
            f.write('\n-----\n\n')
        elif cell_tp == 'codecell':
            f.write('\n-----py\n\n')
        f.write(block)
    f.close()
    """

    if nb_version == 3:
        # Catch the title as the first heading
        m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE)
        title = m.group(1).strip() if m else ''
        # md below is not used for anything
        if authors:
            authors = eval(authors)
            md = new_metadata(name=title, authors=authors)
        else:
            md = new_metadata(name=title)
        nb = new_notebook(worksheets=[nb], metadata=new_metadata())
        # Let us make v4 notebook here by upgrading
        from IPython.nbformat.v4 import upgrade
        nb = upgrade(nb)
        import IPython.nbformat.v4.nbjson as nbjson

        # Convert nb to json format
        filestr = nbjson.writes(nb)
    elif nb_version == 4:
        nb = new_notebook(cells=cells)
        from IPython.nbformat import writes
        filestr = writes(nb, version=4)

    # Check that there are no empty cells:
    if '"input": []' in filestr:
        print '*** error: empty cells in notebook - report bug in DocOnce'
        _abort()
    # must do the replacements here at the very end when json is written out
    # \eqref and labels will not work, but labels (only in math) do no harm
    filestr = re.sub(r'([^\\])label\{', r'\g<1>\\\\label{', filestr,
                     flags=re.MULTILINE)
    # \\eqref{} just gives (???) link at this stage - future versions
    # will probably support labels
    #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr)
    # Now we use explicit references to tags
    def subst(m):
        label = m.group(1)
        try:
            return r'[(%s)](#%s)' % (label2tag[label], label)
        except KeyError as e:
            print '*** error: label "%s" is not defined' % str(e)

    filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr)
    """
    # MathJax reference to tag (recall that the equations have both label
    # and tag (know that tag only works well in HTML, but this mjx-eqn-no
    # label does not work in ipynb)
    filestr = re.sub(r'\(ref\{(.+?)\}\)',
                     lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr)
    """
    #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr)

    '''
    # Final fixes: replace all text between cells by markdown code cells
    # Note: the patterns are overlapping so a plain re.sub will not work,
    # here we run through all blocks found and subsitute the first remaining
    # one, one by one.
    pattern = r'   \},\n(.+?)\{\n    "cell_type":'
    begin_pattern = r'^(.+?)\{\n    "cell_type":'
    remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL)
    remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL)
    import string
    for block in remaining_block_begin + remaining_blocks:
        filestr = string.replace(filestr, block, json_markdown(block) + '   ',
                                 maxreplace=1)
    filestr_end = re.sub(r'   \{\n    "cell_type": .+?\n   \},\n', '', filestr,
                         flags=re.DOTALL)
    filestr = filestr.replace(filestr_end, json_markdown(filestr_end))
    filestr = """{
 "metadata": {
  "name": "SOME NAME"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
""" + filestr.rstrip() + '\n'+ \
    json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """
   ],
   "metadata": {}
  }
 ]
}"""
    '''
    return filestr