Пример #1
3
def lines_to_notebook(lines, name=None):
    """
    Convert the lines of an m file into an IPython notebook

    Parameters
    ----------
    lines : list
        A list of strings. Each element is a line in the m file

    Returns
    -------
    notebook : an IPython NotebookNode class instance, containing the
    information required to create a file


    """
    source = []
    md = np.empty(len(lines), dtype=object)
    new_cell = np.empty(len(lines), dtype=object)
    for idx, l in enumerate(lines):
        new_cell[idx], md[idx], this_source = format_line(l)
        # Transitions between markdown and code and vice-versa merit a new
        # cell, even if no newline, or "%%" is found. Make sure not to do this
        # check for the very first line!
        if idx>1 and not new_cell[idx]:
            if md[idx] != md[idx-1]:
                new_cell[idx] = True

        source.append(this_source)
    # This defines the breaking points between cells:
    new_cell_idx = np.hstack([np.where(new_cell)[0], -1])

    # Listify the sources:
    cell_source = [source[new_cell_idx[i]:new_cell_idx[i+1]]
                   for i in range(len(new_cell_idx)-1)]
    cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx)-1)]
    cells = []

    # Append the notebook with loading matlab magic extension
    notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
                    + "pymat.load_ipython_extension(ip)"
    cells.append(nbformat.new_code_cell(notebook_head, language='python'))

    for cell_idx, cell_s in enumerate(cell_source):
        if cell_md[cell_idx]:
            cells.append(nbformat.new_text_cell('markdown', cell_s))
        else:
            cell_s.insert(0, '%%matlab\n')
            cells.append(nbformat.new_code_cell(cell_s, language='matlab'))

    ws = nbformat.new_worksheet(cells=cells)
    notebook = nbformat.new_notebook(metadata=nbformat.new_metadata(),
                                 worksheets=[ws])
    return notebook
Пример #2
0
    def build_notebook(self):
        """Build a reveal slides notebook in memory for use with tests.  
        Overrides base in TransformerTestsBase"""

        outputs = [
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="a")
        ]

        slide_metadata = {'slideshow': {'slide_type': 'slide'}}
        subslide_metadata = {'slideshow': {'slide_type': 'subslide'}}

        cells = [
            nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs),
            nbformat.new_text_cell('markdown',
                                   source="",
                                   metadata=slide_metadata),
            nbformat.new_code_cell(input="", prompt_number=2, outputs=outputs),
            nbformat.new_text_cell('markdown',
                                   source="",
                                   metadata=slide_metadata),
            nbformat.new_text_cell('markdown',
                                   source="",
                                   metadata=subslide_metadata)
        ]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
Пример #3
0
def convert(py_file, ipynb_file):
    cells = []
    imports = ""  # needed for each file

    for cell, line_type, line in parse_blocks(py_file):
        if line_type == file_type:
            # write cell to file
            try:
                fname = line.split()[1]
            except:
                raise Exception(
                    "Markdown for file output must be the "
                    + "following format\n#to_file filename.py: "
                    + "{}\n".format(line)
                )
                sys.exit(0)

            with open(fname, "w") as f:
                f.write(cell)
                new_cell = "%run {}".format(fname)
                ipynb_cell = nbf.new_code_cell(new_cell)
                cells.append(ipynb_cell)

        else:
            # convert cell to ipynb cell
            ipynb_cell = nbf.new_code_cell(cell)
            cells.append(ipynb_cell)

    # create new notebook
    nb = nbf.new_notebook()
    nb["worksheets"].append(nbf.new_worksheet(cells=cells))

    with open(ipynb_file, "w") as f:
        nbf.write(nb, f, "ipynb")
 def test_match_tests_double_test(self):
     """Is an error raised when a test id is used twice?"""
     cell1 = new_code_cell()
     cell1.metadata = dict(assignment=dict(cell_type="grade", id="foo", points=""))
     cell2 = new_code_cell()
     cell2.metadata = dict(assignment=dict(cell_type="autograder", id="foo_test"))
     cell3 = new_text_cell("markdown")
     cell3.metadata = dict(assignment=dict(cell_type="autograder", id="foo_test"))
     cells = [cell1, cell2, cell3]
     assert_raises(RuntimeError, self.preprocessor._match_tests, cells)
Пример #5
0
def test_basic_notebook_merge():
    notebook = nbformat.new_notebook()
    code_cell = nbformat.new_code_cell(input=['a', 'b'])
    notebook['worksheets'] = [{'cells': [code_cell]}]
    notebook2 = nbformat.new_notebook()
    notebook3 = nbformat.new_notebook()
    code_cell = nbformat.new_code_cell(input=['a', 'b'])
    notebook3['worksheets'] = [{'cells': [code_cell]}]
    result = notebook_merge(notebook, notebook2, notebook3)
    result_cells = result['worksheets'][0]['cells']
    state = result_cells[0]['metadata']['state']
    assert state == 'added'
 def generateCells(self, crossTabList, grpFieldString, datasetName,
                   grpField):
     for fieldList in crossTabList:
         fieldListString, fieldListDfString, k_anon_size = self.generateFieldListStrings(
             fieldList, "fieldListString", "fieldList")
         crossStuff = grpField + "\n" + fieldListString + "\n" + fieldListDfString + "\n" + k_anon_size + "\n"
         crossStuff = crossStuff + "crossBy, single_cell_info = makeCrossTabInfo( grpField, grpFieldList, fieldList, fieldListString,  datasetName, k_anon_size)"
         crossStuff = crossStuff + "\n" + "single_cell_info"
         cell = nbf.new_code_cell(crossStuff)
         self.cells.append(cell)
         cell = nbf.new_code_cell("crossBy")
         self.cells.append(cell)
Пример #7
0
 def visit_literal_block(self, node):
     raw_text = node.astext()
     #only include lines that begin with <<<<
     #we want the example code and not the example output
     processed_text = '\n'.join([line.split('>>>')[1][1:] for line in raw_text.split('\n') if line.startswith('>>>')]) 
     c = nbformat.new_code_cell(input=processed_text)
     self.add_cell(c)
Пример #8
0
    def _create_code_cell():
        source = """print("something")
### BEGIN SOLUTION
print("hello")
### END SOLUTION"""
        cell = new_code_cell(input=source)
        return cell
Пример #9
0
    def build_notebook(self):
        """Build a notebook in memory for use with preprocessor tests"""

        outputs = [
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="a"),
            nbformat.new_output(output_type="text", output_text="b"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="c"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="d"),
            nbformat.new_output(output_type="stream",
                                stream="stderr",
                                output_text="e"),
            nbformat.new_output(output_type="stream",
                                stream="stderr",
                                output_text="f"),
            nbformat.new_output(output_type="png", output_png=b'Zw==')
        ]  #g

        cells = [
            nbformat.new_code_cell(input="$ e $",
                                   prompt_number=1,
                                   outputs=outputs),
            nbformat.new_text_cell('markdown', source="$ e $")
        ]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
Пример #10
0
def test_basic_notebook_merge():
    notebook = nbformat.new_notebook()
    code_cell = nbformat.new_code_cell(input=['a', 'b'])
    notebook['worksheets'] = [
        {'cells': [code_cell]}
    ]
    notebook2 = nbformat.new_notebook()
    notebook3 = nbformat.new_notebook()
    code_cell = nbformat.new_code_cell(input=['a', 'b'])
    notebook3['worksheets'] = [
        {'cells': [code_cell]}
    ]
    result = notebook_merge(notebook, notebook2, notebook3)
    result_cells = result['worksheets'][0]['cells']
    state = result_cells[0]['metadata']['state']
    assert state == 'added'
Пример #11
0
    def test_coalesce_sequenced_streams(self):
        """Can the coalesce streams preprocessor merge a sequence of streams?"""

        outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
                   nbformat.new_output(
                       output_type="stream", stream="stdout", output_text="1"),
                   nbformat.new_output(
                       output_type="stream", stream="stdout", output_text="2"),
                   nbformat.new_output(
                       output_type="stream", stream="stdout", output_text="3"),
                   nbformat.new_output(
                       output_type="stream", stream="stdout", output_text="4"),
                   nbformat.new_output(
                       output_type="stream", stream="stdout", output_text="5"),
                   nbformat.new_output(
                       output_type="stream", stream="stdout", output_text="6"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
        cells = [nbformat.new_code_cell(
            input="# None", prompt_number=1, outputs=outputs)]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
        res = self.build_resources()
        nb, res = coalesce_streams(nb, res)
        outputs = nb.worksheets[0].cells[0].outputs
        self.assertEqual(outputs[0].text, u'01234567')
Пример #12
0
 def add_code_cell(self, nb):
     output = current.new_output("display_data",
                                 output_javascript="alert('hi');")
     cell = current.new_code_cell("print('hi')", outputs=[output])
     if not nb.worksheets:
         nb.worksheets.append(current.new_worksheet())
     nb.worksheets[0].cells.append(cell)
Пример #13
0
    def test_coalesce_replace_streams(self):
        """Are \\r characters handled?"""
        outputs = [
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="z"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="\ra"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="\nz\rb"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="\nz"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="\rc\n"),
            nbformat.new_output(output_type="stream",
                                stream="stdout",
                                output_text="z\rz\rd")
        ]
        cells = [
            nbformat.new_code_cell(input="# None",
                                   prompt_number=1,
                                   outputs=outputs)
        ]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
        res = self.build_resources()
        nb, res = coalesce_streams(nb, res)
        outputs = nb.worksheets[0].cells[0].outputs
        self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
Пример #14
0
    def build_notebook(self):
        """Build a reveal slides notebook in memory for use with tests.  
        Overrides base in TransformerTestsBase"""

        outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="a")]
        
        slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
        subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}

        cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs),
               nbformat.new_text_cell('markdown', source="", metadata=slide_metadata),
               nbformat.new_code_cell(input="", prompt_number=2, outputs=outputs),
               nbformat.new_text_cell('markdown', source="", metadata=slide_metadata),
               nbformat.new_text_cell('markdown', source="", metadata=subslide_metadata)]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
    def _create_code_cell():
        source = """# YOUR CODE HERE
{% if solution %}
print "hello"
{% endif %}
"""
        cell = new_code_cell(input=source, prompt_number=2, outputs=["foo"])
        return cell
 def test_replace_test_source_code(self):
     """Is the code source properly replaced?"""
     cell = new_code_cell("\n", metadata=dict(
         assignment=dict(id="test1_for_problem1")))
     self.preprocessor._load_autograder_tests_file()
     self.preprocessor._replace_test_source(cell)
     assert cell.input == "# blah blah blah"
     assert cell.metadata['assignment']['weight'] == 0.3333333333333333
     assert cell.metadata['assignment']['points'] == 1
 def test_get_score_ok(self):
     """Is the score zero when there was no error?"""
     cell = new_code_cell("\n", metadata=dict(
         assignment=dict(id="test1_for_problem1")))
     cell.outputs = []
     self.preprocessor._load_autograder_tests_file()
     score, total_score = self.preprocessor._get_score(cell)
     assert score == 1
     assert total_score == 1
 def test_get_score_error(self):
     """Is the score zero when there was an error?"""
     cell = new_code_cell("\n", metadata=dict(
         assignment=dict(id="test1_for_problem1")))
     cell.outputs = [NotebookNode()]
     cell.outputs[0]['output_type'] = 'pyerr'
     self.preprocessor._load_autograder_tests_file()
     score, total_score = self.preprocessor._get_score(cell)
     assert score == 0
     assert total_score == 1
Пример #19
0
 def visit_literal_block(self, node):
     raw_text = node.astext()
     #only include lines that begin with >>>
     #we want the example code and not the example output
     processed_text = '\n'.join([
         line.split('>>>')[1][1:] for line in raw_text.split('\n')
         if line.startswith('>>>')
     ])
     c = nbformat.new_code_cell(input=processed_text)
     self.add_cell(c)
Пример #20
0
    def build_notebook(self):
        """Build a reveal slides notebook in memory for use with tests.  
        Overrides base in TransformerTestsBase"""

        outputs = [nbformat.new_output(output_type="svg", output_svg=self.simple_svg)]
        
        slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
        subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}

        cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs)]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
Пример #21
0
    def build_notebook(self):
        """Build a reveal slides notebook in memory for use with tests.  
        Overrides base in PreprocessorTestsBase"""

        outputs = [nbformat.new_output(output_type="svg", output_svg=self.simple_svg)]
        
        slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
        subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}

        cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs)]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
Пример #22
0
    def test_empty_code_cell(self):
        """No empty code cells in rst"""
        nbname = self._get_notebook()
        with io.open(nbname, encoding='utf8') as f:
            nb = current.read(f, 'json')

        exporter = self.exporter_class()

        (output, resources) = exporter.from_notebook_node(nb)
        # add an empty code cell
        nb.worksheets[0].cells.append(current.new_code_cell(input=""))
        (output2, resources) = exporter.from_notebook_node(nb)
        # adding an empty code cell shouldn't change output
        self.assertEqual(output.strip(), output2.strip())
    def generateCrossTabsReport(self, datasetName, src_dir, fnName,
                                crossTabList, grpFieldString):
        dfcell = 'df, titleStuff = makeOutput(datasetName, src_dir , fnName )'
        grpFieldString, grpField = self.generateGrpListStrings(
            grpFieldString, "grpFieldList", "grpField")
        stuff = '''
datasetName = "%s"
src_dir = '%s'
fnName = '%s'
        ''' % (datasetName, src_dir, fnName)
        stuff = stuff + "\n" + grpFieldString + "\n" + dfcell + "\n" + "titleStuff"
        cell = nbf.new_code_cell(stuff)
        self.cells.append(cell)
        self.generateCells(crossTabList, grpFieldString, datasetName, grpField)
Пример #24
0
    def test_run_nb(self):
        """Test %run notebook.ipynb"""
        from IPython.nbformat import current
        nb = current.new_notebook(worksheets=[
            current.new_worksheet(cells=[
                current.new_text_cell("The Ultimate Question of Everything"),
                current.new_code_cell("answer=42")
            ])
        ])
        src = current.writes(nb, 'json')
        self.mktmp(src, ext='.ipynb')

        _ip.magic("run %s" % self.fname)

        nt.assert_equal(_ip.user_ns['answer'], 42)
Пример #25
0
    def build_notebook(self):
        """Build a notebook in memory for use with preprocessor tests"""

        outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="a"),
                   nbformat.new_output(output_type="text", output_text="b"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="c"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="d"),
                   nbformat.new_output(output_type="stream", stream="stderr", output_text="e"),
                   nbformat.new_output(output_type="stream", stream="stderr", output_text="f"),
                   nbformat.new_output(output_type="png", output_png='Zw==')] #g
        
        cells=[nbformat.new_code_cell(input="$ e $", prompt_number=1,outputs=outputs),
               nbformat.new_text_cell('markdown', source="$ e $")]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
Пример #26
0
    def test_coalesce_replace_streams(self):
        """Are \\r characters handled?"""
        outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="z"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="\ra"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="\nz\rb"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="\nz"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="\rc\n"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="z\rz\rd")]
        cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]

        nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
        res = self.build_resources()
        nb, res = coalesce_streams(nb, res)
        outputs = nb.worksheets[0].cells[0].outputs
        self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
Пример #27
0
 def test_empty_code_cell(self):
     """No empty code cells in rst"""
     nbname = self._get_notebook()
     with io.open(nbname, encoding='utf8') as f:
         nb = current.read(f, 'json')
     
     exporter = self.exporter_class()
     
     (output, resources) = exporter.from_notebook_node(nb)
     # add an empty code cell
     nb.worksheets[0].cells.append(
         current.new_code_cell(input="")
     )
     (output2, resources) = exporter.from_notebook_node(nb)
     # adding an empty code cell shouldn't change output
     self.assertEqual(output.strip(), output2.strip())
Пример #28
0
 def test_run_nb(self):
     """Test %run notebook.ipynb"""
     from IPython.nbformat import current
     nb = current.new_notebook(
         worksheets=[
             current.new_worksheet(cells=[
                 current.new_code_cell("answer=42")
             ])
         ]
     )
     src = current.writes(nb, 'json')
     self.mktmp(src, ext='.ipynb')
     
     _ip.magic("run %s" % self.fname)
     
     nt.assert_equal(_ip.user_ns['answer'], 42)
Пример #29
0
def write_notebook(flow, options):
    """See http://nbviewer.ipython.org/gist/fperez/9716279"""
    from IPython.nbformat import current as nbf
    nb = nbf.new_notebook()

    cells = [
        #nbf.new_text_cell('heading', "This is an auto-generated notebook for %s" % os.path.basename(pseudopath)),
        nbf.new_code_cell("""\
##%%javascript
##IPython.OutputArea.auto_scroll_threshold = 9999;

from __future__ import print_function
from abipy import abilab
%matplotlib inline
mpld3 = abilab.mpld3_enable_notebook()

import pylab
pylab.rcParams['figure.figsize'] = (25.0, 10.0)
import seaborn as sns
#sns.set(style="dark", palette="Set2")
sns.set(style='ticks', palette='Set2')"""),

        nbf.new_code_cell("flow = abilab.Flow.pickle_load('%s')" % flow.workdir),
        nbf.new_code_cell("flow.show_dependencies()"),
        nbf.new_code_cell("flow.check_status(show=True, verbose=0)"),
        nbf.new_code_cell("flow.show_inputs(nids=None, wslice=None)"),
        nbf.new_code_cell("flow.inspect(nids=None, wslice=None)"),
        nbf.new_code_cell("flow.show_abierrors()"),
        nbf.new_code_cell("flow.show_qouts()"),
    ]

    # Now that we have the cells, we can make a worksheet with them and add it to the notebook:
    nb['worksheets'].append(nbf.new_worksheet(cells=cells))

    # Next, we write it to a file on disk that we can then open as a new notebook.
    # Note: This should be as easy as: nbf.write(nb, fname), but the current api is a little more verbose and needs a real file-like object.
    import tempfile
    _, tmpfname = tempfile.mkstemp(suffix='.ipynb', text=True)

    with open(tmpfname, 'w') as fh:
        nbf.write(nb, fh, 'ipynb')

    os.system("ipython notebook %s" % tmpfname)
Пример #30
0
def to_notebook(infile, hr_separated=False):
    """Given markdown, returns an ipynb compliant JSON string"""

    parser = markdown.DocParser()
    ast = json.loads(markdown.ASTtoJSON(
        parser.parse(infile.read())))

    cells = [current.new_text_cell('markdown', '')]

    for block in ast.get('children', []):
        if block['t'] in ["IndentedCode", "FencedCode"]:
            cells.append(current.new_code_cell(
                block['string_content'].rstrip()
            ))
        elif block['t'] in ['SetextHeader', 'ATXHeader']:
            src = '{} {}'.format(
                '#' * block.get('level', 1),
                ''.join(block['strings'])
            ).rstrip()
            if hr_separated and cells[-1]['cell_type'] is 'markdown':
                cells[-1]['source'] += '\n\n{}'.format(src)
            else:
                cells.append(current.new_text_cell('markdown', src))
        elif block['t'] in ['HorizontalRule']:
            # We don't render horizontal rules
            if hr_separated:
                cells.append(current.new_text_cell('markdown', ''))
        else:
            src = '\n'.join(block['strings']).rstrip()
            if hr_separated and cells[-1]['cell_type'] is 'markdown':
                cells[-1]['source'] += '\n\n{}'.format(src)
            else:
                cells.append(current.new_text_cell('markdown', src))

    cells = tidy_notebook(cells[:])

    worksheet = current.new_worksheet(cells=cells)

    nb = current.new_notebook(
        basename(infile.name).split('.')[:-1],
        worksheets=[worksheet]
    )

    # using the indent option leaves a bunch of trailing whitespace. No thanks!
    return json.dumps(nb, indent=2).replace(' \n', '\n')
Пример #31
0
    def test_coalesce_sequenced_streams(self):
        """Can the coalesce streams preprocessor merge a sequence of streams?"""
        outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="1"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="2"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="3"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="4"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="5"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="6"),
                   nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
        cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
        worksheets = [nbformat.new_worksheet(cells=cells)]

        nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
        res = self.build_resources()
        nb, res = coalesce_streams(nb, res)
        outputs = nb.worksheets[0].cells[0].outputs
        self.assertEqual(outputs[0].text, u'01234567')
Пример #32
0
    def notebook(self, s):
        """Export and convert IPython notebooks.

        This function can export the current IPython history to a notebook file
        or can convert an existing notebook file into a different format. For
        example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
        To export the history to "foo.py" do "%notebook -e foo.py". To convert
        "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
        formats include (json/ipynb, py).
        """
        args = magic_arguments.parse_argstring(self.notebook, s)

        from IPython.nbformat import current
        args.filename = unquote_filename(args.filename)
        if args.export:
            fname, name, format = current.parse_filename(args.filename)
            cells = []
            hist = list(self.shell.history_manager.get_range())
            for session, prompt_number, input in hist[:-1]:
                cells.append(
                    current.new_code_cell(prompt_number=prompt_number,
                                          input=input))
            worksheet = current.new_worksheet(cells=cells)
            nb = current.new_notebook(name=name, worksheets=[worksheet])
            with io.open(fname, 'w', encoding='utf-8') as f:
                current.write(nb, f, format)
        elif args.format is not None:
            old_fname, old_name, old_format = current.parse_filename(
                args.filename)
            new_format = args.format
            if new_format == u'xml':
                raise ValueError('Notebooks cannot be written as xml.')
            elif new_format == u'ipynb' or new_format == u'json':
                new_fname = old_name + u'.ipynb'
                new_format = u'json'
            elif new_format == u'py':
                new_fname = old_name + u'.py'
            else:
                raise ValueError('Invalid notebook format: %s' % new_format)
            with io.open(old_fname, 'r', encoding='utf-8') as f:
                nb = current.read(f, old_format)
            with io.open(new_fname, 'w', encoding='utf-8') as f:
                current.write(nb, f, new_format)
Пример #33
0
    def test_contents_manager(self):
        "make sure ContentsManager returns right files (ipynb, bin, txt)."

        nbdir = self.notebook_dir.name
        base = self.base_url()

        nb = new_notebook(name='testnb')
        
        ws = new_worksheet()
        nb.worksheets = [ws]
        ws.cells.append(new_heading_cell(u'Created by test ³'))
        cc1 = new_code_cell(input=u'print(2*6)')
        cc1.outputs.append(new_output(output_text=u'12', output_type='stream'))
        ws.cells.append(cc1)

        with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w', 
            encoding='utf-8') as f:
            write(nb, f, format='ipynb')

        with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
            f.write(b'\xff' + os.urandom(5))
            f.close()

        with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
            f.write(u'foobar')
            f.close()

        r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
        self.assertEqual(r.status_code, 200)
        self.assertIn('print(2*6)', r.text)
        json.loads(r.text)

        r = requests.get(url_path_join(base, 'files', 'test.bin'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'application/octet-stream')
        self.assertEqual(r.content[:1], b'\xff')
        self.assertEqual(len(r.content), 6)

        r = requests.get(url_path_join(base, 'files', 'test.txt'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'text/plain')
        self.assertEqual(r.text, 'foobar')
    def __init__(self):
        self.fxns = '''

# functions used throughout analysis

from __future__ import division
from IPython.core.display import HTML
import numpy as np
import pandas as pd
from collections import Counter
from scipy.stats import mode
import sys
sys.path.append('/home/ubuntu/workspace/pydev/')
from CrossTabUtils import *
pd.set_option('display.max_colwidth', 1)
'''
        self.nb = nbf.new_notebook()
        self.cells = []
        cell = nbf.new_code_cell(self.fxns)
        self.cells.append(cell)
Пример #35
0
    def test_contents_manager(self):
        "make sure ContentsManager returns right files (ipynb, bin, txt)."

        nbdir = self.notebook_dir.name
        base = self.base_url()

        nb = new_notebook(name='testnb')

        ws = new_worksheet()
        nb.worksheets = [ws]
        ws.cells.append(new_heading_cell(u'Created by test ³'))
        cc1 = new_code_cell(input=u'print(2*6)')
        cc1.outputs.append(new_output(output_text=u'12', output_type='stream'))
        ws.cells.append(cc1)

        with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w', encoding='utf-8') as f:
            write(nb, f, format='ipynb')

        with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
            f.write(b'\xff' + os.urandom(5))
            f.close()

        with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
            f.write(u'foobar')
            f.close()

        r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
        self.assertEqual(r.status_code, 200)
        self.assertIn('print(2*6)', r.text)
        json.loads(r.text)

        r = requests.get(url_path_join(base, 'files', 'test.bin'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'application/octet-stream')
        self.assertEqual(r.content[:1], b'\xff')
        self.assertEqual(len(r.content), 6)

        r = requests.get(url_path_join(base, 'files', 'test.txt'))
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers['content-type'], 'text/plain')
        self.assertEqual(r.text, 'foobar')
Пример #36
0
    def notebook(self, s):
        """Export and convert IPython notebooks.

        This function can export the current IPython history to a notebook file
        or can convert an existing notebook file into a different format. For
        example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
        To export the history to "foo.py" do "%notebook -e foo.py". To convert
        "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
        formats include (json/ipynb, py).
        """
        args = magic_arguments.parse_argstring(self.notebook, s)

        from IPython.nbformat import current
        args.filename = unquote_filename(args.filename)
        if args.export:
            fname, name, format = current.parse_filename(args.filename)
            cells = []
            hist = list(self.shell.history_manager.get_range())
            for session, prompt_number, input in hist[:-1]:
                cells.append(current.new_code_cell(prompt_number=prompt_number,
                                                   input=input))
            worksheet = current.new_worksheet(cells=cells)
            nb = current.new_notebook(name=name,worksheets=[worksheet])
            with io.open(fname, 'w', encoding='utf-8') as f:
                current.write(nb, f, format)
        elif args.format is not None:
            old_fname, old_name, old_format = current.parse_filename(args.filename)
            new_format = args.format
            if new_format == u'xml':
                raise ValueError('Notebooks cannot be written as xml.')
            elif new_format == u'ipynb' or new_format == u'json':
                new_fname = old_name + u'.ipynb'
                new_format = u'json'
            elif new_format == u'py':
                new_fname = old_name + u'.py'
            else:
                raise ValueError('Invalid notebook format: %s' % new_format)
            with io.open(old_fname, 'r', encoding='utf-8') as f:
                nb = current.read(f, old_format)
            with io.open(new_fname, 'w', encoding='utf-8') as f:
                current.write(nb, f, new_format)
    def setUp(self):
        nbdir = self.notebook_dir.name
        
        if not os.path.isdir(pjoin(nbdir, 'foo')):
            os.mkdir(pjoin(nbdir, 'foo'))
        
        nb = new_notebook(name='testnb')
        
        ws = new_worksheet()
        nb.worksheets = [ws]
        ws.cells.append(new_heading_cell(u'Created by test ³'))
        cc1 = new_code_cell(input=u'print(2*6)')
        cc1.outputs.append(new_output(output_text=u'12'))
        cc1.outputs.append(new_output(output_png=png_green_pixel, output_type='pyout'))
        ws.cells.append(cc1)
        
        with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
                     encoding='utf-8') as f:
            write(nb, f, format='ipynb')

        self.nbconvert_api = NbconvertAPI(self.base_url())
Пример #38
0
def save_intermediate(km, ws, cell, save_name=None):
    """
    Conditionally save __save__ variables on error, and perhaps add a cell to
    load them up again into the worksheet.

    This will be utilized via a command-line flag.
    """
    saving_tmpl = """#error happend here, execute this cell to reload state
_load = False
_name = '%s'
if '__save__' in locals():
    d = {}
    for k in __save__:
        try:
            d[k] = locals()[k]
        # for things that are meant to be saved but that haven't executed yet
        except KeyError:
            d[k] = None
    np.savez(_name, **d)
    print "saved " + _name + '.npz'
if _load:
    loaded = np.load(_name+'.npz')
    print loaded.keys(), "now injected into this namespace"
    locals().update(loaded)
    """
    if not save_name:
        save_name = ''
    save_name += '_error'
    save_cell = cur.new_code_cell(saving_tmpl%save_name,"-1")
    if hasattr(km, 'saveonerror') and km.saveonerror:
        idx = ws.cells.index(cell)
        outs, exec_count = run_cell(km, save_cell)
        save_cell.outputs = outs
        # XXX: the problem with this approach is that after the first error,
        # the whole notebook must be regenerated in order to not have this
        # saving code in it
        ws.cells.insert(idx+1, save_cell)


    pass
def new_notebook_from_string(notebookname, filename, sourcestring):
    root = ast.parse(sourcestring, filename=filename, mode='exec')
    x = DetermineBlocks()

    for child in ast.iter_child_nodes(root):
        print(child.lineno, child)
        x.visit(child)
    x.end()
    sourcelines = sourcestring.splitlines()
    cells = []
    for block in x.blocks:
        print(block)
        blocklines = sourcelines[block[1] - 1:block[2]]
        blocksrc = '\n'.join(blocklines)
        if len(blocksrc) > 0:
            cell = notebook_format.new_code_cell(input=blocksrc)
            cells.append(cell)

    ws = notebook_format.new_worksheet(cells=cells)
    result = notebook_format.new_notebook(worksheets=[ws])
    result.metadata.name = notebookname
    return result
def new_notebook_from_string(notebookname, filename, sourcestring):
    root = ast.parse(sourcestring, filename=filename, mode='exec')
    x = DetermineBlocks()
    
    for child in ast.iter_child_nodes(root):
        print child.lineno, child
        x.visit(child)
    x.end()
    sourcelines = sourcestring.splitlines()
    cells = []
    for block in x.blocks:
        print block
        blocklines = sourcelines[block[1]-1:block[2]]
        blocksrc = '\n'.join(blocklines)
        if len(blocksrc) > 0:
            cell = notebook_format.new_code_cell(input=blocksrc)
            cells.append(cell)
            
    ws = notebook_format.new_worksheet(cells=cells)
    result = notebook_format.new_notebook(worksheets=[ws])
    result.metadata.name = notebookname
    return result
def remote_solutions(nb):
    """Generate a version of the notebook with stripped exercises solutions"""
    for ws in nb.worksheets:
        inside_solution = False
        cells_to_remove = []
        for i, cell in enumerate(ws.cells):
            if cell.cell_type == 'heading':
                inside_solution = False
            elif cell.cell_type == 'markdown':
                first_line = cell.source.split("\n")[0].strip()
                if first_line.lower() in ("**exercise:**", "**exercise**:"):
                    inside_solution = True
                    # Insert a new code cell to work on the exercise
                    ws.cells.insert(i + 1, current.new_code_cell())
                    continue
            if inside_solution:
                if cell.cell_type == 'code' and not hasattr(cell, 'input'):
                    # Leave blank code cells
                    continue
                cells_to_remove.append(cell)
        for cell in cells_to_remove:
            ws.cells.remove(cell)
def remote_solutions(nb):
    """Generate a version of the notebook with stripped exercises solutions"""
    for ws in nb.worksheets:
        inside_solution = False
        cells_to_remove = []
        for i, cell in enumerate(ws.cells):
            if cell.cell_type == 'heading':
                inside_solution = False
            elif cell.cell_type == 'markdown':
                first_line = cell.source.split("\n")[0].strip()
                if first_line.lower() in ("**exercise:**", "**exercise**:"):
                    inside_solution = True
                    # Insert a new code cell to work on the exercise
                    ws.cells.insert(i + 1, current.new_code_cell())
                    continue
            if inside_solution:
                if cell.cell_type == 'code' and not hasattr(cell, 'input'):
                    # Leave blank code cells
                    continue
                cells_to_remove.append(cell)
        for cell in cells_to_remove:
            ws.cells.remove(cell)
Пример #43
0
    def setUp(self):
        nbdir = self.notebook_dir.name

        if not os.path.isdir(pjoin(nbdir, 'foo')):
            os.mkdir(pjoin(nbdir, 'foo'))

        nb = new_notebook(name='testnb')

        ws = new_worksheet()
        nb.worksheets = [ws]
        ws.cells.append(new_heading_cell(u'Created by test ³'))
        cc1 = new_code_cell(input=u'print(2*6)')
        cc1.outputs.append(new_output(output_text=u'12'))
        cc1.outputs.append(
            new_output(output_png=png_green_pixel, output_type='pyout'))
        ws.cells.append(cc1)

        with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'),
                     'w',
                     encoding='utf-8') as f:
            write(nb, f, format='ipynb')

        self.nbconvert_api = NbconvertAPI(self.base_url())
Пример #44
0
    def to_ipython_cell(self, df):
        code = []
        outputs = []
        local = {"data": self.data, "parameters": self.parameters}
        for line in self.code.split("\n"):
            if line.endswith("# INPUT"):
                a, b = line.split("=")
                a, b = a.strip(), b.strip()
                value = eval(b, None, local)
                line = '{a} = {value}'.format(a=a, value=repr(value))
                code.append(line)
            elif line.endswith("# OUTPUT"):
                a, b = line.split("=")
                outputs += [x.strip() for x in a.split(",")]

                code.append(line[4:-len("# OUTPUT")].rstrip())

            elif "def" not in line and not line.startswith("@") and "return" not in line:
                code.append(line[4:])   # get rid of first indent

        code = "\n".join(code)
        gs = {"bz": bz, "pd": pd, "df": df, "stats": stats, "np":np, "StatisticalTest":StatisticalTest}
        ls = {}
        exec(code, gs, ls)

        output_cells = []
        for output in outputs:
            kwargs = {}
            if hasattr(ls[output], "to_html"):
                kwargs["output_html"] = ls[output].to_html()

            o = current.new_output("display_data", str(ls[output]), **kwargs)
            output_cells.append(o)

            code += "\ndisplay({})".format(output)

        return current.new_code_cell(code, outputs=output_cells)
Пример #45
0
 def add_code_cell(self, nb):
     output = current.new_output("display_data", output_javascript="alert('hi');")
     cell = current.new_code_cell("print('hi')", outputs=[output])
     if not nb.worksheets:
         nb.worksheets.append(current.new_worksheet())
     nb.worksheets[0].cells.append(cell)
Пример #46
0
 def add_code_cell(self, lines):
     c = nbformat.new_code_cell(input='\n'.join(lines))
     self.add_cell(c)
Пример #47
0
 def _init_cells(self):
     self._cells = [
         nbf.new_code_cell('%load_ext rmagic',
                           metadata=META_SKIP)
     ]
Пример #48
0
    def run(self):
        path = os.path.abspath(os.path.expanduser(self.args.path))
        final_dir = os.path.abspath(os.path.expanduser(self.args.final_dir))
        if not path.endswith(".yaml") and not path.endswith(".yml"):
            raise ValueError
        filename = os.path.basename(path)
        new_filename = "Mission" + filename.replace(".yml", ".ipynb").replace(".yaml", ".ipynb")
        final_dest = os.path.join(final_dir, new_filename)
        mission, screens = mission_loader(path)

        nb = nbf.new_notebook()

        mission_cell = nbf.new_text_cell('markdown', self.assemble_mission_cell(mission).strip())
        cells = [mission_cell]

        for screen in screens:
            text = self.assemble_screen_meta(screen)
            text += "\n\n"
            if screen["type"] == "code":
                text += "# " + screen["name"]
                text += "\n\n"
                text += screen["left_text"]
                if "instructions" in screen:
                    text += "\n\n"
                    text += "## Instructions\n\n"
                    text += screen["instructions"]
                if "hint" in screen:
                    text += "\n\n"
                    text += "## Hint\n\n"
                    text += screen["hint"]
            elif screen["type"] == "video":
                text += "# " + screen["name"]
                text += "\n\n"
                text += screen["video"]
            elif screen["type"] == "text":
                text += "# " + screen["name"]
                text += "\n\n"
                text += screen["text"]
            cell = nbf.new_text_cell('markdown', text.strip())
            cells.append(cell)

            if screen["type"] == "code":
                text = ""
                if "initial" not in screen and "answer" not in screen:
                    text += screen["initial_display"]
                else:
                    items = [
                        {"key": "initial", "name": "## Initial"},
                        {"key": "initial_display", "name": "## Display"},
                        {"key": "answer", "name": "## Answer"},
                        {"key": "check_val", "name": "## Check val"},
                        {"key": "check_vars", "name": "## Check vars"},
                        {"key": "check_code_run", "name": "## Check code run"}
                    ]

                    for item in items:
                        if item["key"] in screen and len(str(screen[item["key"]]).strip()) > 0:
                            if item["key"] == "check_vars" and len(screen[item["key"]]) == 0:
                                continue
                            text += item["name"] + "\n\n"
                            if item["key"] == "check_val":
                                text += '"' + str(screen[item["key"]]).strip().replace("\n", "\\n") + '"'
                            else:
                                text += str(screen[item["key"]]).strip()
                            text += "\n\n"
                cell = nbf.new_code_cell(input=text.strip())
                cells.append(cell)

        nb['worksheets'].append(nbf.new_worksheet(cells=cells))

        with open(final_dest, 'w+') as f:
            nbf.write(nb, f, 'ipynb')

        # Copy any associated files over
        original_dir = os.path.dirname(path)
        for f in os.listdir(original_dir):
            full_path = os.path.join(original_dir, f)
            if os.path.isfile(full_path):
                if not f.endswith(".yaml") and not f.endswith(".yml") and not f.endswith(".ipynb"):
                    shutil.copy2(full_path, os.path.join(final_dir, f))
Пример #49
0
 def add_code_cell(self, lines):
     c = nbformat.new_code_cell(input='\n'.join(lines))
     self.add_cell(c)
Пример #50
0
    def generate_report(reviews, dataset_name, file_name, load_reviews_code):

        nb = nbf.new_notebook()
        title = '# ' + dataset_name + ' Dataset Analysis'
        title_cell = nbf.new_text_cell(u'markdown', title)

        rda = ReviewsDatasetAnalyzer(reviews)
        num_reviews = len(rda.reviews)
        num_users = len(rda.user_ids)
        num_items = len(rda.item_ids)
        user_avg_reviews = float(num_reviews) / num_users
        item_avg_reviews = float(num_reviews) / num_items
        sparsity = rda.calculate_sparsity_approx()

        fact_sheet_text =\
            '## Fact Sheet\n' +\
            'The ' + dataset_name + ' contains:\n' +\
            '* ' + str(num_reviews) + ' reviews\n' +\
            '* Made by ' + str(num_users) + ' users\n' +\
            '* About ' + str(num_items) + ' items\n' +\
            '* It has an approximated sparsity of ' + str(sparsity) + '\n' +\
            '\nNow we are going to analyze the number of reviews per user and ' \
            'per item'
        fact_sheet_cell = nbf.new_text_cell(u'markdown', fact_sheet_text)

        reviews_analysis_code =\
            'import sys\n' +\
            'sys.path.append(\'/Users/fpena/UCC/Thesis/projects/yelp/source/python\')\n' +\
            'from etl import ETLUtils\n\n' +\
            'from etl.reviews_dataset_analyzer import ReviewsDatasetAnalyzer\n' +\
            '\n# Load reviews\n' + load_reviews_code + '\n' +\
            'rda = ReviewsDatasetAnalyzer(reviews)\n'
        reviews_analysis_cell = nbf.new_code_cell(reviews_analysis_code)

        user_analysis_text =\
            '## Users Reviews Analysis\n' +\
            '* The average number of reviews per user is ' + str(user_avg_reviews) + '\n' +\
            '* The minimum number of reviews a user has is ' + str(min(rda.users_count)) + '\n' +\
            '* The maximum number of reviews a user has is ' + str(max(rda.users_count))
        user_analysis_cell = nbf.new_text_cell(u'markdown', user_analysis_text)

        counts_per_user_code =\
            '# Number of reviews per user\n' +\
            'users_summary = rda.summarize_reviews_by_field(\'user_id\')\n' +\
            'print(\'Average number of reviews per user\', float(rda.num_reviews)/rda.num_users)\n' +\
            'users_summary.plot(kind=\'line\', rot=0)'
        counts_per_user_cell = nbf.new_code_cell(counts_per_user_code)

        item_analysis_text =\
            '## Items Reviews Analysis\n' +\
            '* The average number of reviews per item is ' + str(item_avg_reviews) + '\n' +\
            '* The minimum number of reviews an item has is ' + str(min(rda.items_count)) + '\n' +\
            '* The maximum number of reviews an item has is ' + str(max(rda.items_count))
        item_analysis_cell = nbf.new_text_cell(u'markdown', item_analysis_text)

        counts_per_item_code =\
            '# Number of reviews per item\n' +\
            'items_summary = rda.summarize_reviews_by_field(\'offering_id\')\n' +\
            'print(\'Average number of reviews per item\', float(rda.num_reviews)/rda.num_items)\n' +\
            'items_summary.plot(kind=\'line\', rot=0)'
        counts_per_item_cell = nbf.new_code_cell(counts_per_item_code)

        common_items_text =\
            '## Number of items 2 users have in common\n' +\
            'In this section we are going to count the number of items two ' \
            'users have in common'
        common_items_text_cell = nbf.new_text_cell(u'markdown', common_items_text)

        common_items_code =\
            '# Number of items 2 users have in common\n' +\
            'common_item_counts = rda.count_items_in_common()\n' +\
            'plt.plot(common_item_counts.keys(), common_item_counts.values())\n'
        common_items_code_cell = nbf.new_code_cell(common_items_code)

        common_items_box_code =\
            'from pylab import boxplot\n' +\
            'my_data = [key for key, value in common_item_counts.iteritems() for i in xrange(value)]\n' +\
            'mean_common_items = float(sum(my_data))/len(my_data)\n' +\
            'print(\'Average number of common items between two users:\', mean_common_items)\n' +\
            'boxplot(my_data)'
        common_items_box_cell = nbf.new_code_cell(common_items_box_code)

        cells = []
        cells.append(title_cell)
        cells.append(fact_sheet_cell)
        cells.append(reviews_analysis_cell)
        cells.append(user_analysis_cell)
        cells.append(counts_per_user_cell)
        cells.append(item_analysis_cell)
        cells.append(counts_per_item_cell)
        cells.append(common_items_text_cell)
        cells.append(common_items_code_cell)
        cells.append(common_items_box_cell)
        nb['worksheets'].append(nbf.new_worksheet(cells=cells))

        with open(file_name, 'w') as f:
            nbf.write(nb, f, 'ipynb')
 def test_match_tests_no_match(self):
     """Is an error raised when an autograding cell cannot be matched?"""
     cell = new_code_cell()
     cell.metadata = dict(assignment=dict(cell_type="autograder"))
     cells = [cell]
     assert_raises(RuntimeError, self.preprocessor._match_tests, cells)
Пример #52
0
 def _init_cells(self):
     self._cells = [
         nbf.new_code_cell('%load_ext rmagic', metadata=META_SKIP)
     ]
Пример #53
0
def new_code_cell():
    nlines = random.randint(0, 30)
    input = [str(random.random()) for i in range(nlines)]
    code_cell = nbformat.new_code_cell(input=input)
    return code_cell
Пример #54
0
 def process_codeblock(self, codeblock):
     """ Process a code block from pandoc json """
     attrs, contents = codeblock
     if self.guess_language(contents) == 'R':
         contents = '%%R\n' + contents
     self._add_cell(nbf.new_code_cell(contents))