Exemplo n.º 1
0
def test_notebook_manifest2(fxsoldir, fxhdftest2):
    # multiple files & hdfs dependency test
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook5.ipynb')
    assert os.path.isfile(path)
    mpath = get_notebook_manifest_path(path)
    assert os.path.isfile(mpath)
    update_notebook_by_run(path)
    manifest = Manifest(True, path)
    assert len(manifest.depends.files) == 2
    assert len(manifest.depends.hdf) == 2
    assert len(manifest._dep_files_chksum) == 2
    assert len(manifest._dep_hdf_chksum) == 2
    assert manifest._out_hdf_chksum is None

    path = os.path.join(nbdir, 'test-notebook6.ipynb')
    mpath = get_notebook_manifest_path(path)
    with pytest.raises(RecursiveReference):
        Manifest(False, path)
Exemplo n.º 2
0
    def __init__(self, check_depends=True, explicit_nbpath=None):
        super(Manifest, self).__init__()

        if explicit_nbpath is None:
            nbdir = get_notebook_dir()
            logging.debug(u"nbdir {}".format(nbdir))
            nbrpath = get_notebook_rpath()
            logging.debug(u"nbrpath {}".format(nbrpath))
            self._nbapath = os.path.join(nbdir, nbrpath)
            self._path = os.path.join(nbdir,
                                      get_notebook_manifest_path(nbrpath))
        else:
            logging.debug(u"explicit_nbpath {}".format(explicit_nbpath))
            self._nbapath = explicit_nbpath
            self._path = get_notebook_manifest_path(explicit_nbpath)

        logging.debug(u"find manifest {}".format(self._path))
        if not os.path.isfile(self._path.encode('utf8')):
            raise ManifestNotExist()
        self._init_checksum(check_depends)
        logging.debug("Manifest __init__ done")
Exemplo n.º 3
0
def test_notebook_manifest_error():
    nbdir = get_notebook_dir()
    nbapath = os.path.join(nbdir, 'test-notebook-manifest-error.ipynb')
    try:
        Manifest(False, nbapath)
    except SyntaxError:
        mpath = get_notebook_manifest_path(nbapath)
        with open(mpath, 'r') as f:
            data = json.loads(f.read())
        cells = data['cells']
        assert 'invalid syntax' in cells[0]['outputs'][0]['traceback'][0]
    else:
        assert False
Exemplo n.º 4
0
def test_notebook_manifest1(fxsoldir):
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook3.ipynb')
    assert os.path.isfile(path)
    mpath = get_notebook_manifest_path(path)
    assert os.path.isfile(mpath)

    # check manifest being written
    before = os.stat(mpath).st_mtime
    update_notebook_by_run(path)
    assert os.stat(mpath).st_mtime > before

    # check hdf store
    from wzdat.util import HDF
    with HDF('haje01') as hdf:
        df = hdf.store.select('test')
        assert len(df) == 7560

    # check manifest checksum
    import json
    with open(mpath, 'r') as f:
        data = json.loads(f.read())
    cells = data['cells']
    assert len(cells) == 2
    chksums = cells[1]['source']
    assert 'WARNING' in chksums[0]
    assert 'last_run' in chksums[2]
    assert 'elapsed' in chksums[3]
    assert 'max_memory' in chksums[4]
    assert 'error' in chksums[5]
    assert 'depends' in chksums[6]
    assert '8875249185536240278' in chksums[7]
    # check output checksum
    assert 'output' in chksums[9]
    assert '-2394538446589678049' in chksums[10]

    manifest = Manifest(False, path)
    assert type(manifest.last_run) is datetime
    assert manifest._out_hdf_chksum is None

    # rewrite manifest output by hdf put
    manifest.output.hdf.put(df, data_columns=['level'])

    # select manifest output by hdf select
    path = os.path.join(nbdir, 'test-notebook4.ipynb')
    manifest = Manifest(True, path)
    df = manifest.depends.hdf.select("index>Timestamp('2014-03-01') &"
                                     "level='INFO'", columns=['level', 'node'])
    assert len(df) == 1125
    assert len(df.columns) == 2