def test_notebook_manifest2(fxsoldir, fxhdftest2): # multiple files & hdfs dependency test nbdir = get_notebook_dir() path = os.path.join(nbdir, 'test-notebook5.ipynb') assert os.path.isfile(path) mpath = get_notebook_manifest_path(path) assert os.path.isfile(mpath) update_notebook_by_run(path) manifest = Manifest(True, path) assert len(manifest.depends.files) == 2 assert len(manifest.depends.hdf) == 2 assert len(manifest._dep_files_chksum) == 2 assert len(manifest._dep_hdf_chksum) == 2 assert manifest._out_hdf_chksum is None path = os.path.join(nbdir, 'test-notebook6.ipynb') mpath = get_notebook_manifest_path(path) with pytest.raises(RecursiveReference): Manifest(False, path)
def __init__(self, check_depends=True, explicit_nbpath=None): super(Manifest, self).__init__() if explicit_nbpath is None: nbdir = get_notebook_dir() logging.debug(u"nbdir {}".format(nbdir)) nbrpath = get_notebook_rpath() logging.debug(u"nbrpath {}".format(nbrpath)) self._nbapath = os.path.join(nbdir, nbrpath) self._path = os.path.join(nbdir, get_notebook_manifest_path(nbrpath)) else: logging.debug(u"explicit_nbpath {}".format(explicit_nbpath)) self._nbapath = explicit_nbpath self._path = get_notebook_manifest_path(explicit_nbpath) logging.debug(u"find manifest {}".format(self._path)) if not os.path.isfile(self._path.encode('utf8')): raise ManifestNotExist() self._init_checksum(check_depends) logging.debug("Manifest __init__ done")
def test_notebook_manifest_error(): nbdir = get_notebook_dir() nbapath = os.path.join(nbdir, 'test-notebook-manifest-error.ipynb') try: Manifest(False, nbapath) except SyntaxError: mpath = get_notebook_manifest_path(nbapath) with open(mpath, 'r') as f: data = json.loads(f.read()) cells = data['cells'] assert 'invalid syntax' in cells[0]['outputs'][0]['traceback'][0] else: assert False
def test_notebook_manifest1(fxsoldir): nbdir = get_notebook_dir() path = os.path.join(nbdir, 'test-notebook3.ipynb') assert os.path.isfile(path) mpath = get_notebook_manifest_path(path) assert os.path.isfile(mpath) # check manifest being written before = os.stat(mpath).st_mtime update_notebook_by_run(path) assert os.stat(mpath).st_mtime > before # check hdf store from wzdat.util import HDF with HDF('haje01') as hdf: df = hdf.store.select('test') assert len(df) == 7560 # check manifest checksum import json with open(mpath, 'r') as f: data = json.loads(f.read()) cells = data['cells'] assert len(cells) == 2 chksums = cells[1]['source'] assert 'WARNING' in chksums[0] assert 'last_run' in chksums[2] assert 'elapsed' in chksums[3] assert 'max_memory' in chksums[4] assert 'error' in chksums[5] assert 'depends' in chksums[6] assert '8875249185536240278' in chksums[7] # check output checksum assert 'output' in chksums[9] assert '-2394538446589678049' in chksums[10] manifest = Manifest(False, path) assert type(manifest.last_run) is datetime assert manifest._out_hdf_chksum is None # rewrite manifest output by hdf put manifest.output.hdf.put(df, data_columns=['level']) # select manifest output by hdf select path = os.path.join(nbdir, 'test-notebook4.ipynb') manifest = Manifest(True, path) df = manifest.depends.hdf.select("index>Timestamp('2014-03-01') &" "level='INFO'", columns=['level', 'node']) assert len(df) == 1125 assert len(df.columns) == 2