def test_notebook_manifest2(fxsoldir, fxhdftest2):
# multiple files & hdfs dependency test
nbdir = get_notebook_dir()
path = os.path.join(nbdir, 'test-notebook5.ipynb')
assert os.path.isfile(path)
mpath = get_notebook_manifest_path(path)
assert os.path.isfile(mpath)
update_notebook_by_run(path)
manifest = Manifest(True, path)
assert len(manifest.depends.files) == 2
assert len(manifest.depends.hdf) == 2
assert len(manifest._dep_files_chksum) == 2
assert len(manifest._dep_hdf_chksum) == 2
assert manifest._out_hdf_chksum is None
path = os.path.join(nbdir, 'test-notebook6.ipynb')
mpath = get_notebook_manifest_path(path)
with pytest.raises(RecursiveReference):
Manifest(False, path)
def __init__(self, check_depends=True, explicit_nbpath=None):
super(Manifest, self).__init__()
if explicit_nbpath is None:
nbdir = get_notebook_dir()
logging.debug(u"nbdir {}".format(nbdir))
nbrpath = get_notebook_rpath()
logging.debug(u"nbrpath {}".format(nbrpath))
self._nbapath = os.path.join(nbdir, nbrpath)
self._path = os.path.join(nbdir,
get_notebook_manifest_path(nbrpath))
else:
logging.debug(u"explicit_nbpath {}".format(explicit_nbpath))
self._nbapath = explicit_nbpath
self._path = get_notebook_manifest_path(explicit_nbpath)
logging.debug(u"find manifest {}".format(self._path))
if not os.path.isfile(self._path.encode('utf8')):
raise ManifestNotExist()
self._init_checksum(check_depends)
logging.debug("Manifest __init__ done")
def test_notebook_manifest_error():
nbdir = get_notebook_dir()
nbapath = os.path.join(nbdir, 'test-notebook-manifest-error.ipynb')
try:
Manifest(False, nbapath)
except SyntaxError:
mpath = get_notebook_manifest_path(nbapath)
with open(mpath, 'r') as f:
data = json.loads(f.read())
cells = data['cells']
assert 'invalid syntax' in cells[0]['outputs'][0]['traceback'][0]
else:
assert False
def test_notebook_manifest1(fxsoldir):
nbdir = get_notebook_dir()
path = os.path.join(nbdir, 'test-notebook3.ipynb')
assert os.path.isfile(path)
mpath = get_notebook_manifest_path(path)
assert os.path.isfile(mpath)
# check manifest being written
before = os.stat(mpath).st_mtime
update_notebook_by_run(path)
assert os.stat(mpath).st_mtime > before
# check hdf store
from wzdat.util import HDF
with HDF('haje01') as hdf:
df = hdf.store.select('test')
assert len(df) == 7560
# check manifest checksum
import json
with open(mpath, 'r') as f:
data = json.loads(f.read())
cells = data['cells']
assert len(cells) == 2
chksums = cells[1]['source']
assert 'WARNING' in chksums[0]
assert 'last_run' in chksums[2]
assert 'elapsed' in chksums[3]
assert 'max_memory' in chksums[4]
assert 'error' in chksums[5]
assert 'depends' in chksums[6]
assert '8875249185536240278' in chksums[7]
# check output checksum
assert 'output' in chksums[9]
assert '-2394538446589678049' in chksums[10]
manifest = Manifest(False, path)
assert type(manifest.last_run) is datetime
assert manifest._out_hdf_chksum is None
# rewrite manifest output by hdf put
manifest.output.hdf.put(df, data_columns=['level'])
# select manifest output by hdf select
path = os.path.join(nbdir, 'test-notebook4.ipynb')
manifest = Manifest(True, path)
df = manifest.depends.hdf.select("index>Timestamp('2014-03-01') &"
"level='INFO'", columns=['level', 'node'])
assert len(df) == 1125
assert len(df.columns) == 2