def test_notebook_dependency(fxsoldir, fxnewfile):
# run notebook first
nbdir = get_notebook_dir()
path = os.path.join(nbdir, 'test-notebook3.ipynb')
assert os.path.isfile(path)
with HDF('haje01') as hdf:
if 'test' in hdf.store:
del hdf.store['test']
update_notebook_by_run(path)
manifest = Manifest(True, path)
assert manifest._prev_files_chksum == manifest._dep_files_chksum
with HDF('haje01') as hdf:
prev_hdf_chksum = dataframe_checksum(hdf.store['test'])
print "prev_hdf_chksum {}".format(prev_hdf_chksum)
print len(hdf.store['test'])
# add new file
with open(fxnewfile, 'w') as f:
f.write('2014-03-05 23:30 [ERROR] - Async\n')
manifest = Manifest(False, path)
assert manifest._depend_files_changed
assert manifest._prev_files_chksum != manifest._dep_files_chksum
# run notebok again
update_notebook_by_run(path)
with HDF('haje01') as hdf:
new_hdf_chksum = dataframe_checksum(hdf.store['test'])
print "new_hdf_chksum {}".format(new_hdf_chksum)
print len(hdf.store['test'])
# check check
assert prev_hdf_chksum != new_hdf_chksum
def test_notebook_nodata():
nbdir = get_notebook_dir()
path = os.path.join(nbdir, 'test-notebook-nodata.ipynb')
assert os.path.isfile(path)
update_notebook_by_run(path)
rv = notebook_outputs_to_html(path)
assert 'NoDataFound' in rv
def test_notebook_run():
path = os.path.join(get_notebook_dir(), 'test-notebook.ipynb')
assert os.path.isfile(path)
before = os.stat(path).st_mtime
update_notebook_by_run(path)
assert os.stat(path).st_mtime > before
runnbs = [ri[0] for ri in iter_run_info()]
assert path in runnbs
def run_notebook(path):
assert path[0] == '/', "Need absolute path"
path = path.decode('utf-8') if type(path) == str else path
logging.debug(u'run_notebook {}'.format(path))
try:
update_notebook_by_run(path)
except NoDataFound, e:
logging.debug(unicode(e))
def rerun_notebook(nbpath):
print(u'rerun_notebook {}'.format(nbpath))
rerun_notebook.update_state(state='PROGRESS', meta=0)
try:
update_notebook_by_run(nbpath)
except NoDataFound, e:
logging.debug('NoDataFound')
return [unicode(e)]
def fxhdftest2():
with HDF('haje01') as hdf:
if 'test' not in hdf.store:
# check hdf notebook has run
path = find_hdf_notebook_path('haje01', 'test')
assert path is not None
update_notebook_by_run(path)
test = hdf.store['test']
if 'test2' not in hdf.store:
hdf.store['test2'] = test
yield None
def test_notebook_manifest1(fxsoldir):
nbdir = get_notebook_dir()
path = os.path.join(nbdir, 'test-notebook3.ipynb')
assert os.path.isfile(path)
mpath = get_notebook_manifest_path(path)
assert os.path.isfile(mpath)
# check manifest being written
before = os.stat(mpath).st_mtime
update_notebook_by_run(path)
assert os.stat(mpath).st_mtime > before
# check hdf store
from wzdat.util import HDF
with HDF('haje01') as hdf:
df = hdf.store.select('test')
assert len(df) == 7560
# check manifest checksum
import json
with open(mpath, 'r') as f:
data = json.loads(f.read())
cells = data['cells']
assert len(cells) == 2
chksums = cells[1]['source']
assert 'WARNING' in chksums[0]
assert 'last_run' in chksums[2]
assert 'elapsed' in chksums[3]
assert 'max_memory' in chksums[4]
assert 'error' in chksums[5]
assert 'depends' in chksums[6]
assert '8875249185536240278' in chksums[7]
# check output checksum
assert 'output' in chksums[9]
assert '-2394538446589678049' in chksums[10]
manifest = Manifest(False, path)
assert type(manifest.last_run) is datetime
assert manifest._out_hdf_chksum is None
# rewrite manifest output by hdf put
manifest.output.hdf.put(df, data_columns=['level'])
# select manifest output by hdf select
path = os.path.join(nbdir, 'test-notebook4.ipynb')
manifest = Manifest(True, path)
df = manifest.depends.hdf.select("index>Timestamp('2014-03-01') &"
"level='INFO'", columns=['level', 'node'])
assert len(df) == 1125
assert len(df.columns) == 2
def test_notebook_error():
path = os.path.join(get_notebook_dir(), 'test-notebook-error.ipynb')
assert os.path.isfile(path)
try:
update_notebook_by_run(path)
except ValueError:
pass
assert check_notebook_error_and_changed(path) == (True, False)
touch(path)
assert check_notebook_error_and_changed(path) == (True, True)
from wzdat import rundb
redis_ri = rundb.get_run_info(path)
rundb.remove_run_info(path)
manifest_ri = get_run_info(path)
# both elapsed time is equal
assert redis_ri[1] == manifest_ri[1]
# both error msg is equal
assert redis_ri[-1] == manifest_ri[-1]
def test_notebook_manifest2(fxsoldir, fxhdftest2):
# multiple files & hdfs dependency test
nbdir = get_notebook_dir()
path = os.path.join(nbdir, 'test-notebook5.ipynb')
assert os.path.isfile(path)
mpath = get_notebook_manifest_path(path)
assert os.path.isfile(mpath)
update_notebook_by_run(path)
manifest = Manifest(True, path)
assert len(manifest.depends.files) == 2
assert len(manifest.depends.hdf) == 2
assert len(manifest._dep_files_chksum) == 2
assert len(manifest._dep_hdf_chksum) == 2
assert manifest._out_hdf_chksum is None
path = os.path.join(nbdir, 'test-notebook6.ipynb')
mpath = get_notebook_manifest_path(path)
with pytest.raises(RecursiveReference):
Manifest(False, path)
def _run_resolved(self, updaterun, notebook, resolved, runs):
'''Run notebook after all its dependencies resolved.'''
logging.debug(u"_run_resolved '{}'".format(notebook.path))
notebook.reload_manifest()
path = notebook.path
# Only run when dependecies changed and notebook has no error or
# changed
error, changed = check_notebook_error_and_changed(path)
logging.debug("nb error {}, nb changed {}".format(error, changed))
if updaterun:
# run notebook when its depends changed or had fixed after error
if notebook.manifest._need_run: # or (error and changed):
try:
update_notebook_by_run(path)
except NoDataFound, e:
logging.debug(unicode(e))
runs.append(notebook)
elif error and not changed:
logging.debug(u"_run_resolved - skip unfixed {}".format(path))
else:
logging.debug(u"no need to run")