예제 #1
0
def test_notebook_dependency(fxsoldir, fxnewfile):
    # run notebook first
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook3.ipynb')
    assert os.path.isfile(path)

    with HDF('haje01') as hdf:
        if 'test' in hdf.store:
            del hdf.store['test']

    update_notebook_by_run(path)
    manifest = Manifest(True, path)
    assert manifest._prev_files_chksum == manifest._dep_files_chksum
    with HDF('haje01') as hdf:
        prev_hdf_chksum = dataframe_checksum(hdf.store['test'])
        print "prev_hdf_chksum {}".format(prev_hdf_chksum)
        print len(hdf.store['test'])

    # add new file
    with open(fxnewfile, 'w') as f:
        f.write('2014-03-05 23:30 [ERROR] - Async\n')

    manifest = Manifest(False, path)
    assert manifest._depend_files_changed
    assert manifest._prev_files_chksum != manifest._dep_files_chksum

    # run notebok again
    update_notebook_by_run(path)
    with HDF('haje01') as hdf:
        new_hdf_chksum = dataframe_checksum(hdf.store['test'])
        print "new_hdf_chksum {}".format(new_hdf_chksum)
        print len(hdf.store['test'])

    # check check
    assert prev_hdf_chksum != new_hdf_chksum
예제 #2
0
def test_notebook_nodata():
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook-nodata.ipynb')
    assert os.path.isfile(path)
    update_notebook_by_run(path)
    rv = notebook_outputs_to_html(path)
    assert 'NoDataFound' in rv
예제 #3
0
def test_notebook_run():
    path = os.path.join(get_notebook_dir(), 'test-notebook.ipynb')
    assert os.path.isfile(path)
    before = os.stat(path).st_mtime
    update_notebook_by_run(path)
    assert os.stat(path).st_mtime > before
    runnbs = [ri[0] for ri in iter_run_info()]
    assert path in runnbs
예제 #4
0
파일: jobs.py 프로젝트: haje01/wzdat
def run_notebook(path):
    assert path[0] == '/', "Need absolute path"
    path = path.decode('utf-8') if type(path) == str else path
    logging.debug(u'run_notebook {}'.format(path))
    try:
        update_notebook_by_run(path)
    except NoDataFound, e:
        logging.debug(unicode(e))
예제 #5
0
파일: tasks.py 프로젝트: haje01/wzdat
def rerun_notebook(nbpath):
    print(u'rerun_notebook {}'.format(nbpath))
    rerun_notebook.update_state(state='PROGRESS', meta=0)
    try:
        update_notebook_by_run(nbpath)
    except NoDataFound, e:
        logging.debug('NoDataFound')
        return [unicode(e)]
예제 #6
0
def fxhdftest2():
    with HDF('haje01') as hdf:
        if 'test' not in hdf.store:
            # check hdf notebook has run
            path = find_hdf_notebook_path('haje01', 'test')
            assert path is not None
            update_notebook_by_run(path)
        test = hdf.store['test']
        if 'test2' not in hdf.store:
            hdf.store['test2'] = test
    yield None
예제 #7
0
def test_notebook_manifest1(fxsoldir):
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook3.ipynb')
    assert os.path.isfile(path)
    mpath = get_notebook_manifest_path(path)
    assert os.path.isfile(mpath)

    # check manifest being written
    before = os.stat(mpath).st_mtime
    update_notebook_by_run(path)
    assert os.stat(mpath).st_mtime > before

    # check hdf store
    from wzdat.util import HDF
    with HDF('haje01') as hdf:
        df = hdf.store.select('test')
        assert len(df) == 7560

    # check manifest checksum
    import json
    with open(mpath, 'r') as f:
        data = json.loads(f.read())
    cells = data['cells']
    assert len(cells) == 2
    chksums = cells[1]['source']
    assert 'WARNING' in chksums[0]
    assert 'last_run' in chksums[2]
    assert 'elapsed' in chksums[3]
    assert 'max_memory' in chksums[4]
    assert 'error' in chksums[5]
    assert 'depends' in chksums[6]
    assert '8875249185536240278' in chksums[7]
    # check output checksum
    assert 'output' in chksums[9]
    assert '-2394538446589678049' in chksums[10]

    manifest = Manifest(False, path)
    assert type(manifest.last_run) is datetime
    assert manifest._out_hdf_chksum is None

    # rewrite manifest output by hdf put
    manifest.output.hdf.put(df, data_columns=['level'])

    # select manifest output by hdf select
    path = os.path.join(nbdir, 'test-notebook4.ipynb')
    manifest = Manifest(True, path)
    df = manifest.depends.hdf.select("index>Timestamp('2014-03-01') &"
                                     "level='INFO'", columns=['level', 'node'])
    assert len(df) == 1125
    assert len(df.columns) == 2
예제 #8
0
def test_notebook_error():
    path = os.path.join(get_notebook_dir(), 'test-notebook-error.ipynb')
    assert os.path.isfile(path)
    try:
        update_notebook_by_run(path)
    except ValueError:
        pass
    assert check_notebook_error_and_changed(path) == (True, False)
    touch(path)
    assert check_notebook_error_and_changed(path) == (True, True)

    from wzdat import rundb
    redis_ri = rundb.get_run_info(path)
    rundb.remove_run_info(path)
    manifest_ri = get_run_info(path)
    # both elapsed time is equal
    assert redis_ri[1] == manifest_ri[1]
    # both error msg is equal
    assert redis_ri[-1] == manifest_ri[-1]
예제 #9
0
def test_notebook_manifest2(fxsoldir, fxhdftest2):
    # multiple files & hdfs dependency test
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook5.ipynb')
    assert os.path.isfile(path)
    mpath = get_notebook_manifest_path(path)
    assert os.path.isfile(mpath)
    update_notebook_by_run(path)
    manifest = Manifest(True, path)
    assert len(manifest.depends.files) == 2
    assert len(manifest.depends.hdf) == 2
    assert len(manifest._dep_files_chksum) == 2
    assert len(manifest._dep_hdf_chksum) == 2
    assert manifest._out_hdf_chksum is None

    path = os.path.join(nbdir, 'test-notebook6.ipynb')
    mpath = get_notebook_manifest_path(path)
    with pytest.raises(RecursiveReference):
        Manifest(False, path)
예제 #10
0
 def _run_resolved(self, updaterun, notebook, resolved, runs):
     '''Run notebook after all its dependencies resolved.'''
     logging.debug(u"_run_resolved '{}'".format(notebook.path))
     notebook.reload_manifest()
     path = notebook.path
     # Only run when dependecies changed and notebook has no error or
     # changed
     error, changed = check_notebook_error_and_changed(path)
     logging.debug("nb error {}, nb changed {}".format(error, changed))
     if updaterun:
         # run notebook when its depends changed or had fixed after error
         if notebook.manifest._need_run:  # or (error and changed):
             try:
                 update_notebook_by_run(path)
             except NoDataFound, e:
                 logging.debug(unicode(e))
             runs.append(notebook)
         elif error and not changed:
             logging.debug(u"_run_resolved - skip unfixed {}".format(path))
         else:
             logging.debug(u"no need to run")