예제 #1
0
def test_notebook_dependency(fxsoldir, fxnewfile):
    # run notebook first
    nbdir = get_notebook_dir()
    path = os.path.join(nbdir, 'test-notebook3.ipynb')
    assert os.path.isfile(path)

    with HDF('haje01') as hdf:
        if 'test' in hdf.store:
            del hdf.store['test']

    update_notebook_by_run(path)
    manifest = Manifest(True, path)
    assert manifest._prev_files_chksum == manifest._dep_files_chksum
    with HDF('haje01') as hdf:
        prev_hdf_chksum = dataframe_checksum(hdf.store['test'])
        print "prev_hdf_chksum {}".format(prev_hdf_chksum)
        print len(hdf.store['test'])

    # add new file
    with open(fxnewfile, 'w') as f:
        f.write('2014-03-05 23:30 [ERROR] - Async\n')

    manifest = Manifest(False, path)
    assert manifest._depend_files_changed
    assert manifest._prev_files_chksum != manifest._dep_files_chksum

    # run notebok again
    update_notebook_by_run(path)
    with HDF('haje01') as hdf:
        new_hdf_chksum = dataframe_checksum(hdf.store['test'])
        print "new_hdf_chksum {}".format(new_hdf_chksum)
        print len(hdf.store['test'])

    # check check
    assert prev_hdf_chksum != new_hdf_chksum
예제 #2
0
파일: manifest.py 프로젝트: haje01/wzdat
 def _append(self, value, aformat, append, columns, dropna, **kwargs):
     import pandas as pd
     ktuple = tuple(((k, v if isinstance(v, tuple) else tuple(v)) for k,
                     v in kwargs.items()))
     _args = (dataframe_checksum(value), self.uname, self.sname, aformat,
              append, columns, dropna, ktuple)
     # escape None for hashing
     args = map(lambda x: 0 if x is None else x, _args)
     chksum = hash(tuple(args))
     # logging.debug('  chksum: {}'.format(chksum))
     self.manifest._out_hdf_chksum = chksum
     logging.debug("HDFStore append - _out_hdf_chksum {}".format(chksum))
     with HDF(self.uname) as hdf:
         hdf.store.append(self.sname, value, aformat, append, columns,
                          dropna, **kwargs)
         # write checksum
         hsname = HDF_CHKSUM_FMT.format(self.sname)
         if append:
             pchksum = int(hdf.store[hsname])
             chksum = pchksum + chksum
         hdf.store.append(hsname, pd.Series([str(chksum)]), append=False)