def __init__(self, con, path=None, fileobj=None): self.con = con self.path = path if path: self.reader = open(self.path, 'rb') self.size = os.path.getsize(self.path) else: self.reader = fileobj fileobj.seek(0, 2) self.size = fileobj.tell() fileobj.seek(0) self.rs = Rollsum() self.blob_size = 0 # Store Span the instance of the chunk self.spans = [] # Total size self.n = 0 # buffer to store the chunk self.buf = '' self.buf_spans = {} # To generate the end report. self.cnt = { 'skipped': 0, 'skipped_size': 0, 'uploaded': 0, 'uploaded_size': 0 }
def benchmark_rollsum(): bytes_size = 1024 * 1024 * 5 rs = Rollsum() splits = 0 for i in range(bytes_size): rs.roll(random.randint(0, 255)) if rs.on_split(): rs.bits() splits += 1 every = int(bytes_size / splits) print 'num splits: {0}; every {1} bytes.'.format(splits, every)
def rsum(offset, length): """ Test function that returns Rollsum digest. """ rs = Rollsum() for b in buf[offset:length]: rs.roll(b) return rs.digest()