def read_chunk_from_local(chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE-offset)) from dpark.accumulator import LocalReadBytes name = '%02X/chunk_%016X_%08X.mfs' % (chunkid & 0xFF, chunkid, version) for d in mfsdirs: p = os.path.join(d, name) if os.path.exists(p): if os.path.getsize(p) < CHUNKHDRSIZE + offset + size: logger.error('%s is not completed: %d < %d', name, os.path.getsize(p), CHUNKHDRSIZE + offset + size) return #raise ValueError("size too large") f = open(p) f.seek(CHUNKHDRSIZE + offset) while size > 0: to_read = min(size, 640*1024) data = f.read(to_read) if not data: return LocalReadBytes.add(len(data)) yield data size -= len(data) f.close() return else: logger.warning("%s was not found", name)
def read_chunk_from_local(chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE - offset)) from dpark.accumulator import ReadBytes, LocalReadBytes name = '%02X/chunk_%016X_%08X.mfs' % (chunkid & 0xFF, chunkid, version) for d in mfsdirs: p = os.path.join(d, name) if os.path.exists(p): if os.path.getsize(p) < CHUNKHDRSIZE + offset + size: print p, 'is not completed', os.path.getsize( p), '<', CHUNKHDRSIZE + offset + size return #raise ValueError("size too large") f = open(p) f.seek(CHUNKHDRSIZE + offset) while size > 0: to_read = min(size, 1024 * 1024 * 4) data = f.read(to_read) if not data: return LocalReadBytes.add(len(data)) yield data size -= len(data) f.close() return
def read_chunk_from_local(chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE - offset)) from dpark.accumulator import LocalReadBytes name = '%02X/chunk_%016X_%08X.mfs' % (chunkid & 0xFF, chunkid, version) for d in mfsdirs: p = os.path.join(d, name) if os.path.exists(p): if os.path.getsize(p) < CHUNKHDRSIZE + offset + size: logger.error('%s is not completed: %d < %d', name, os.path.getsize(p), CHUNKHDRSIZE + offset + size) return #raise ValueError("size too large") f = open(p) f.seek(CHUNKHDRSIZE + offset) while size > 0: to_read = min(size, 640 * 1024) data = f.read(to_read) if not data: return LocalReadBytes.add(len(data)) yield data size -= len(data) f.close() return else: logger.warning("%s was not found", name)
def read_chunk_from_local(chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE-offset)) from dpark.accumulator import ReadBytes, LocalReadBytes name = '%02X/chunk_%016X_%08X.mfs' % (chunkid & 0xFF, chunkid, version) for d in mfsdirs: p = os.path.join(d, name) if os.path.exists(p): if os.path.getsize(p) < CHUNKHDRSIZE + offset + size: print p, 'is not completed', os.path.getsize(p), '<', CHUNKHDRSIZE + offset + size return #raise ValueError("size too large") f = open(p) f.seek(CHUNKHDRSIZE + offset) while size > 0: to_read = min(size, 1024*1024*4) data = f.read(to_read) if not data: return LocalReadBytes.add(len(data)) yield data size -= len(data) f.close() return