def _resizebuf(buf, newsize):
"Return a copy of BUF of size NEWSIZE."
newbuf = np.newbuffer(newsize)
if newsize > len(buf):
newbuf[:len(buf)] = buf
else:
newbuf[:] = buf[:len(newbuf)]
return newbuf
def _resizebuf(buf,newsize):
"Return a copy of BUF of size NEWSIZE."
newbuf = np.newbuffer(newsize)
if newsize > len(buf):
newbuf[:len(buf)]=buf
else:
newbuf[:]=buf[:len(newbuf)]
return newbuf
def histToFileLayout(path, name, hist, layout, overwrite=False): pathLayout = os.path.join(path, layout) ensureDir(pathLayout) fileName = name+".hist."+layout if not isFile(pathLayout, fileName) or isEmpty(pathLayout, fileName) or overwrite: bufferSize=len(hist)*len(hist[0])*len(hist[0][0]) #intType = 'I' if globalBiggest > 64000 else 'H' #intType = 'B' if globalBiggest < 256 else 'H' intType = 'H' # Assume all numbers fit in a short (breaks if full hd and 4x4 regions) intSize = struct.calcsize(intType) bf = np.newbuffer(bufferSize*intSize) with open(os.path.join(pathLayout, fileName), "wb") as of: if layout == "linear": histToFileLinear(of, hist, intType, bf) elif layout == "binned": histToFileBinned(of, hist, intType, bf) elif layout == "reg-linear": histToFileRegionLinear(of, hist, intType, bf) elif layout == "reg-binned": histToFileRegionBinned(of, hist, intType, bf) return pathLayout, fileName
def fromfile(infile,
type=None,
shape=None,
sizing=STRICT,
typecode=None,
dtype=None):
if isinstance(infile, (str, unicode)):
infile = open(infile, 'rb')
dtype = type2dtype(typecode, type, dtype, True)
if shape is None:
shape = (-1, )
if not isinstance(shape, tuple):
shape = (shape, )
if (list(shape).count(-1) > 1):
raise ValueError("At most one unspecified dimension in shape")
if -1 not in shape:
if sizing != STRICT:
raise ValueError("sizing must be STRICT if size complete")
arr = np.empty(shape, dtype)
bytesleft = arr.nbytes
bytesread = 0
while (bytesleft > _BLOCKSIZE):
data = infile.read(_BLOCKSIZE)
if len(data) != _BLOCKSIZE:
raise EarlyEOFError(
"Unexpected EOF reading data for size complete array")
arr.data[bytesread:bytesread + _BLOCKSIZE] = data
bytesread += _BLOCKSIZE
bytesleft -= _BLOCKSIZE
if bytesleft > 0:
data = infile.read(bytesleft)
if len(data) != bytesleft:
raise EarlyEOFError(
"Unexpected EOF reading data for size complete array")
arr.data[bytesread:bytesread + bytesleft] = data
return arr
##shape is incompletely specified
##read until EOF
##implementation 1: naively use memory blocks
##problematic because memory allocation can be double what is
##necessary (!)
##the most common case, namely reading in data from an unchanging
##file whose size may be determined before allocation, should be
##quick -- only one allocation will be needed.
recsize = dtype.itemsize * np.product([i for i in shape if i != -1])
blocksize = max(_BLOCKSIZE / recsize, 1) * recsize
##try to estimate file size
try:
curpos = infile.tell()
infile.seek(0, 2)
endpos = infile.tell()
infile.seek(curpos)
except (AttributeError, IOError):
initsize = blocksize
else:
initsize = max(1, (endpos - curpos) / recsize) * recsize
buf = np.newbuffer(initsize)
bytesread = 0
while 1:
data = infile.read(blocksize)
if len(data) != blocksize: ##eof
break
##do we have space?
if len(buf) < bytesread + blocksize:
buf = _resizebuf(buf, len(buf) + blocksize)
## or rather a=resizebuf(a,2*len(a)) ?
assert len(buf) >= bytesread + blocksize
buf[bytesread:bytesread + blocksize] = data
bytesread += blocksize
if len(data) % recsize != 0:
if sizing == STRICT:
raise SizeMismatchError("Filesize does not match specified shape")
if sizing == WARN:
_warnings.warn("Filesize does not match specified shape",
SizeMismatchWarning)
try:
infile.seek(-(len(data) % recsize), 1)
except AttributeError:
_warnings.warn("Could not rewind (no seek support)",
FileSeekWarning)
except IOError:
_warnings.warn("Could not rewind (IOError in seek)",
FileSeekWarning)
datasize = (len(data) / recsize) * recsize
if len(buf) != bytesread + datasize:
buf = _resizebuf(buf, bytesread + datasize)
buf[bytesread:bytesread + datasize] = data[:datasize]
##deduce shape from len(buf)
shape = list(shape)
uidx = shape.index(-1)
shape[uidx] = len(buf) / recsize
a = np.ndarray(shape=shape, dtype=type, buffer=buf)
if a.dtype.char == '?':
np.not_equal(a, 0, a)
return a
def fromfile(infile, type=None, shape=None, sizing=STRICT,
typecode=None, dtype=None):
if isinstance(infile, (str, unicode)):
infile = open(infile, 'rb')
dtype = type2dtype(typecode, type, dtype, True)
if shape is None:
shape = (-1,)
if not isinstance(shape, tuple):
shape = (shape,)
if (list(shape).count(-1)>1):
raise ValueError("At most one unspecified dimension in shape")
if -1 not in shape:
if sizing != STRICT:
raise ValueError("sizing must be STRICT if size complete")
arr = np.empty(shape, dtype)
bytesleft=arr.nbytes
bytesread=0
while(bytesleft > _BLOCKSIZE):
data = infile.read(_BLOCKSIZE)
if len(data) != _BLOCKSIZE:
raise EarlyEOFError("Unexpected EOF reading data for size complete array")
arr.data[bytesread:bytesread+_BLOCKSIZE]=data
bytesread += _BLOCKSIZE
bytesleft -= _BLOCKSIZE
if bytesleft > 0:
data = infile.read(bytesleft)
if len(data) != bytesleft:
raise EarlyEOFError("Unexpected EOF reading data for size complete array")
arr.data[bytesread:bytesread+bytesleft]=data
return arr
##shape is incompletely specified
##read until EOF
##implementation 1: naively use memory blocks
##problematic because memory allocation can be double what is
##necessary (!)
##the most common case, namely reading in data from an unchanging
##file whose size may be determined before allocation, should be
##quick -- only one allocation will be needed.
recsize = int(dtype.itemsize * np.product([i for i in shape if i != -1]))
blocksize = max(_BLOCKSIZE//recsize, 1)*recsize
##try to estimate file size
try:
curpos=infile.tell()
infile.seek(0,2)
endpos=infile.tell()
infile.seek(curpos)
except (AttributeError, IOError):
initsize=blocksize
else:
initsize=max(1,(endpos-curpos)//recsize)*recsize
buf = np.newbuffer(initsize)
bytesread=0
while 1:
data=infile.read(blocksize)
if len(data) != blocksize: ##eof
break
##do we have space?
if len(buf) < bytesread+blocksize:
buf=_resizebuf(buf,len(buf)+blocksize)
## or rather a=resizebuf(a,2*len(a)) ?
assert len(buf) >= bytesread+blocksize
buf[bytesread:bytesread+blocksize]=data
bytesread += blocksize
if len(data) % recsize != 0:
if sizing == STRICT:
raise SizeMismatchError("Filesize does not match specified shape")
if sizing == WARN:
_warnings.warn("Filesize does not match specified shape",
SizeMismatchWarning)
try:
infile.seek(-(len(data) % recsize),1)
except AttributeError:
_warnings.warn("Could not rewind (no seek support)",
FileSeekWarning)
except IOError:
_warnings.warn("Could not rewind (IOError in seek)",
FileSeekWarning)
datasize = (len(data)//recsize) * recsize
if len(buf) != bytesread+datasize:
buf=_resizebuf(buf,bytesread+datasize)
buf[bytesread:bytesread+datasize]=data[:datasize]
##deduce shape from len(buf)
shape = list(shape)
uidx = shape.index(-1)
shape[uidx]=len(buf) // recsize
a = np.ndarray(shape=shape, dtype=type, buffer=buf)
if a.dtype.char == '?':
np.not_equal(a, 0, a)
return a
import numpy numpy.newbuffer(1)
