def _dump(self, file, protocol=2): if hasattr(file, "write"): ctx = contextlib_nullcontext(file) else: ctx = open(os_fspath(file), "wb") with ctx as f: pickle.dump(self, f, protocol=protocol)
def __new__(subtype, filename, dtype=uint8, mode='r+', offset=0, shape=None, order='C'): # Import here to minimize 'import numpy' overhead import mmap import os.path try: mode = mode_equivalents[mode] except KeyError as e: if mode not in valid_filemodes: raise ValueError("mode must be one of {!r} (got {!r})".format( valid_filemodes + list(mode_equivalents.keys()), mode)) from None if mode == 'w+' and shape is None: raise ValueError("shape must be given") if hasattr(filename, 'read'): f_ctx = contextlib_nullcontext(filename) else: f_ctx = open(os_fspath(filename), ('r' if mode == 'c' else mode) + 'b') with f_ctx as fid: fid.seek(0, 2) flen = fid.tell() descr = dtypedescr(dtype) _dbytes = descr.itemsize if shape is None: bytes = flen - offset if bytes % _dbytes: raise ValueError("Size of available data is not a " "multiple of the data-type size.") size = bytes // _dbytes shape = (size, ) else: if not isinstance(shape, tuple): shape = (shape, ) size = np.intp( 1) # avoid default choice of np.int_, which might overflow for k in shape: size *= k bytes = int(offset + size * _dbytes) if mode in ('w+', 'r+') and flen < bytes: fid.seek(bytes - 1, 0) fid.write(b'\0') fid.flush() if mode == 'c': acc = mmap.ACCESS_COPY elif mode == 'r': acc = mmap.ACCESS_READ else: acc = mmap.ACCESS_WRITE start = offset - offset % mmap.ALLOCATIONGRANULARITY bytes -= start array_offset = offset - start mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start) self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm, offset=array_offset, order=order) self._mmap = mm self.offset = offset self.mode = mode if is_pathlib_path(filename): # special case - if we were constructed with a pathlib.path, # then filename is a path object, not a string self.filename = filename.resolve() elif hasattr(fid, "name") and isinstance(fid.name, str): # py3 returns int for TemporaryFile().name self.filename = os.path.abspath(fid.name) # same as memmap copies (e.g. memmap + 1) else: self.filename = None return self
def __new__(subtype, filename, dtype=uint8, mode='r+', offset=0, shape=None, order='C'): # Import here to minimize 'import numpy' overhead import mmap import os.path try: mode = mode_equivalents[mode] except KeyError: if mode not in valid_filemodes: raise ValueError("mode must be one of %s" % (valid_filemodes + list(mode_equivalents.keys()))) if mode == 'w+' and shape is None: raise ValueError("shape must be given") if hasattr(filename, 'read'): f_ctx = contextlib_nullcontext(filename) else: f_ctx = open(os_fspath(filename), ('r' if mode == 'c' else mode)+'b') with f_ctx as fid: fid.seek(0, 2) flen = fid.tell() descr = dtypedescr(dtype) _dbytes = descr.itemsize if shape is None: bytes = flen - offset if bytes % _dbytes: raise ValueError("Size of available data is not a " "multiple of the data-type size.") size = bytes // _dbytes shape = (size,) else: if not isinstance(shape, tuple): shape = (shape,) size = np.intp(1) # avoid default choice of np.int_, which might overflow for k in shape: size *= k bytes = long(offset + size*_dbytes) if mode == 'w+' or (mode == 'r+' and flen < bytes): fid.seek(bytes - 1, 0) fid.write(b'\0') fid.flush() if mode == 'c': acc = mmap.ACCESS_COPY elif mode == 'r': acc = mmap.ACCESS_READ else: acc = mmap.ACCESS_WRITE start = offset - offset % mmap.ALLOCATIONGRANULARITY bytes -= start array_offset = offset - start mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start) self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm, offset=array_offset, order=order) self._mmap = mm self.offset = offset self.mode = mode if is_pathlib_path(filename): # special case - if we were constructed with a pathlib.path, # then filename is a path object, not a string self.filename = filename.resolve() elif hasattr(fid, "name") and isinstance(fid.name, basestring): # py3 returns int for TemporaryFile().name self.filename = os.path.abspath(fid.name) # same as memmap copies (e.g. memmap + 1) else: self.filename = None return self
def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): """Create an array from binary file data Parameters ---------- fd : str or file type If file is a string or a path-like object then that file is opened, else it is assumed to be a file object. The file object must support random access (i.e. it must have tell and seek methods). dtype : data-type, optional valid dtype for all arrays shape : int or tuple of ints, optional shape of each array. offset : int, optional Position in the file to start reading from. formats, names, titles, aligned, byteorder : If `dtype` is ``None``, these arguments are passed to `numpy.format_parser` to construct a dtype. See that function for detailed documentation Returns ------- np.recarray record array consisting of data enclosed in file. Examples -------- >>> from tempfile import TemporaryFile >>> a = np.empty(10,dtype='f8,i4,a5') >>> a[5] = (0.5,10,'abcde') >>> >>> fd=TemporaryFile() >>> a = a.newbyteorder('<') >>> a.tofile(fd) >>> >>> _ = fd.seek(0) >>> r=np.core.records.fromfile(fd, formats='f8,i4,a5', shape=10, ... byteorder='<') >>> print(r[5]) (0.5, 10, 'abcde') >>> r.shape (10,) """ if dtype is None and formats is None: raise TypeError("fromfile() needs a 'dtype' or 'formats' argument") # NumPy 1.19.0, 2020-01-01 shape = _deprecate_shape_0_as_None(shape) if shape is None: shape = (-1, ) elif isinstance(shape, int): shape = (shape, ) if hasattr(fd, 'readinto'): # GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface. # Example of fd: gzip, BytesIO, BufferedReader # file already opened ctx = contextlib_nullcontext(fd) else: # open file ctx = open(os_fspath(fd), 'rb') with ctx as fd: if offset > 0: fd.seek(offset, 1) size = get_remaining_size(fd) if dtype is not None: descr = sb.dtype(dtype) else: descr = format_parser(formats, names, titles, aligned, byteorder).dtype itemsize = descr.itemsize shapeprod = sb.array(shape).prod(dtype=nt.intp) shapesize = shapeprod * itemsize if shapesize < 0: shape = list(shape) shape[shape.index(-1)] = size // -shapesize shape = tuple(shape) shapeprod = sb.array(shape).prod(dtype=nt.intp) nbytes = shapeprod * itemsize if nbytes > size: raise ValueError( "Not enough bytes left in file for specified shape and type") # create the array _array = recarray(shape, descr) nbytesread = fd.readinto(_array.data) if nbytesread != nbytes: raise IOError("Didn't read as many bytes as expected") return _array
def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): """Create an array from binary file data If file is a string or a path-like object then that file is opened, else it is assumed to be a file object. The file object must support random access (i.e. it must have tell and seek methods). >>> from tempfile import TemporaryFile >>> a = np.empty(10,dtype='f8,i4,a5') >>> a[5] = (0.5,10,'abcde') >>> >>> fd=TemporaryFile() >>> a = a.newbyteorder('<') >>> a.tofile(fd) >>> >>> _ = fd.seek(0) >>> r=np.core.records.fromfile(fd, formats='f8,i4,a5', shape=10, ... byteorder='<') >>> print(r[5]) (0.5, 10, 'abcde') >>> r.shape (10,) """ if dtype is None and formats is None: raise TypeError("fromfile() needs a 'dtype' or 'formats' argument") # NumPy 1.19.0, 2020-01-01 shape = _deprecate_shape_0_as_None(shape) if shape is None: shape = (-1,) elif isinstance(shape, (int, long)): shape = (shape,) if isfileobj(fd): # file already opened ctx = contextlib_nullcontext(fd) else: # open file ctx = open(os_fspath(fd), 'rb') with ctx as fd: if (offset > 0): fd.seek(offset, 1) size = get_remaining_size(fd) if dtype is not None: descr = sb.dtype(dtype) else: descr = format_parser(formats, names, titles, aligned, byteorder)._descr itemsize = descr.itemsize shapeprod = sb.array(shape).prod(dtype=nt.intp) shapesize = shapeprod * itemsize if shapesize < 0: shape = list(shape) shape[shape.index(-1)] = size // -shapesize shape = tuple(shape) shapeprod = sb.array(shape).prod(dtype=nt.intp) nbytes = shapeprod * itemsize if nbytes > size: raise ValueError( "Not enough bytes left in file for specified shape and type") # create the array _array = recarray(shape, descr) nbytesread = fd.readinto(_array.data) if nbytesread != nbytes: raise IOError("Didn't read as many bytes as expected") return _array
def save_np_arrays(outfile, outdict): from numpy.compat import contextlib_nullcontext with contextlib_nullcontext(outfile) as f: np.savez_compressed(f, **outdict)