def load(file, memmap=False):
"""Load a binary file.
Read a binary file (either a pickle, or a binary .npy/.npz file) and
return the result.
Parameters
----------
file : file-like object or string
the file to read. It must support seek and read methods
memmap : bool
If true, then memory-map the .npy file or unzip the .npz file into
a temporary directory and memory-map each component
This has no effect for a pickle.
Returns
-------
result : array, tuple, dict, etc.
data stored in the file.
If file contains pickle data, then whatever is stored in the pickle is
returned.
If the file is .npy file, then an array is returned.
If the file is .npz file, then a dictionary-like object is returned
which has a filename:array key:value pair for every file in the zip.
Raises
------
IOError
"""
if isinstance(file, type("")):
fid = _file(file, "rb")
else:
fid = file
if memmap:
raise NotImplementedError
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = 'PK\x03\x04'
N = len(format.MAGIC_PREFIX)
magic = fid.read(N)
fid.seek(-N, 1) # back-up
if magic.startswith(_ZIP_PREFIX): # zip-file (assume .npz)
return NpzFile(fid)
elif magic == format.MAGIC_PREFIX: # .npy file
return format.read_array(fid)
else: # Try a pickle
try:
return _cload(fid)
except:
raise IOError, \
"Failed to interpret file %s as a pickle" % repr(file)
def load(file, memmap=False):
"""Load a binary file.
Read a binary file (either a pickle, or a binary .npy/.npz file) and
return the result.
Parameters
----------
file : file-like object or string
the file to read. It must support seek and read methods
memmap : bool
If true, then memory-map the .npy file or unzip the .npz file into
a temporary directory and memory-map each component
This has no effect for a pickle.
Returns
-------
result : array, tuple, dict, etc.
data stored in the file.
If file contains pickle data, then whatever is stored in the pickle is
returned.
If the file is .npy file, then an array is returned.
If the file is .npz file, then a dictionary-like object is returned
which has a filename:array key:value pair for every file in the zip.
Raises
------
IOError
"""
if isinstance(file, type("")):
fid = _file(file,"rb")
else:
fid = file
if memmap:
raise NotImplementedError
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = 'PK\x03\x04'
N = len(format.MAGIC_PREFIX)
magic = fid.read(N)
fid.seek(-N,1) # back-up
if magic.startswith(_ZIP_PREFIX): # zip-file (assume .npz)
return NpzFile(fid)
elif magic == format.MAGIC_PREFIX: # .npy file
return format.read_array(fid)
else: # Try a pickle
try:
return _cload(fid)
except:
raise IOError, \
"Failed to interpret file %s as a pickle" % repr(file)
def __getitem__(self, key):
# FIXME: This seems like it will copy strings around
# more than is strictly necessary. The zipfile
# will read the string and then
# the format.read_array will copy the string
# to another place in memory.
# It would be better if the zipfile could read
# (or at least uncompress) the data
# directly into the array memory.
member = 0
if key in self._files:
member = 1
elif key in self.files:
member = 1
key += '.npy'
if member:
bytes = self.zip.read(key)
if bytes.startswith(format.MAGIC_PREFIX):
value = cStringIO.StringIO(bytes)
return format.read_array(value)
else:
return bytes
else:
raise KeyError, "%s is not a file in the archive" % key
def __getitem__(self, key):
# FIXME: This seems like it will copy strings around
# more than is strictly necessary. The zipfile
# will read the string and then
# the format.read_array will copy the string
# to another place in memory.
# It would be better if the zipfile could read
# (or at least uncompress) the data
# directly into the array memory.
member = 0
if key in self._files:
member = 1
elif key in self.files:
member = 1
key += '.npy'
if member:
bytes = self.zip.read(key)
if bytes.startswith(format.MAGIC_PREFIX):
value = cStringIO.StringIO(bytes)
return format.read_array(value)
else:
return bytes
else:
raise KeyError, "%s is not a file in the archive" % key
def load(file, mmap_mode=None, dist=False):
"""
Load a pickled, ``.npy``, or ``.npz`` binary file.
Parameters
----------
file : file-like object or string
The file to read. It must support ``seek()`` and ``read()`` methods.
mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
If not None, then memory-map the file, using the given mode
(see `numpy.memmap`). The mode has no effect for pickled or
zipped files.
A memory-mapped array is stored on disk, and not directly loaded
into memory. However, it can be accessed and sliced like any
ndarray. Memory mapping is especially useful for accessing
small fragments of large files without reading the entire file
into memory.
Returns
-------
result : array, tuple, dict, etc.
Data stored in the file.
Raises
------
IOError
If the input file does not exist or cannot be read.
Notes
-----
- If the file contains pickle data, then whatever is stored in the
pickle is returned.
- If the file is a ``.npy`` file, then an array is returned.
- If the file is a ``.npz`` file, then a dictionary-like object is
returned, containing ``{filename: array}`` key-value pairs, one for
each file in the archive.
Examples
--------
Store data to disk, and load it again:
>>> np.save('/tmp/123', np.array([[1, 2, 3], [4, 5, 6]]))
>>> np.load('/tmp/123.npy')
array([[1, 2, 3],
[4, 5, 6]])
Mem-map the stored array, and then access the second row
directly from disk:
>>> X = np.load('/tmp/123.npy', mmap_mode='r')
>>> X[1, :]
memmap([4, 5, 6])
"""
import gzip
if isinstance(file, basestring):
fid = _file(file,"rb")
elif isinstance(file, gzip.GzipFile):
fid = seek_gzip_factory(file)
else:
fid = file
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = 'PK\x03\x04'
N = len(format.MAGIC_PREFIX)
magic = fid.read(N)
fid.seek(-N,1) # back-up
if magic.startswith(_ZIP_PREFIX): # zip-file (assume .npz)
return NpzFile(fid)
elif magic == format.MAGIC_PREFIX: # .npy file
if mmap_mode:
return format.open_memmap(file, mode=mmap_mode)
else:
# DISTNUMPY
return format.read_array(fid, dist)
else: # Try a pickle
try:
return _cload(fid)
except:
raise IOError, \
"Failed to interpret file %s as a pickle" % repr(file)
def load(file, mmap_mode=None):
"""
Load a pickled, ``.npy``, or ``.npz`` binary file.
Parameters
----------
file : file-like object or string
The file to read. It must support ``seek()`` and ``read()`` methods.
mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
If not None, then memory-map the file, using the given mode
(see `numpy.memmap`). The mode has no effect for pickled or
zipped files.
A memory-mapped array is stored on disk, and not directly loaded
into memory. However, it can be accessed and sliced like any
ndarray. Memory mapping is especially useful for accessing
small fragments of large files without reading the entire file
into memory.
Returns
-------
result : array, tuple, dict, etc.
Data stored in the file.
Raises
------
IOError
If the input file does not exist or cannot be read.
Notes
-----
- If the file contains pickle data, then whatever is stored in the
pickle is returned.
- If the file is a ``.npy`` file, then an array is returned.
- If the file is a ``.npz`` file, then a dictionary-like object is
returned, containing ``{filename: array}`` key-value pairs, one for
each file in the archive.
Examples
--------
Store data to disk, and load it again:
>>> np.save('/tmp/123', np.array([[1, 2, 3], [4, 5, 6]]))
>>> np.load('/tmp/123.npy')
array([[1, 2, 3],
[4, 5, 6]])
Mem-map the stored array, and then access the second row
directly from disk:
>>> X = np.load('/tmp/123.npy', mmap_mode='r')
>>> X[1, :]
memmap([4, 5, 6])
"""
import gzip
if isinstance(file, basestring):
fid = _file(file, "rb")
elif isinstance(file, gzip.GzipFile):
fid = seek_gzip_factory(file)
else:
fid = file
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = 'PK\x03\x04'
N = len(format.MAGIC_PREFIX)
magic = fid.read(N)
fid.seek(-N, 1) # back-up
if magic.startswith(_ZIP_PREFIX): # zip-file (assume .npz)
return NpzFile(fid)
elif magic == format.MAGIC_PREFIX: # .npy file
if mmap_mode:
return format.open_memmap(file, mode=mmap_mode)
else:
return format.read_array(fid)
else: # Try a pickle
try:
return _cload(fid)
except:
raise IOError, \
"Failed to interpret file %s as a pickle" % repr(file)
def load(file, memmap=False):
"""
Load a pickled, ``.npy``, or ``.npz`` binary file.
Parameters
----------
file : file-like object or string
The file to read. It must support ``seek()`` and ``read()`` methods.
memmap : bool
If True, then memory-map the ``.npy`` file (or unzip the ``.npz`` file
into a temporary directory and memory-map each component). This has
no effect for a pickled file.
Returns
-------
result : array, tuple, dict, etc.
Data stored in the file.
Raises
------
IOError
If the input file does not exist or cannot be read.
Notes
-----
- If file contains pickle data, then whatever is stored in the
pickle is returned.
- If the file is a ``.npy`` file, then an array is returned.
- If the file is a ``.npz`` file, then a dictionary-like object is
returned, containing {filename: array} key-value pairs, one for
every file in the archive.
Examples
--------
>>> np.save('/tmp/123', np.array([1, 2, 3])
>>> np.load('/tmp/123.npy')
array([1, 2, 3])
"""
if isinstance(file, basestring):
fid = _file(file,"rb")
else:
fid = file
if memmap:
raise NotImplementedError
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = 'PK\x03\x04'
N = len(format.MAGIC_PREFIX)
magic = fid.read(N)
fid.seek(-N,1) # back-up
if magic.startswith(_ZIP_PREFIX): # zip-file (assume .npz)
return NpzFile(fid)
elif magic == format.MAGIC_PREFIX: # .npy file
return format.read_array(fid)
else: # Try a pickle
try:
return _cload(fid)
except:
raise IOError, \
"Failed to interpret file %s as a pickle" % repr(file)
def load(file, memmap=False):
"""
Load a pickled, ``.npy``, or ``.npz`` binary file.
Parameters
----------
file : file-like object or string
The file to read. It must support ``seek()`` and ``read()`` methods.
memmap : bool
If True, then memory-map the ``.npy`` file (or unzip the ``.npz`` file
into a temporary directory and memory-map each component). This has
no effect for a pickled file.
Returns
-------
result : array, tuple, dict, etc.
Data stored in the file.
Raises
------
IOError
If the input file does not exist or cannot be read.
Notes
-----
- If file contains pickle data, then whatever is stored in the
pickle is returned.
- If the file is a ``.npy`` file, then an array is returned.
- If the file is a ``.npz`` file, then a dictionary-like object is
returned, containing {filename: array} key-value pairs, one for
every file in the archive.
Examples
--------
>>> np.save('/tmp/123', np.array([1, 2, 3])
>>> np.load('/tmp/123.npy')
array([1, 2, 3])
"""
if isinstance(file, basestring):
fid = _file(file, "rb")
else:
fid = file
if memmap:
raise NotImplementedError
# Code to distinguish from NumPy binary files and pickles.
_ZIP_PREFIX = 'PK\x03\x04'
N = len(format.MAGIC_PREFIX)
magic = fid.read(N)
fid.seek(-N, 1) # back-up
if magic.startswith(_ZIP_PREFIX): # zip-file (assume .npz)
return NpzFile(fid)
elif magic == format.MAGIC_PREFIX: # .npy file
return format.read_array(fid)
else: # Try a pickle
try:
return _cload(fid)
except:
raise IOError, \
"Failed to interpret file %s as a pickle" % repr(file)