def _load_index_file(self):
with open(self.index_path, 'rb') as f:
self.version = pickle.load(f)
if isinstance(self.version, tuple):
if (self.version[0] > self.VERSION
or self.version[1] > pickle.HIGHEST_PROTOCOL):
raise CacheIOError("Unsupported cache index file format.")
self._index = pickle.load(f)
else:
self._index = self.version
self.version = self._get_legacy_version()
assert isinstance(self.version, tuple)
assert isinstance(self._index, dict)
def load_mnist(filepath=None, validation=False):
"""Load the MNIST dataset.
Parameters
----------
filepath : str (optional, Default: None)
Path to the previously downloaded 'mnist.pkl.gz' file.
If `None`, the file will be downloaded to the current directory.
validation : boolean (optional, Default: False)
Whether to provide the validation data as a separate set (True),
or combine it into the training data (False).
Returns
-------
train_set : (n_train, n_pixels) ndarray, (n_train,) ndarray
A tuple of the training image array and label array.
validation_set : (n_valid, n_pixels) ndarray, (n_valid,) ndarray
A tuple of the validation image array and label array (if `validation`)
test_set : (n_test, n_pixels) ndarray, (n_test,) ndarray
A tuple of the testing image array and label array.
"""
if filepath is None:
filepath = get_mnist_pkl_gz()
filepath = os.path.expanduser(filepath)
with gzip.open(filepath, 'rb') as f:
train_set, valid_set, test_set = pickle.load(f)
if validation:
return train_set, valid_set, test_set
else: # combine valid into train
train_set = (np.vstack((train_set[0], valid_set[0])),
np.hstack((train_set[1], valid_set[1])))
return train_set, test_set
def _load_index(self):
assert self._lock.acquired
try:
with open(self.filename, 'rb') as f:
return pickle.load(f)
except IOError as err:
if err.errno == errno.ENOENT:
return {}
else:
raise
def unpickle_tarfile(tar, name):
tarextract = tar.extractfile(name)
return pickle.load(tarextract)
def pickle_load_bytes(file, *args, **kwargs):
if not PY2:
kwargs.setdefault('encoding', 'bytes')
return pickle.load(file, *args, **kwargs)
def pickle_load(file, *args, **kwargs):
if not PY2:
kwargs.setdefault('encoding', 'latin1')
return pickle.load(file, *args, **kwargs)
def load_model_pickle(loadfile):
loadfile = os.path.expanduser(loadfile)
with open(loadfile, 'rb') as f:
return pickle.load(f)