def load_svmlight_file(f, n_features=None, dtype=np.float64,
multilabel=False):
"""Load datasets in the svmlight / libsvm format into sparse CSR matrix
This format is a text-based format, with one sample per line. It does
not store zero valued features hence is suitable for sparse dataset.
The first element of each line can be used to store a target variable
to predict.
This format is used as the default format for both svmlight and the
libsvm command line programs.
Parsing a text based source can be expensive. When working on
repeatedly on the same dataset, it is recommended to wrap this
loader with joblib.Memory.cache to store a memmapped backup of the
CSR results of the first call and benefit from the near instantaneous
loading of memmapped structures for the subsequent calls.
This implementation is naive: it does allocate too much memory and
is slow since written in python. On large datasets it is recommended
to use an optimized loader such as:
https://github.com/mblondel/svmlight-loader
Parameters
----------
f: str or file-like
(Path to) a file to load.
n_features: int or None
The number of features to use. If None, it will be inferred. This
argument is useful to load several files that are subsets of a
bigger sliced dataset: each subset might not have example of
every feature, hence the inferred shape might vary from one
slice to another.
multilabel: boolean, optional
Samples may have several labels each (see
http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)
Returns
-------
(X, y)
where X is a scipy.sparse matrix of shape (n_samples, n_features),
y is a ndarray of shape (n_samples,), or, in the multilabel case,
a list of tuples of length n_samples.
"""
if hasattr(f, "read"):
return _load_svmlight_file(f, n_features, dtype, multilabel)
with open(f) as f:
return _load_svmlight_file(f, n_features, dtype, multilabel)
def load_svmlight_file(file_path, buffer_mb=40, query_id=False):
"""Load datasets in the svmlight / libsvm format into sparse CSR matrix
This format is a text-based format, with one sample per line. It does
not store zero valued features hence is suitable for sparse dataset.
The first element of each line can be used to store a target variable
to predict.
This format is used as the default format for both svmlight and the
libsvm command line programs.
Parsing a text based source can be expensive. When working on
repeatedly on the same dataset, it is recommended to wrap this
loader with joblib.Memory.cache to store a memmapped backup of the
CSR results of the first call and benefit from the near instantaneous
loading of memmapped structures for the subsequent calls.
Parameters
----------
file_path: str
Path to a file to load.
buffer_mb : integer
Buffer size to use for low level read
query_id : bool
True if the query ids has to be loaded, false otherwise
Returns
-------
(X, y, [query_ids])
where X is a dense numpy matrix of shape (n_samples, n_features) and type dtype,
y is a ndarray of shape (n_samples,).
query_ids is a ndarray of shape(nsamples,) if query_id is True, it is not returned otherwise
"""
data, labels, qids = _load_svmlight_file(file_path, buffer_mb)
# reshape the numpy array into a matrix
n_samples = len(labels)
n_features = len(data) / n_samples
data.shape = (n_samples, n_features)
if data.dtype != np.float32:
new_data = data.astype(dtype=np.float32)
del data
data = new_data
# Convert infinite values to max_float representation (SVM reader problem)
# This patch is needed because some dataset have infinite values and because
# the split condition is <=, while sole software uses <. In order to
# reconduct the former condition to the latter, we slightly decrease the
# split. However, slightly decreasing inf does not have any effect.
data[data == np.inf] = np.finfo(data.dtype).max
if not query_id:
return data, labels
else:
return data, labels, qids
def load_svmlight_file(file_path, other_file_path=None,
n_features=None, buffer_mb=40):
"""Load datasets in the svmlight / libsvm format directly into
scipy sparse CSR matrices.
Parameters
----------
file_path: str
Path to a file to load.
other_file_path: str or None
Path to another file to load. scikit-learn will make sure that the
number of features in the returned matrix is the same as for
file_path.
n_features: int or None
The number of features to use. If None, it will be inferred.
buffer_mb: int (default: 40)
The size of the buffer used while loading the dataset in mega-bytes.
Returns
-------
(X, y)
where X is a scipy.sparse matrix of shape (n_samples, n_features),
y is a ndarray of shape (n_samples,),
or, if other_file_path is not None,
(X1, y1, X2, y2)
where X1 and X2 are scipy.sparse matrices of shape
(n_samples1, n_features) and
(n_samples2, n_features),
y1 and y2 are ndarrays of shape (n_samples1,) and (n_samples2,).
Note
----
When fitting a model to a matrix X_train and evaluating it against a matrix
X_test, it is essential that X_train and X_test have the same number of
features (X_train.shape[1] == X_test.shape[1]). This may not be the case if
you load them with load_svmlight_format separately. To address this
problem, we recommend to use load_svmlight_format(train_file, test_file)
or load_svmlight_format(test_file, n_features=X_train.shape[1]).
"""
data, indices, indptr, labels = _load_svmlight_file(file_path, buffer_mb)
if n_features is not None:
shape = (indptr.shape[0] - 1, n_features)
else:
shape = None # inferred
X_train = sp.csr_matrix((data, indices, indptr), shape)
ret = [X_train, labels]
if other_file_path is not None:
tup = _load_svmlight_file(other_file_path, buffer_mb)
data, indices, indptr, labels = tup
if n_features is None:
n_features = X_train.shape[1]
shape = (indptr.shape[0] - 1, n_features)
X_test = sp.csr_matrix((data, indices, indptr), shape)
ret.append(X_test)
ret.append(labels)
return tuple(ret)
def load_svmlight_file(file_path,
other_file_path=None,
n_features=None,
buffer_mb=40):
"""Load datasets in the svmlight / libsvm format directly into
scipy sparse CSR matrices.
Parameters
----------
file_path: str
Path to a file to load.
other_file_path: str or None
Path to another file to load. scikit-learn will make sure that the
number of features in the returned matrix is the same as for
file_path.
n_features: int or None
The number of features to use. If None, it will be inferred.
buffer_mb: int (default: 40)
The size of the buffer used while loading the dataset in mega-bytes.
Returns
-------
(X, y)
where X is a scipy.sparse matrix of shape (n_samples, n_features),
y is a ndarray of shape (n_samples,),
or, if other_file_path is not None,
(X1, y1, X2, y2)
where X1 and X2 are scipy.sparse matrices of shape
(n_samples1, n_features) and
(n_samples2, n_features),
y1 and y2 are ndarrays of shape (n_samples1,) and (n_samples2,).
Note
----
When fitting a model to a matrix X_train and evaluating it against a matrix
X_test, it is essential that X_train and X_test have the same number of
features (X_train.shape[1] == X_test.shape[1]). This may not be the case if
you load them with load_svmlight_format separately. To address this
problem, we recommend to use load_svmlight_format(train_file, test_file)
or load_svmlight_format(test_file, n_features=X_train.shape[1]).
"""
data, indices, indptr, labels = _load_svmlight_file(file_path, buffer_mb)
if n_features is not None:
shape = (indptr.shape[0] - 1, n_features)
else:
shape = None # inferred
X_train = sp.csr_matrix((data, indices, indptr), shape)
ret = [X_train, labels]
if other_file_path is not None:
tup = _load_svmlight_file(other_file_path, buffer_mb)
data, indices, indptr, labels = tup
if n_features is None:
n_features = X_train.shape[1]
shape = (indptr.shape[0] - 1, n_features)
X_test = sp.csr_matrix((data, indices, indptr), shape)
ret.append(X_test)
ret.append(labels)
return tuple(ret)
