def _load_document_classification(dataset_path, metadata, set_, sparse, **kw):
"""Loader implementation for the DocumentClassification format"""
target = []
target_names = {}
filenames = []
vectorizer = kw.get('vectorizer')
if vectorizer is None:
if sparse:
vectorizer = SparseHashingVectorizer()
else:
vectorizer = HashingVectorizer()
# TODO: make it possible to plug a several pass system to filter-out tokens
# that occur in more than 30% of the documents for instance.
# TODO: use joblib.Parallel or multiprocessing to parallelize the following
# (provided this is not IO bound)
dataset_path = os.path.join(dataset_path, set_)
folders = [f for f in sorted(os.listdir(dataset_path))
if os.path.isdir(os.path.join(dataset_path, f))]
for label, folder in enumerate(folders):
target_names[label] = folder
folder_path = os.path.join(dataset_path, folder)
documents = [os.path.join(folder_path, d)
for d in sorted(os.listdir(folder_path))]
vectorizer.vectorize_files(documents)
target.extend(len(documents) * [label])
filenames.extend(documents)
return Bunch(data=vectorizer.get_vectors(), target=np.array(target),
target_names=target_names, filenames=filenames,
DESCR=metadata.get('description'))
