def multiDocs2DocFreqInfo(labeled_documents:AvailableInputTypes, n_jobs:int=1, path_working_dir:str=tempfile.mkdtemp(), is_use_cache: bool = True)->SetDocumentInformation: """This function generates information for constructing document-frequency matrix. """ assert isinstance(labeled_documents, (SqliteDict, dict)) type_flag = set([judge_feature_type(docs) for docs in list(labeled_documents.values())]) assert len(type_flag)==1 counted_frequency = joblib.Parallel(n_jobs=n_jobs)( joblib.delayed(generate_document_dict)(key, docs) for key, docs in sorted(list(labeled_documents.items()), key=lambda key_value_tuple: key_value_tuple[0])) ### construct [{}] structure for input of DictVectorizer() ### seq_feature_documents = (dict(label_freqCounter_tuple[1]) for label_freqCounter_tuple in counted_frequency) ### Save index-string dictionary if is_use_cache: dict_matrix_index = init_cache_object('matrix_element_object', path_working_dir) else: dict_matrix_index = {} # use sklearn feature-extraction vec = DictVectorizer() dict_matrix_index['matrix_object'] = vec.fit_transform(seq_feature_documents).tocsr() dict_matrix_index['feature2id'] = {feat:feat_id for feat_id, feat in enumerate(vec.get_feature_names())} dict_matrix_index['label2id'] = {label_freqCounter_tuple[0]:label_id for label_id, label_freqCounter_tuple in enumerate(counted_frequency)} return SetDocumentInformation(dict_matrix_index)
def multiDocs2DocFreqInfo(labeled_documents:AvailableInputTypes, n_jobs:int=1)->SetDocumentInformation: """This function generates information for constructing document-frequency matrix. """ assert isinstance(labeled_documents, (SqliteDict, dict)) type_flag = set([judge_feature_type(docs) for docs in labeled_documents.values()]) assert len(type_flag)==1 counted_frequency = joblib.Parallel(n_jobs=n_jobs)( joblib.delayed(generate_document_dict)(key, docs) for key, docs in sorted(labeled_documents.items(), key=lambda key_value_tuple: key_value_tuple[0])) feature_documents = [dict(label_freqCounter_tuple[1]) for label_freqCounter_tuple in counted_frequency] # use sklearn feature-extraction vec = DictVectorizer() matrix_object = vec.fit_transform(feature_documents).tocsr() feature2id = {feat:feat_id for feat_id, feat in enumerate(vec.get_feature_names())} label2id = {label_freqCounter_tuple[0]:label_id for label_id, label_freqCounter_tuple in enumerate(counted_frequency)} return SetDocumentInformation(matrix_object, label2id, feature2id)