def __init__(self, X, mX, y, term_idx_store, category_idx_store, metadata_idx_store, raw_texts, unigram_frequency_path=None): ''' Parameters ---------- X : csr_matrix term document matrix mX : csr_matrix metadata-document matrix y : np.array category index array term_idx_store : IndexStore Term indices category_idx_store : IndexStore Catgory indices metadata_idx_store : IndexStore Document metadata indices raw_texts : np.array or pd.Series Raw texts unigram_frequency_path : str or None Path to term frequency file. ''' TermDocMatrix.__init__(self, X, mX, y, term_idx_store, category_idx_store, metadata_idx_store, unigram_frequency_path) self._raw_texts = raw_texts
def __init__(self,
X,
mX,
y,
term_idx_store,
category_idx_store,
metadata_idx_store,
raw_texts,
unigram_frequency_path=None):
'''
Parameters
----------
X : csr_matrix
term document matrix
mX : csr_matrix
metadata-document matrix
y : np.array
category index array
term_idx_store : IndexStore
Term indices
category_idx_store : IndexStore
Catgory indices
metadata_idx_store : IndexStore
Document metadata indices
raw_texts : np.array or pd.Series
Raw texts
unigram_frequency_path : str or None
Path to term frequency file.
'''
TermDocMatrix.__init__(self, X, mX, y, term_idx_store,
category_idx_store, metadata_idx_store,
unigram_frequency_path)
self._raw_texts = raw_texts
def build_from_category_whitespace_delimited_text(category_text_iter):
'''
Parameters
----------
category_text_iter iterator of (string category name, one line per sentence, whitespace-delimited text) pairs
Returns
-------
TermDocMatrix
'''
y = []
X_factory = CSRMatrixFactory()
term_idx_store = IndexStore()
category_idx_store = IndexStore()
mX_factory = CSRMatrixFactory()
for doci, (category, text) in enumerate(category_text_iter):
y.append(category_idx_store.getidx(category))
term_freq = Counter()
for sent in text.strip(string.punctuation).lower().split('\n'):
unigrams = []
for tok in sent.strip().split():
unigrams.append(tok)
bigrams = list(map(' '.join, zip(unigrams[:-1], unigrams[1:])))
for term in unigrams + bigrams:
term_freq[term_idx_store.getidx(term)] += 1
for word_idx, freq in term_freq.items():
X_factory[doci, word_idx] = freq
metadata_idx_store = IndexStore()
return TermDocMatrix(X=X_factory.get_csr_matrix(),
mX=mX_factory.get_csr_matrix(),
y=np.array(y),
term_idx_store=term_idx_store,
metadata_idx_store=metadata_idx_store,
category_idx_store=category_idx_store)
def build(self):
'''
Returns
-------
TermDocMatrix
'''
constructor_kwargs = self._get_build_kwargs()
return TermDocMatrix(**constructor_kwargs)
def _apply_pipeline_and_get_build_instance(self, X_factory, mX_factory,
category_idx_store, df,
parse_pipeline, term_idx_store,
metadata_idx_store, y):
df.apply(parse_pipeline.parse, axis=1)
y = np.array(y)
X, mX = self._build_sparse_matrices(y, X_factory, mX_factory)
tdm = TermDocMatrix(X, mX, y, term_idx_store, category_idx_store,
metadata_idx_store)
return tdm
def build(self):
'''
Returns
-------
CorpusDF
'''
if self.text_df is not None:
if self.parsed_col is not None:
if self.category_col is None:
self.text_df = self.text_df.assign(
Category=self.category_idx_store.getvalbatch(self.y))
self.category_col = 'Category'
return ParsedCorpus(
df=self.text_df,
X=self.X,
mX=self.mX,
y=self.y,
parsed_col=self.parsed_col,
term_idx_store=self.term_idx_store,
category_idx_store=self.category_idx_store,
metadata_idx_store=self.metadata_idx_store,
unigram_frequency_path=self.unigram_frequency_path,
category_col=self.category_col)
elif self.text_col is not None:
return CorpusDF(
df=self.text_df,
X=self.X,
mX=self.mX,
y=self.y,
text_col=self.text_col,
term_idx_store=self.term_idx_store,
category_idx_store=self.category_idx_store,
metadata_idx_store=self.metadata_idx_store,
unigram_frequency_path=self.unigram_frequency_path)
return TermDocMatrix(
X=self.X,
mX=self.mX,
y=self.y,
term_idx_store=self.term_idx_store,
category_idx_store=self.category_idx_store,
metadata_idx_store=self.metadata_idx_store,
unigram_frequency_path=self.unigram_frequency_path)
def _build_from_category_spacy_doc_iter(self, category_doc_iter):
'''
Parameters
----------
category_doc_iter : iterator of (string category name, spacy.tokens.doc.Doc) pairs
Returns
----------
t : TermDocMatrix
'''
term_idx_store = IndexStore()
category_idx_store = IndexStore()
metadata_idx_store = IndexStore()
X, mX, y = self._get_features_and_labels_from_documents_and_indexes(
category_doc_iter, category_idx_store, term_idx_store,
metadata_idx_store)
return TermDocMatrix(X,
mX,
y,
term_idx_store=term_idx_store,
category_idx_store=category_idx_store,
metadata_idx_store=metadata_idx_store)