def test__turn_counts_into_matrix(self): cat_counts, not_cat_counts = self._get_counts() X = LogOddsUninformativePriorScore._turn_counts_into_matrix(cat_counts, not_cat_counts) np.testing.assert_almost_equal(X, np.array([[1, 100], [5, 510], [1, 100], [9, 199]]))
def test_get_score(self): cat_counts, not_cat_counts = self._get_counts() scores = LogOddsUninformativePriorScore.get_score(cat_counts, not_cat_counts) np.testing.assert_almost_equal( scores, #np.array([ 0.0590679, 0.1006782, 0.0590679, -0.1475645]) np.array([ 0.4447054, 0.9433088, 0.4447054, -0.9971462]) )
def test_get_score(self):
cat_counts, not_cat_counts = self._get_counts()
scores = LogOddsUninformativePriorScore.get_score(
cat_counts, not_cat_counts)
np.testing.assert_almost_equal(
scores,
#np.array([ 0.0590679, 0.1006782, 0.0590679, -0.1475645])
np.array([0.4447054, 0.9433088, 0.4447054, -0.9971462]))
import spacy
from scattertext import SampleCorpora, produce_scattertext_explorer
from scattertext.CorpusFromPandas import CorpusFromPandas
from scattertext.termscoring.LogOddsUniformativePriorScore import LogOddsUninformativePriorScore
nlp = spacy.load('en_core_web_sm')
convention_df = SampleCorpora.ConventionData2012.get_data()
corpus = CorpusFromPandas(convention_df,
category_col='party',
text_col='text',
nlp=nlp).build()
term_freq_df = corpus.get_term_freq_df()
scores = -(LogOddsUninformativePriorScore.get_thresholded_score(
term_freq_df['democrat freq'],
term_freq_df['republican freq'],
alpha_w=2.,
threshold=0.1))
html = produce_scattertext_explorer(corpus,
category='democrat',
category_name='Democratic',
not_category_name='Republican',
scores=scores,
sort_by_dist=False,
gray_zero_scores=True,
minimum_term_frequency=5,
width_in_pixels=1000,
metadata=convention_df['speaker'])
open('./demo_insignificant_greyed_out.html', 'wb').write(html.encode('utf-8'))
print('Open ./demo_insignificant_greyed_out.html in Chrome or Firefox.')
def test__turn_pvals_into_scores(self): p_vals = np.array([0.01, 0.99, 0.5, 0.1, 0.9]) scores = LogOddsUninformativePriorScore._turn_pvals_into_scores(p_vals) np.testing.assert_almost_equal(scores, [0.98, -0.98, -0., 0.8, -0.8])
def test__turn_counts_into_matrix(self):
cat_counts, not_cat_counts = self._get_counts()
X = LogOddsUninformativePriorScore._turn_counts_into_matrix(
cat_counts, not_cat_counts)
np.testing.assert_almost_equal(
X, np.array([[1, 100], [5, 510], [1, 100], [9, 199]]))
def test__turn_pvals_into_scores(self):
p_vals = np.array([0.01, 0.99, 0.5, 0.1, 0.9])
scores = LogOddsUninformativePriorScore._turn_pvals_into_scores(p_vals)
np.testing.assert_almost_equal(scores, [0.98, -0.98, -0., 0.8, -0.8])
def test_get_score_threshold(self):
cat_counts, not_cat_counts = self._get_counts()
scores = LogOddsUninformativePriorScore.get_thresholded_score(
cat_counts, not_cat_counts)
np.testing.assert_almost_equal(
scores, np.array([0., 0.9433088, 0., -0.9971462]))
def test_get_delta_hats(self):
cat_counts, not_cat_counts = self._get_counts()
scores = LogOddsUninformativePriorScore.get_delta_hats(
cat_counts, not_cat_counts)
np.testing.assert_almost_equal(
scores, np.array([-0.6095321, -1.0345766, -0.6095321, 1.5201005]))
