def get_scores_for_category(self):
cat_counts, not_cat_counts = self._get_counts()
scores = ScaledFScore.get_scores_for_category(cat_counts, not_cat_counts)
np.testing.assert_almost_equal(scores,
[0.23991183969723384, 0.24969810634506373, 0.23991183969723384,
0.27646711056272855, 0.92885244834997516, 0.42010144843632563,
0.49166017105966719, 0.0, 0.0, 0.50262304057984664])
def test_get_scores(self):
cat_counts, not_cat_counts = self._get_counts()
scores = ScaledFScore.get_scores(cat_counts, not_cat_counts, beta=1.)
np.testing.assert_almost_equal(
scores,
np.array([
0.2689108, 0., 0.2689108, 0.1266617, 1., 0.5, 0.5590517, 0.5,
0.5, 0.5720015
]))
def get_scores_for_category(self):
cat_counts, not_cat_counts = self._get_counts()
scores = ScaledFScore.get_scores_for_category(cat_counts,
not_cat_counts)
np.testing.assert_almost_equal(scores, [
0.23991183969723384, 0.24969810634506373, 0.23991183969723384,
0.27646711056272855, 0.92885244834997516, 0.42010144843632563,
0.49166017105966719, 0.0, 0.0, 0.50262304057984664
])
def _get_scaled_f_score_from_counts(self, cat_word_counts, not_cat_word_counts, scaler_algo,
beta=DEFAULT_BETA):
'''
scaler = self._get_scaler_function(scaler_algo)
p_word_given_category = cat_word_counts.astype(np.float64) / cat_word_counts.sum()
p_category_given_word = cat_word_counts.astype(np.float64) / (cat_word_counts + not_cat_word_counts)
scores \
= self._computer_harmoic_mean_of_probabilities_over_non_zero_in_category_count_terms(
cat_word_counts, p_category_given_word, p_word_given_category, scaler
)
'''
return ScaledFScore.get_scores(cat_word_counts, not_cat_word_counts, scaler_algo, beta=beta)
def get_scaled_f_scores_vs_background(self, scaler_algo='none', beta=1.):
'''
Parameters
----------
scaler_algo : str
see get_scaled_f_scores, default 'none'
beta : float
default 1.
Returns
-------
pd.DataFrame of scaled_f_score scores compared to background corpus
'''
df = self.get_term_and_background_counts()
df['Scaled f-score'] = ScaledFScore.get_scores_for_category(
df['corpus'], df['background'], scaler_algo, beta)
return df.sort_values(by='Scaled f-score', ascending=False)
def test_get_scores_zero_median(self):
cat_counts = np.array([0, 0, 0, 0, 0, 0, 1, 2])
not_cat_counts = np.array([1, 1, 2, 1, 1, 1, 1, 3])
ScaledFScore.get_scores(cat_counts, not_cat_counts)
def test_get_scores_zero_all_same(self):
cat_counts = np.array([0, 0, 0, 0, 0, 0, 1, 2])
not_cat_counts = np.array([1, 1, 2, 1, 1, 1, 1, 2])
scores = ScaledFScore.get_scores(cat_counts, not_cat_counts)
np.testing.assert_almost_equal(scores,
[0.5, 0.5, 0, 0.5, 0.5, 0.5, 0.5, 1.])
def test_get_scores(self):
cat_counts, not_cat_counts = self._get_counts()
scores = ScaledFScore.get_scores(cat_counts, not_cat_counts, beta=1.)
np.testing.assert_almost_equal(scores,
np.array([0.2689108, 0., 0.2689108, 0.1266617, 1.,
0.5, 0.5590517, 0.5, 0.5, 0.5720015]))
def test_get_scores_zero_median(self):
cat_counts = np.array([0, 0, 0, 0, 0, 0, 1, 2])
not_cat_counts = np.array([1, 1, 2, 1, 1, 1, 1, 3])
ScaledFScore.get_scores(cat_counts, not_cat_counts)
def test_get_scores_zero_all_same(self):
cat_counts = np.array([0, 0, 0, 0, 0, 0, 1, 2])
not_cat_counts = np.array([1, 1, 2, 1, 1, 1, 1, 2])
scores = ScaledFScore.get_scores(cat_counts, not_cat_counts)
np.testing.assert_almost_equal(scores, [0.5, 0.5, 0, 0.5, 0.5, 0.5, 0.5, 1.])