def test_get_scores_zero_all_same(self): cat_counts = np.array([0, 0, 0, 0, 0, 0, 1, 2]) not_cat_counts = np.array([1, 1, 2, 1, 1, 1, 1, 2]) scores = CornerScore.get_scores(cat_counts, not_cat_counts) np.testing.assert_almost_equal(scores, np.array([0.5, 0.5, 0.15625, 0.5, 0.5, 0.5, 0.8391308, 0.6685437]))
def get_scores_for_category(self): cat_counts, not_cat_counts = self._get_counts() scores = CornerScore.get_scores_for_category(cat_counts, not_cat_counts) np.testing.assert_almost_equal(scores, np.array([0.9300538, 1.0198039, 0.9300538, 0.9055385, 0.2, 0.7433034, 0.585235, 0.9861541, 0.9861541, 0.3605551]))
def get_corner_scores(self, category):
''' Computes corner score, which is inversely correlated
to the Rudder score to the nearest upper-left or lower-right corner.
Parameters
----------
category : str
category name to score
Returns
-------
np.array
'''
return CornerScore.get_scores(
*self._get_catetgory_and_non_category_word_counts(category)
)
def to_dict(self,
category,
category_name=None,
not_category_name=None,
scores=None,
transform=percentile_alphabetical,
title_case_names=False,
not_categories=None,
neutral_categories=None,
extra_categories=None,
background_scorer=None):
'''
Parameters
----------
category : str
Category to annotate. Exact value of category.
category_name : str, optional
Name of category which will appear on web site. Default None is same as category.
not_category_name : str, optional
Name of ~category which will appear on web site. Default None is same as "not " + category.
scores : np.array, optional
Scores to use for coloring. Defaults to None, or RankDifference scores
transform : function, optional
Function for ranking terms. Defaults to scattertext.Scalers.percentile_lexicographic.
title_case_names : bool, default False
Title case category name and no-category name?
not_categories : list, optional
List of categories to use as "not category". Defaults to all others.
neutral_categories : list, optional
List of categories to use as neutral. Defaults [].
extra_categories : list, optional
List of categories to use as extra. Defaults [].
background_scorer : CharacteristicScorer, optional
Used for bg scores
Returns
-------
Dictionary that encodes the scatter chart
information. The dictionary can be dumped as a json document, and
used in scattertext.html
{info: {category_name: ..., not_category_name},
data: [{term:,
x:frequency [0-1],
y:frequency [0-1],
ox: score,
oy: score,
s: score,
os: original score,
p: p-val,
cat25k: freq per 25k in category,
cat: count in category,
ncat: count in non-category,
catdocs: [docnum, ...],
ncatdocs: [docnum, ...]
ncat25k: freq per 25k in non-category}, ...]}}
'''
if self.used:
raise Exception("Cannot reuse a ScatterChart constructor")
all_categories = self.term_doc_matrix.get_categories()
assert category in all_categories
if not_categories is None:
not_categories = [c for c in all_categories if c != category]
neutral_categories = []
extra_categories = []
elif neutral_categories is None:
neutral_categories = [
c for c in all_categories
if c not in [category] + not_categories
]
extra_categories = []
elif extra_categories is None:
extra_categories = [
c for c in all_categories
if c not in [category] + not_categories + neutral_categories
]
all_categories = [
category
] + not_categories + neutral_categories + extra_categories
df = self._get_term_category_frequencies()
self._add_x_and_y_coords_to_term_df_if_injected(df)
if scores is None:
scores = self._get_default_scores(category, not_categories, df)
category_column_name = category + ' freq'
df['category score'] = CornerScore.get_scores_for_category(
df[category_column_name],
df[[c + ' freq' for c in not_categories]].sum(axis=1))
if self.scatterchartdata.term_significance is not None:
df['p'] = get_p_vals(df, category_column_name,
self.scatterchartdata.term_significance)
df['not category score'] = CornerScore.get_scores_for_category(
df[[c + ' freq' for c in not_categories]].sum(axis=1),
df[category_column_name])
df['color_scores'] = scores
if self.scatterchartdata.terms_to_include is None:
df = self._filter_bigrams_by_minimum_not_category_term_freq(
category_column_name, not_categories, df)
df = filter_bigrams_by_pmis(
self._filter_by_minimum_term_frequency(all_categories, df),
threshold_coef=self.scatterchartdata.pmi_threshold_coefficient)
if self.scatterchartdata.filter_unigrams:
df = filter_out_unigrams_that_only_occur_in_one_bigram(df)
if len(df) == 0:
raise NoWordMeetsTermFrequencyRequirementsError()
df['category score rank'] = rankdata(df['category score'],
method='ordinal')
df['not category score rank'] = rankdata(df['not category score'],
method='ordinal')
if self.scatterchartdata.max_terms and self.scatterchartdata.max_terms < len(
df):
assert self.scatterchartdata.max_terms > 0
df = self._limit_max_terms(category, df)
df = df.reset_index()
if self.x_coords is None:
self.x_coords, self.y_coords = self._get_coordinates_from_transform_and_jitter_frequencies \
(category, df, not_categories, transform)
df['x'], df['y'] = self.x_coords, self.y_coords
df['ox'], df['oy'] = self.x_coords, self.y_coords
df['not cat freq'] = df[[x + ' freq'
for x in not_categories]].sum(axis=1)
if neutral_categories != []:
df['neut cat freq'] = df[[x + ' freq' for x in neutral_categories
]].sum(axis=1).fillna(0)
if extra_categories != []:
df['extra cat freq'] = df[[x + ' freq' for x in extra_categories
]].sum(axis=1).fillna(0)
json_df = df[['x', 'y', 'ox', 'oy', 'term']]
if self.scatterchartdata.term_significance:
json_df['p'] = df['p']
self._add_term_freq_to_json_df(json_df, df, category)
json_df['s'] = percentile_min(df['color_scores'])
json_df['os'] = df['color_scores']
if background_scorer:
bg_scores = background_scorer.get_scores(self.term_doc_matrix)
json_df['bg'] = bg_scores[1].loc[json_df.term].values
elif not self.scatterchartdata.use_non_text_features:
json_df['bg'] = self._get_corpus_characteristic_scores(json_df)
self._preform_axis_rescale(json_df, self._rescale_x, 'x')
self._preform_axis_rescale(json_df, self._rescale_y, 'y')
if self.scatterchartdata.terms_to_include is not None:
json_df = self._use_only_selected_terms(json_df)
category_terms = list(json_df.sort_values('s')['term'][:10])
not_category_terms = list(json_df.sort_values('s')['term'][:10])
if category_name is None:
category_name = category
if not_category_name is None:
not_category_name = 'Not ' + category_name
def better_title(x):
if title_case_names:
return ' '.join(
[t[0].upper() + t[1:].lower() for t in x.split()])
else:
return x
j = {
'info': {
'category_name': better_title(category_name),
'not_category_name': better_title(not_category_name),
'category_terms': category_terms,
'not_category_terms': not_category_terms,
'category_internal_name': category,
'not_category_internal_names': not_categories,
'categories': self.term_doc_matrix.get_categories(),
'neutral_category_internal_names': neutral_categories,
'extra_category_internal_names': extra_categories
}
}
if self.metadata_term_lists is not None:
j['metalists'] = self.metadata_term_lists
if self.metadata_descriptions is not None:
j['metadescriptions'] = self.metadata_descriptions
if self.term_colors is not None:
j['info']['term_colors'] = self.term_colors
#j['data'] = json_df.sort_values(by=['x', 'y', 'term']).to_dict(orient='records')
j['data'] = json_df.to_dict(orient='records')
return j
def test_get_scores(self): cat_counts, not_cat_counts = self._get_counts() scores = CornerScore.get_scores(cat_counts, not_cat_counts) np.testing.assert_almost_equal(scores, np.array([0.1820027, 0.2828427, 0.1820027, 0.5, 0.9292893, 0.2378287, 0.7930882, 0.1845603, 0.1845603, 0.8725245]))
def test_get_scores_zero_median(self): cat_counts = np.array([0, 0, 0, 0, 0, 0, 1, 2]) not_cat_counts = np.array([1, 1, 2, 1, 1, 1, 1, 3]) CornerScore.get_scores(cat_counts, not_cat_counts)