def _get_default_scores(self, category, df): category_column_name = category + ' freq' cat_word_counts = df[category_column_name] not_cat_word_counts = self._get_not_category_term_frequency( category_column_name, df) scores = ScaledFScore.get_scores(cat_word_counts, not_cat_word_counts) return scores
def _get_default_scores(self, category, other_categories, df): category_column_name = category + ' freq' cat_word_counts = df[category_column_name] not_cat_word_counts = df[[c + ' freq' for c in other_categories]].sum(axis=1) scores = ScaledFScore.get_scores(cat_word_counts, not_cat_word_counts) return scores
def _get_default_scores(self, category, df): category_column_name = category + ' freq' cat_word_counts = df[category_column_name] not_cat_word_counts = df[[ c for c in df.columns if c != category_column_name ]].sum(axis=1) scores = ScaledFScore.get_scores(cat_word_counts, not_cat_word_counts) return scores
def get_p_vals(self, X): ''' Imputes p-values from the Z-scores of `ScaledFScore` scores. Assuming incorrectly that the scaled f-scores are normally distributed. Parameters ---------- X : np.array Array of word counts, shape (N, 2) where N is the vocab size. X[:,0] is the positive class, while X[:,1] is the negative class. Returns ------- np.array of p-values ''' f_scores = ScaledFScore.get_scores(X[:,0], X[:,1], self.scaler_algo, self.beta) z_scores = (f_scores - np.mean(f_scores))/(np.std(f_scores)/np.sqrt(len(f_scores))) return norm.cdf(z_scores)