def testNormalizedLogRatioMeasure(self):
        """Test normalized_log_ratio_measure()"""
        obtained = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator, normalize=True)[0]
        # Answer should be ~ -0.182321557 / -ln(1 / 5) = -0.113282753
        expected = -0.113282753
        self.assertAlmostEqual(expected, obtained)

        mean, std = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator, normalize=True, with_std=True)[0]
        self.assertAlmostEqual(expected, mean)
        self.assertEqual(0.0, std)
    def testLogRatioMeasure(self):
        """Test log_ratio_measure()"""
        obtained = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator)[0]
        # Answer should be ~ ln{(1 / 5) / [(3 / 5) * (2 / 5)]} = -0.182321557
        expected = -0.182321557
        self.assertAlmostEqual(expected, obtained)

        mean, std = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator, with_std=True)[0]
        self.assertAlmostEqual(expected, mean)
        self.assertEqual(0.0, std)
    def testLogRatioMeasure(self):
        """Test log_ratio_measure()"""
        obtained = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator)[0]
        # Answer should be ~ ln{(1 / 5) / [(3 / 5) * (2 / 5)]} = -0.182321557
        expected = -0.182321557
        self.assertAlmostEqual(expected, obtained)

        mean, std = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator, with_std=True)[0]
        self.assertAlmostEqual(expected, mean)
        self.assertEqual(0.0, std)
    def testNormalizedLogRatioMeasure(self):
        """Test normalized_log_ratio_measure()"""
        obtained = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator, normalize=True)[0]
        # Answer should be ~ -0.182321557 / -ln(1 / 5) = -0.113282753
        expected = -0.113282753
        self.assertAlmostEqual(expected, obtained)

        mean, std = direct_confirmation_measure.log_ratio_measure(
            self.segmentation, self.accumulator, normalize=True,
            with_std=True)[0]
        self.assertAlmostEqual(expected, mean)
        self.assertEqual(0.0, std)
 def testLogRatioMeasure(self):
     """Test log_ratio_measure()"""
     obtained = direct_confirmation_measure.log_ratio_measure(
         self.segmentation, self.posting_list, self.num_docs)[0]
     # Answer should be ~ ln{(1 / 5) / [(3 / 5) * (2 / 5)]} = -0.182321557
     expected = -0.182321557
     self.assertAlmostEqual(obtained, expected)
 def testNormalizedLogRatioMeasure(self):
     """Test normalized_log_ratio_measure()"""
     obtained = direct_confirmation_measure.log_ratio_measure(
         self.segmentation,
         self.posting_list,
         self.num_docs,
         normalize=True)[0]
     # Answer should be ~ -0.182321557 / -ln(1 / 5) = -0.113282753
     expected = -0.113282753
     self.assertAlmostEqual(obtained, expected)
Exemplo n.º 7
0
def pmi(topics, corpus, dictionary=None):
    """

    """
    unique_ids = unique_ids = set(chain(*topics))

    accumulator = inverted_accumulator(corpus,
                                       unique_ids,
                                       dictionary=dictionary)

    segments = segmentation.s_one_pre(topics)

    scores = direct_confirmation_measure.log_ratio_measure(
        segments, accumulator)

    return np.array(scores)
def _pair_npmi(pair, accumulator):
    """Compute normalized pairwise mutual information (**NPMI**) between a pair of words.

    Parameters
    ----------
    pair : (int, int)
        The pair of words (word_id1, word_id2).
    accumulator : :class:`~gensim.topic_coherence.text_analysis.InvertedIndexAccumulator`
        Word occurrence accumulator from probability_estimation.

    Return
    ------
    float
        NPMI between a pair of words.

    """
    return log_ratio_measure([[pair]], accumulator, True)[0]
def _pair_npmi(pair, accumulator):
    """Compute normalized pairwise mutual information (**NPMI**) between a pair of words.

    Parameters
    ----------
    pair : (int, int)
        The pair of words (word_id1, word_id2).
    accumulator : :class:`~gensim.topic_coherence.text_analysis.InvertedIndexAccumulator`
        Word occurrence accumulator from probability_estimation.

    Return
    ------
    float
        NPMI between a pair of words.

    """
    return log_ratio_measure([[pair]], accumulator, True)[0]
    #Make accumulator
    accumulator = probability_estimation.p_boolean_document(
        corpus, segmented_topics)

    #Perform the measurements and print results

    lcp = direct_confirmation_measure.log_conditional_probability(
        segmented_topics, accumulator)

    with codecs.open(outputfile1, encoding='utf-8', mode='w',
                     errors='ignore') as outputFile:
        for item in lcp:
            outputFile.write('%s \n' % (item))

    pmi = direct_confirmation_measure.log_ratio_measure(
        segmented_topics, accumulator)

    with codecs.open(outputfile2, encoding='utf-8', mode='w',
                     errors='ignore') as outputFile:
        for item in pmi:
            outputFile.write('%s \n' % (item))

    cosim = indirect_confirmation_measure.cosine_similarity(
        segmented_topics, accumulator, unsegmented_topics, 'nlr', 1)

    with codecs.open(outputfile3, encoding='utf-8', mode='w',
                     errors='ignore') as outputFile:
        for item in cosim:
            outputFile.write('%s \n' % (item))

    npmi = direct_confirmation_measure.log_ratio_measure(segmented_topics,
Exemplo n.º 11
0
def _pair_npmi(pair, accumulator):
    """Compute normalized pairwise mutual information (NPMI) between a pair of words.
    The pair is an iterable of (word_id1, word_id2).
    """
    return direct_confirmation_measure.log_ratio_measure([[pair]], accumulator,
                                                         True)[0]
 def testNormalizedLogRatioMeasure(self):
     """Test normalized_log_ratio_measure()"""
     obtained = direct_confirmation_measure.log_ratio_measure(self.segmentation, self.posting_list, self.num_docs, normalize=True)[0]
     # Answer should be ~ -0.182321557 / -ln(1 / 5) = -0.113282753
     expected = -0.113282753
     self.assertAlmostEqual(obtained, expected)
 def testLogRatioMeasure(self):
     """Test log_ratio_measure()"""
     obtained = direct_confirmation_measure.log_ratio_measure(self.segmentation, self.posting_list, self.num_docs)[0]
     # Answer should be ~ ln{(1 / 5) / [(3 / 5) * (2 / 5)]} = -0.182321557
     expected = -0.182321557
     self.assertAlmostEqual(obtained, expected)
Exemplo n.º 14
0
def _pair_npmi(pair, accumulator):
    """Compute normalized pairwise mutual information (NPMI) between a pair of words.
    The pair is an iterable of (word_id1, word_id2).
    """
    return direct_confirmation_measure.log_ratio_measure([[pair]], accumulator, True)[0]