Example #1
0
    def test_trigram(self):
        unigrams = NGramSpace(3)

        x = unigrams.parse("This is a sentence")
        y = unigrams.parse("This is another sentence")

        self.assertEqual([1, 2], x)
        self.assertEqual([3, 4], y)

        self.assertEqual(0, overlap(x, y))
        self.assertEqual(0, overlap(y, x))

        self.assertEqual(0, jaccard(x, y))
        self.assertEqual(0, jaccard(y, x))
Example #2
0
    def test_bigram(self):
        unigrams = NGramSpace(2)

        x = unigrams.parse("This is a sentence")
        y = unigrams.parse("This is another sentence")

        self.assertEqual([1, 2, 3], x)
        self.assertEqual([1, 4, 5], y)

        self.assertEqual(1, overlap(x, y))
        self.assertEqual(1, overlap(y, x))

        self.assertEqual(1.0 / 5.0, jaccard(x, y))
        self.assertEqual(1.0 / 5.0, jaccard(y, x))
    def test_trigram(self):
        unigrams = NGramSpace(3)

        x = unigrams.parse('This is a sentence')
        y = unigrams.parse('This is another sentence')

        self.assertEqual([1, 2], x)
        self.assertEqual([3, 4], y)

        self.assertEqual(0, overlap(x, y))
        self.assertEqual(0, overlap(y, x))

        self.assertEqual(0, jaccard(x, y))
        self.assertEqual(0, jaccard(y, x))
    def test_bigram(self):
        unigrams = NGramSpace(2)

        x = unigrams.parse('This is a sentence')
        y = unigrams.parse('This is another sentence')

        self.assertEqual([1, 2, 3], x)
        self.assertEqual([1, 4, 5], y)

        self.assertEqual(1, overlap(x, y))
        self.assertEqual(1, overlap(y, x))

        self.assertEqual(1.0 / 5.0, jaccard(x, y))
        self.assertEqual(1.0 / 5.0, jaccard(y, x))
 def __init__(self, docs):
     self.num_docs = len(docs)
     self.assignments = range(0, self.num_docs)
     
     self.distance = SymmetricMatrix(self.num_docs)
     count = 0
     for i in range(0, self.num_docs):
         for j in range(0, i + 1):
             self.distance[i, j] = 1.0 - jaccard(docs[i], docs[j])
             
             count += 1
             if count % 1000000 == 0:
                 print "Computed %d distances out of %d..." % (count, self.num_docs * self.num_docs / 2)
     
     
     for i in range(0, self.num_docs):
         for j in range(0, i):
             if self.distance[i, j] == 0 and self.assignments[i] != self.assignments[j]:
                 self.merge(i, j)
    def __init__(self, docs):
        self.num_docs = len(docs)
        self.assignments = range(0, self.num_docs)

        self.distance = SymmetricMatrix(self.num_docs)
        count = 0
        for i in range(0, self.num_docs):
            for j in range(0, i + 1):
                self.distance[i, j] = 1.0 - jaccard(docs[i], docs[j])

                count += 1
                if count % 1000000 == 0:
                    print "Computed %d distances out of %d..." % (
                        count, self.num_docs * self.num_docs / 2)

        for i in range(0, self.num_docs):
            for j in range(0, i):
                if self.distance[
                        i,
                        j] == 0 and self.assignments[i] != self.assignments[j]:
                    self.merge(i, j)