def __init__(self, corpus):
"""
:param corpus: corpus over which to create the language model
"""
super(BigramLanguageModel, self).__init__(corpus)
self.ngram_calculator = BasicNgramCalculator(corpus)
self.bigrams = self.ngram_calculator.calculate_ngrams(2, True, False)
def q2_calculate_unique_unigrams_in_reuters_training():
corpus = ReutersTrainingCorpus()
unigram_calculator = UnigramCalculator(corpus)
ngram_calculator = BasicNgramCalculator(corpus)
unique_unigrams = unigram_calculator.get_percentage_unique_unigrams()
unique_unigrams_ngram = ngram_calculator.get_percentage_unique_ngrams(1)
print("Percentage of unique unigrams (unigram calculator): ",
unique_unigrams)
print("Percentage of unique unigrams (ngram calculator): ",
unique_unigrams_ngram)
def q3_calculate_unique_ngrams_in_reuters_training_for_n_2_3_4_5_6():
corpus = ReutersTrainingCorpus()
ngram_calculator = BasicNgramCalculator(corpus)
unique_ngrams_2 = ngram_calculator.get_percentage_unique_ngrams(2)
unique_ngrams_3 = ngram_calculator.get_percentage_unique_ngrams(3)
unique_ngrams_4 = ngram_calculator.get_percentage_unique_ngrams(4)
unique_ngrams_5 = ngram_calculator.get_percentage_unique_ngrams(5)
unique_ngrams_6 = ngram_calculator.get_percentage_unique_ngrams(6)
print("Percentage of unique bigrams: ", unique_ngrams_2)
print("Percentage of unique trigrams: ", unique_ngrams_3)
print("Percentage of unique 4-grams: ", unique_ngrams_4)
print("Percentage of unique 5-grams: ", unique_ngrams_5)
print("Percentage of unique 6-grams: ", unique_ngrams_6)
class TestNgramCalculator:
corpus = Corpus()
basic_ngram_calculator = BasicNgramCalculator(corpus)
ngram_calculator = NgramCalculator(corpus)
flattened_sentences = [
item for sublist in corpus.get_sentences() for item in sublist
]
frequencies = FreqDist(flattened_sentences)
def test_calculate_unigrams(self):
assert dict(
self.frequencies) == self.ngram_calculator.calculate_ngrams(1)
def test_get_percentage_unique_unigrams(self):
assert round(27 / len(list(self.frequencies)), 2) == round(
self.ngram_calculator.get_percentage_unique_ngrams(1), 2)
