def test_occurrence_probability_with_no_occurence_in_one_window(self):
# this test the else branch where the word is not on the window sum_f[word] == 0
tokens = ["sarasa", "sarasa", "lalala", "lalala"]
calculator = InformationValueCalculator(tokens)
res = calculator.occurrence_probability(2, tokens)
expected = {'lalala': [0.0, 1.0], 'sarasa': [1.0, 0.0]}
self.assertEquals(res, expected)
def test_entropy_for_localized_word(self):
tokens = ["foo"] * 2 + ["bar"] * 6
iv_calculator = InformationValueCalculator(tokens)
entropy_dict = iv_calculator.entropy(window_size=2,
tokenized_text=tokens)
self.assertAlmostEqual(entropy_dict["foo"], 0.0)
def test_entropy_for_single_word_text(self):
tokens = ["foo", "foo", "foo"]
iv_calculator = InformationValueCalculator(tokens)
entropy_dict = iv_calculator.entropy(window_size=1,
tokenized_text=tokens)
self.assertItemsEqual(entropy_dict.keys(), ["foo"])
self.assertAlmostEqual(entropy_dict["foo"], 1.0)
def test_iv_for_single_word_text(self):
tokens = ["foo"] * 100
iv_calculator = InformationValueCalculator(tokens)
information_value = iv_calculator.information_value(window_size=10)
self.assertItemsEqual(information_value.keys(), ["foo"])
self.assertAlmostEqual(information_value["foo"], 0.0)
def test_entropy_calculation_is_stable(self):
#Fixture
tokens = ["foo"] * 10000
iv_calculator = InformationValueCalculator(tokens)
#Stimulus
entropy_dict = iv_calculator.entropy(window_size=10,
tokenized_text=tokens)
#Check
self.assertAlmostEqual(entropy_dict["foo"], 1.0)
def test_frequency_for_single_word_single_windows(self):
tokens = ["foo", "foo", "foo"]
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, window_size=3)
self.assertItemsEqual(freq.keys(), ["foo"])
self.assertEqual(len(freq["foo"]), 1)
self.assertAlmostEqual(freq["foo"][0], 1)
def test_occurrence_probability_for_single_word_in_vocabulary(self):
tokens = ["foo", "foo", "foo"]
iv_calculator = InformationValueCalculator(tokens)
p_i = iv_calculator.occurrence_probability(window_size=1,
tokenized_text=tokens)
self.assertItemsEqual(p_i.keys(), ["foo"])
for i in range(3):
self.assertAlmostEqual(p_i["foo"][i], 1 / 3)
def test_entropy_for_non_localized_nor_homogeneous_word(self):
tokens = ["foo", "bar"] * 100 + ["doe"] * 50
random.shuffle(tokens)
iv_calculator = InformationValueCalculator(tokens)
entropy_dict = iv_calculator.entropy(window_size=10,
tokenized_text=tokens)
self.assertNotAlmostEqual(entropy_dict["doe"], 1.0)
self.assertNotAlmostEqual(entropy_dict["doe"], 0.0)
def test_frequency_for_single_word_many_windows(self):
tokens = ["foo", "foo", "foo"]
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, 1)
self.assertItemsEqual(freq.keys(), ["foo"])
self.assertEqual(len(freq["foo"]), 3)
for i in range(3):
self.assertAlmostEqual(freq["foo"][i], 1)
def test_iv_for_very_concentred_word(self):
tokens = ["foo", "bar", "doe"
] * 10 + ["john", "bar"] * 5 + ["foo", "bar", "doe"] * 10
iv_calculator = InformationValueCalculator(tokens)
information_value = iv_calculator.information_value(window_size=10)
print "Iv = %s" % information_value
self.assertItemsEqual(information_value.keys(),
["foo", "john", "bar", "doe"])
self.assertNotAlmostEqual(information_value["john"], 0.0)
def test_frequency_for_many_words_one_window(self):
tokens = ["one", "two", "three", "four", "five"]
window_size = 5
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, window_size=window_size)
self.assertItemsEqual(freq.keys(), tokens)
for token in tokens:
self.assertEqual(len(freq[token]), 1)
self.assertAlmostEqual(freq[token][0], 1 / window_size)
def test_all_frequencies_for_a_given_window_sum_1(self):
words = ["foo", "bar", "john", "doe", "random"]
tokens = words * 20
random.shuffle(tokens)
window_size = 5
number_of_windows = int(math.ceil(len(tokens) / window_size))
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, window_size=window_size)
for window_no in range(number_of_windows):
sum_for_window = sum([freq[word][window_no] for word in words])
self.assertAlmostEqual(sum_for_window, 1.0)
def test_frequency_for_many_words_many_windows(self):
amount_of_windows = 10
window_size = 5
words = ["one", "two", "three", "four", "five"]
tokens = ["one", "two", "three", "four", "five"] * amount_of_windows
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, window_size=window_size)
self.assertItemsEqual(freq.keys(), words)
for token in words:
self.assertEqual(len(freq[token]), amount_of_windows)
for window_no in range(amount_of_windows):
self.assertAlmostEqual(freq[token][window_no], 1 / window_size)
def test_frequencies_against_oracle(self):
tokens = get_moby_dick_tokens()
words = list(set(tokens))
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, window_size=1000)
freq_oracle = iv_oracle.get_frequencies(tokens,
words,
window_size=1000)
for word in words:
self.assertEqual(len(freq[word]), len(freq_oracle[word]))
for i in xrange(len(freq[word])):
if round(freq[word][i] - freq_oracle[word][i], 6) > .0:
self.fail("Frequency differs for %s = %s oracle = %s" %
(word, freq[word], freq_oracle[word]))
def test_entropy_against_oracle(self):
tokens = get_moby_dick_tokens()[:60000]
words = list(set(tokens))
iv_calculator = InformationValueCalculator(tokens)
res = iv_calculator.entropy(window_size=1000, tokenized_text=tokens)
expected = iv_oracle.get_entropy(tokens, words, 1000)
print "Imprimiendo diferencias entre entropias"
for word in iv_calculator.words:
abs_err = res[word] - expected[word]
if res[word] > .0:
rel_err = abs_err / res[word]
if abs_err > .0:
print "%s has rel_err %s got = %s expected = %s" % (
word, rel_err, res[word], expected[word])
def test_occurrence_probability_against_oracle(self):
tokens = get_moby_dick_tokens()
words = list(set(tokens))
iv_calculator = InformationValueCalculator(tokens)
occ_prob = iv_calculator.occurrence_probability(window_size=1000,
tokenized_text=tokens)
occ_prob_oracle = iv_oracle.get_occurrence_probability(
tokens, words, window_size=1000)
for word in words:
ours = occ_prob[word]
theirs = occ_prob_oracle[word]
self.assertEqual(len(ours), len(theirs))
for i in xrange(len(ours)):
if round(ours[i] - theirs[i], 6) > .0:
self.fail(
"occurrence_probability differs for %s = %s oracle = %s"
% (word, ours, theirs))
def test_iv_against_oracle(self):
tokens = get_moby_dick_tokens()
iv_calculator = InformationValueCalculator(tokens)
res = iv_calculator.information_value(window_size=1000)
expected = iv_oracle.get_iv(tokens, 1000)
test_words = [
"whale", "you", "ahab", "is", "ye", "queequeg", "thou", "me", "of",
"he", "captain", "boat", "the", "stubb", "his", "jonah", "was",
"whales", "my"
]
for word in test_words:
abs_err = abs(res[word] - expected[word])
rel_err = abs_err / res[word]
self.assertLessEqual(
rel_err, 0.25, "%s has difference %s got = %s expected = %s" %
(word, rel_err, res[word], expected[word]))
def test_frequency_non_homogeneous_distribution(self):
words = ["foo", "bar", "doe"]
window_size = 3
tokens = [
"foo", "foo", "bar", "doe", "bar", "foo", "doe", "doe", "doe"
]
iv_calculator = InformationValueCalculator(tokens)
freq = iv_calculator.get_frequencies(tokens, window_size=window_size)
self.assertAlmostEqual(freq["foo"][0], 2 / 3)
self.assertAlmostEqual(freq["bar"][0], 1 / 3)
self.assertAlmostEqual(freq["doe"][0], 0 / 3)
self.assertAlmostEqual(freq["foo"][1], 1 / 3)
self.assertAlmostEqual(freq["bar"][1], 1 / 3)
self.assertAlmostEqual(freq["doe"][1], 1 / 3)
self.assertAlmostEqual(freq["foo"][2], 0)
self.assertAlmostEqual(freq["bar"][2], 0)
self.assertAlmostEqual(freq["doe"][2], 1)
def test_occurrence_probability_for_separated_text(self):
tokens = ["foo"] * 3 + ["bar"] * 3 + ["doe"] * 3
iv_calculator = InformationValueCalculator(tokens)
p_i = iv_calculator.occurrence_probability(window_size=3,
tokenized_text=tokens)
self.assertItemsEqual(p_i.keys(), ["foo", "bar", "doe"])
# P_i should be 1 only for first chapter
self.assertAlmostEqual(p_i["foo"][0], 1.0)
self.assertAlmostEqual(p_i["foo"][1], .0)
self.assertAlmostEqual(p_i["foo"][2], .0)
# 1 only for second...
self.assertAlmostEqual(p_i["bar"][0], 0)
self.assertAlmostEqual(p_i["bar"][1], 1.0)
self.assertAlmostEqual(p_i["bar"][2], 0)
# 1 only for the third
self.assertAlmostEqual(p_i["doe"][0], .0)
self.assertAlmostEqual(p_i["doe"][1], .0)
self.assertAlmostEqual(p_i["doe"][2], 1.0)
def test_occurrence_probability_with_large_window(self):
tokens = ["sarasa", "lalal", "pepe", "pepe2"]
calculator = InformationValueCalculator(tokens)
from information_value.calculator import WindowSizeTooLarge
with self.assertRaises(WindowSizeTooLarge):
calculator.occurrence_probability(4, "sarasa")