def test_empty_corpus(self): """ Test the token extractor with an empty corpus. """ extractor = TokenExtractor() candidates = extractor.extract([]) self.assertFalse(len(candidates))
def test_sorting(self): """ Test that the resolver sorts the tokens in descending order of score. """ """ Create the test data """ tokenizer = Tokenizer(min_length=3, stem=False, case_fold=True) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", "Tottenham lose again", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] """ Ensure that the more common candidates are ranked towards the beginning. """ candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) self.assertTrue(scores) resolved, unresolved = Resolver().resolve(scores) self.assertEqual(set(scores.keys()), set(resolved)) self.assertEqual([], unresolved) self.assertEqual('tottenham', resolved[0]) self.assertEqual(set(['manchester', 'united']), set(resolved[1:3]))
def test_sorting(self): """ Test that the resolver sorts the tokens in descending order of score. """ """ Create the test data """ tokenizer = Tokenizer(min_length=3, stem=False, case_fold=True) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", "Tottenham lose again", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) resolved, unresolved = TokenResolver(tokenizer, corpus).resolve(scores) self.assertEqual('tottenham', resolved[0]) self.assertEqual(set(['manchester', 'united']), set(resolved[1:3])) self.assertEqual( set([ 'falter', 'against', 'hotspur', 'unable', 'avoid', 'defeat', 'lose', 'again' ]), set(resolved[3:]))
def test_repeated_tokens(self): """ Test that when tokens are repeated, the frequency that is returned is the term frequency. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "After Erdogan's statement, Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor(tokenizer=tokenizer) scorer = TFScorer() candidates = extractor.extract(corpus) scores = scorer.score(candidates, normalize_scores=False) self.assertEqual(2, scores.get('erdogan'))
def test_score_across_multiple_documents(self): """ Test that the score is based on term frequency. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "Erdogan with threats to attack regime forces 'everywhere' in Syria", "After Erdogan's statement, Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor(tokenizer=tokenizer) scorer = TFScorer() candidates = extractor.extract(corpus) scores = scorer.score(candidates, normalize_scores=False) self.assertEqual(3, scores.get('erdogan'))
def test_normalization(self): """ Test that when normalization is disabled, the returned scores are integers. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "Erdogan with threats to attack regime forces 'everywhere' in Syria", "After Erdogan's statement, Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() scorer = TFScorer() candidates = extractor.extract(corpus) scores = scorer.score(candidates, normalize_scores=False) self.assertEqual(2, scores.get('erdogan'))
def test_max_score(self): """ Test that the maximum score is 1 when normalization is enabled. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "Erdogan with threats to attack regime forces 'everywhere' in Syria", "Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() scorer = TFScorer() candidates = extractor.extract(corpus) scores = scorer.score(candidates) self.assertTrue(all(score <= 1 for score in scores.values()))
def test_score_of_unknown_token(self): """ Test that the score of an unknown token is 0. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "Erdogan with threats to attack regime forces 'everywhere' in Syria", "Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() scorer = TFScorer() candidates = extractor.extract(corpus) scores = scorer.score(candidates) self.assertFalse(scores.get('unknown'))
def test_return_length(self): """ Test that the token extractor returns as many token sets as the number of documents given. """ """ Create the test data. """ tokenizer = Tokenizer(stopwords=stopwords.words("english"), stem=False) posts = [ "Manchester United falter against Tottenham Hotspur", "", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() candidates = extractor.extract(corpus) self.assertEqual(2, len(candidates)) self.assertEqual( set(["manchester", "united", "falter", "tottenham", "hotspur"]), set(candidates[0])) self.assertEqual(set([]), set(candidates[1]))
def test_logarithm_base(self): """ Test that when a logarithmic base is provided, it is used instead of the default base. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "Erdogan with threats to attack regime forces 'everywhere' in Syria", "After Erdogan's statement, Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [ Document(post, tokenizer.tokenize(post)) for post in posts ] extractor = TokenExtractor() scorer = LogDFScorer(base=2) candidates = extractor.extract(corpus) scores = scorer.score(candidates, normalize_scores=False) self.assertEqual(math.log(2 + 1, 2), scores.get('erdogan')) # apply Laplace smoothing
def test_tf_scorer(self): """ Test the basic functionality of the TF scorer. """ """ Create the test data. """ tokenizer = Tokenizer(stem=False) posts = [ "Erdogan with threats to attack regime forces 'everywhere' in Syria", "Damascus says Erdogan 'disconnected from reality' after threats", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() scorer = TFScorer() candidates = extractor.extract(corpus) scores = scorer.score(candidates) self.assertEqual(1, scores.get('erdogan', 0)) self.assertEqual(0.5, scores.get('damascus', 0)) self.assertEqual(1, scores.get('threats', 0))
def test_token_extractor(self): """ Test the token extractor with normal input. """ """ Create the test data. """ tokenizer = Tokenizer(stopwords=stopwords.words("english"), stem=False) posts = [ "Manchester United falter against Tottenham Hotspur", "Mourinho under pressure as Manchester United follow with a loss", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() candidates = extractor.extract(corpus) self.assertEqual( set(["manchester", "united", "falter", "tottenham", "hotspur"]), set(candidates[0])) self.assertEqual( set([ "mourinho", "pressure", "manchester", "united", "follow", "loss" ]), set(candidates[1]))
def test_empty_corpus(self): """ Test that when an empty corpus is given, all candidates are unresolved. """ """ Create the test data """ tokenizer = Tokenizer(min_length=1, stem=False) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) resolved, unresolved = TokenResolver(tokenizer, []).resolve(scores) self.assertEqual(len(scores), len(unresolved))
def test_unknown_token(self): """ Test that when an unknown candidate is given, it is unresolved. """ """ Create the test data """ tokenizer = Tokenizer(min_length=1, stem=False) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) resolved, unresolved = TokenResolver(tokenizer, corpus).resolve({'unknown': 1}) self.assertTrue('unknown' in unresolved)
def test_token_resolver(self): """ Test the token resolver. """ """ Create the test data """ tokenizer = Tokenizer(min_length=1, stem=False) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) resolved, unresolved = TokenResolver(tokenizer, corpus).resolve(scores) self.assertTrue('manchester' in resolved) self.assertTrue('united' in resolved) self.assertTrue('tottenham' in resolved) self.assertTrue('hotspur' in resolved)
def test_different_tokenizer(self): """ Test that when a different tokenizer is used than the one used in extraction, it is used. """ """ Create the test data """ tokenizer = Tokenizer(min_length=1, stem=False) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) resolved, unresolved = TokenResolver(tokenizer, corpus).resolve(scores) self.assertTrue('to' in resolved) resolved, unresolved = TokenResolver( Tokenizer(min_length=3, stem=False), corpus).resolve(scores) self.assertTrue('to' in unresolved)
def test_resolve_all(self): """ Test that when resolving candidates, all of them are returned. """ """ Create the test data """ tokenizer = Tokenizer(min_length=3, stem=False, case_fold=True) posts = [ "Manchester United falter against Tottenham Hotspur", "Manchester United unable to avoid defeat to Tottenham", "Tottenham lose again", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] """ Ensure that all candidates are resolved. """ candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) self.assertTrue(scores) resolved, unresolved = Resolver().resolve(scores) self.assertEqual(set(scores.keys()), set(resolved)) self.assertEqual([], unresolved)
def test_case_folding(self): """ Test that when case-folding is set, the case does not matter. In this test, the stem 'report' can be formed by: #. Reporters - appears twice #. reporters - appears twice #. reports - appears three times Without case-folding, 'reports' would be chosen to represent the token 'report'. 'reports' appears three times, and 'Reporters' and 'reporters' appear twice. With case-folding, 'reports' still appears three times, but 'reporters' appears four times. """ """ Create the test data """ tokenizer = Tokenizer(min_length=1, stem=True) posts = [ "Reporters Without Borders issue statement after reporters are harrassed", "Reporters left waiting all night long: reports", "Two reporters injured before gala: reports", "Queen reacts: reports of her falling ill exaggerated" ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] candidates = TokenExtractor().extract(corpus) scores = TFScorer().score(candidates) scores = ThresholdFilter(0).filter(scores) resolved, unresolved = TokenResolver(tokenizer, corpus, case_fold=False).resolve(scores) self.assertTrue('reports' in resolved) resolved, unresolved = TokenResolver(tokenizer, corpus, case_fold=True).resolve(scores) self.assertTrue('reporters' in resolved)
def test_repeated_tokens_with_custom_tokenizer(self): """ Test that when a custom tokenizer is given, repeated tokenizers appear multiple times. """ """ Create the test data. """ tokenizer = Tokenizer(stopwords=stopwords.words("english"), stem=False) posts = [ "Manchester United back to winning ways after defeating Manchester City.", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() candidates = extractor.extract(corpus) self.assertEqual(1, candidates[0].count('manchester')) extractor = TokenExtractor(tokenizer=tokenizer) candidates = extractor.extract(corpus) self.assertEqual(2, candidates[0].count('manchester'))
def test_extract_with_custom_tokenizer(self): """ Test that when a custom tokenizer is given, it is used instead of the dimensions. """ """ Create the test data, which uses stemming. """ tokenizer = Tokenizer(stopwords=stopwords.words("english"), stem=True) posts = [ "Manchester United back to winning ways", ] corpus = [Document(post, tokenizer.tokenize(post)) for post in posts] extractor = TokenExtractor() candidates = extractor.extract(corpus) self.assertEqual(set(["manchest", "unit", "back", "win", "way"]), set(candidates[0])) extractor = TokenExtractor(tokenizer=Tokenizer( stopwords=stopwords.words('english'), stem=False)) candidates = extractor.extract(corpus) self.assertEqual( set(["manchester", "united", "back", "winning", "ways"]), set(candidates[0]))