Python RegexFilter Examples

Example #1

 def test_filters_punctuation_correctly(self):
     filter_ = RegexFilter(patterns=[r"\(|\)|\"|\."])
     expected_tokens = [
         "this", "45", "sentence", "has", "9943434", "punctuations"
     ]
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens

Example #2

 def test_filters_digits_correctly(self):
     filter_ = RegexFilter(patterns=[r"\d+"])
     expected_tokens = [
         "this", "(", "sentence", ")", "has", '"', "punctuations", '"', "."
     ]
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens

Example #3

File: word_filter_test.py Project: wgc20/GrailQA

 def test_filters_no_match_correctly(self):
     filter_ = RegexFilter(patterns=[r'&'])
     expected_tokens = [
         "this", "45", "(", "sentence", ")", "has", "9943434", '"',
         "punctuations", '"', '.'
     ]
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens

Example #4

File: word_filter_test.py Project: wjn922/allennlp

 def test_filters_multiple_patterns_correctly(self):
     filter_ = RegexFilter(patterns=[r'\(|\)|\"|\.', r'[\d+]'])
     expected_tokens = ["this", "sentence", "has", "punctuations"]
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens

Example #5

File: word_filter_test.py Project: apmoore1/allennlp

 def test_filters_no_match_correctly(self):
     filter_ = RegexFilter(patterns=[r'&'])
     expected_tokens = ["this", "45", "(", "sentence", ")", "has", "9943434", '"', "punctuations", '"', '.']
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens

Example #6

File: word_filter_test.py Project: apmoore1/allennlp

 def test_filters_multiple_patterns_correctly(self):
     filter_ = RegexFilter(patterns=[r'\(|\)|\"|\.', r'[\d+]'])
     expected_tokens = ["this", "sentence", "has", "punctuations"]
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens

Example #7

File: word_filter_test.py Project: apmoore1/allennlp

 def test_filters_punctuation_correctly(self):
     filter_ = RegexFilter(patterns=[r'\(|\)|\"|\.'])
     expected_tokens = ["this", "45", "sentence", "has", "9943434", "punctuations"]
     tokens = [t.text for t in filter_.filter_words(self.sentence)]
     assert tokens == expected_tokens