class TestAbbreviationDetector(unittest.TestCase): def setUp(self): super().setUp() self.nlp = spacy.load("en_core_web_sm") self.detector = AbbreviationDetector(self.nlp) self.text = "Spinal and bulbar muscular atrophy (SBMA) is an \ inherited motor neuron disease caused by the expansion \ of a polyglutamine tract within the androgen receptor (AR). \ SBMA can be caused by this easily." def test_find_abbreviation(self): # Basic case doc = self.nlp("abbreviation (abbrn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "abbreviation" # Hypenation and numbers within abbreviation doc = self.nlp("abbreviation (ab-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "abbreviation" # No match doc = self.nlp("abbreviation (aeb-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form is None # First letter must match start of word. doc = self.nlp("aaaabbreviation (ab-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "aaaabbreviation" # Matching is greedy for first letter (are is not included). doc = self.nlp("more words are considered aaaabbreviation (ab-b9rn)") long = doc[0:5] short = doc[6:7] _, long_form = find_abbreviation(long, short) assert long_form.text == "aaaabbreviation" def test_filter_matches(self): doc = self.nlp(self.text) matches = self.detector.matcher(doc) matches_no_brackets = [(x[0], x[1] + 1, x[2] - 1) for x in matches] filtered = filter_matches(matches_no_brackets, doc) assert len(filtered) == 2 long, short = filtered[0] assert long.text_with_ws == "Spinal and bulbar muscular atrophy " assert short.text == "SBMA" long, short = filtered[1] assert long.text_with_ws == "within the androgen receptor " assert short.text == "AR" def test_abbreviation_detection(self): # Attribute should be registered. doc = self.nlp(self.text) assert doc._.abbreviations == [] doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 3 correct = set() span = doc[33:34] span._.long_form = doc[0:5] correct.add(span) span = doc[6:7] span._.long_form = doc[0:5] correct.add(span) span = doc[29:30] span._.long_form = doc[26:28] correct.add(span) correct_long = {x._.long_form for x in correct} assert set(doc2._.abbreviations) == correct assert {x._.long_form for x in doc2._.abbreviations} == correct_long def test_find(self): doc = self.nlp(self.text) long, shorts = self.detector.find(doc[6:7], doc) assert long.text_with_ws == "Spinal and bulbar muscular atrophy " assert len(shorts) == 2 assert {x.text_with_ws for x in shorts} == {"SBMA", "SBMA "} long, shorts = self.detector.find(doc[7:13], doc) assert shorts == set() def test_issue_158(self): text = ( "The PVO observations showed that the total transterminator flux " "was 23% of that at solar maximum and that the largest reductions in the " "number of ions transported antisunward occurred at the highest altitudes " "(Spenner et al., 1995)." ) doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 def test_issue_192(self): # test for <short> (<long>) pattern text = "blah SBMA (Spinal and bulbar muscular atrophy)" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 1 assert doc2._.abbreviations[0] == doc[1:2] assert doc2._.abbreviations[0]._.long_form == doc[3:8] def test_issue_161(self): # test some troublesome cases in the abbreviation detector text = "H2)]+(14)s.t. (1), (4).Similarly" print(f"Text: {text}") doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = ".(21)In (21), λ" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "map expX (·) : R" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "0,(3)with the following data: (3-i) (q̄" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "Φg(h),ThΦg(v) ) , (h, v)" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "dimension;(S-iii) The optimal control problem obtained in (S-ii) is con-verted" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "z), πut (z)) )" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "repositories he/she already worked with or from previous collaborators. Nevertheless, 88% of the first action of users to a repository (repository discovery) is" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 def test_empty_span(self): text = "(19, 9, 4) Hadamard Designs and Their Residual Designs" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 def test_space_issue(self): text = "by designing A Lite BERT (ALBERT) architecture that has significantly fewer parameters than a traditional BERT architecture." doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 1 assert doc2._.abbreviations[0]._.long_form.text == "A Lite BERT" def test_multiple_spaces(self): text = "by designing A Lite BERT (ALBERT) architecture that has significantly fewer parameters than a traditional BERT architecture." doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 1 assert doc2._.abbreviations[0]._.long_form.text == "A Lite BERT" @pytest.mark.xfail def test_difficult_cases(self): # Don't see an obvious way of solving these. They require something more semantic to distinguish text = "is equivalent to (iv) of Theorem" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0 text = "or to fork.Users work more on their repositories (owners) than on" doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0
class TestAbbreviationDetector(unittest.TestCase): def setUp(self): super().setUp() self.nlp = spacy.load("en_core_web_sm") self.detector = AbbreviationDetector(self.nlp) self.text = "Spinal and bulbar muscular atrophy (SBMA) is an \ inherited motor neuron disease caused by the expansion \ of a polyglutamine tract within the androgen receptor (AR). \ SBMA can be caused by this easily." def test_find_abbreviation(self): # Basic case doc = self.nlp("abbreviation (abbrn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "abbreviation" # Hypenation and numbers within abbreviation doc = self.nlp("abbreviation (ab-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "abbreviation" # No match doc = self.nlp("abbreviation (aeb-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form is None # First letter must match start of word. doc = self.nlp("aaaabbreviation (ab-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "aaaabbreviation" # Matching is greedy for first letter (are is not included). doc = self.nlp("more words are considered aaaabbreviation (ab-b9rn)") long = doc[0:5] short = doc[6:7] _, long_form = find_abbreviation(long, short) assert long_form.text == "aaaabbreviation" def test_filter_matches(self): doc = self.nlp(self.text) matches = self.detector.matcher(doc) matches_no_brackets = [(x[0], x[1] + 1, x[2] -1) for x in matches] filtered = filter_matches(matches_no_brackets, doc) assert len(filtered) == 2 long, short = filtered[0] assert long.string == "Spinal and bulbar muscular atrophy " assert short.string == "SBMA" long, short = filtered[1] assert long.string == "within the androgen receptor " assert short.string == "AR" def test_abbreviation_detection(self): # Attribute should be registered. doc = self.nlp(self.text) assert doc._.abbreviations == [] doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 3 correct = set() span = doc[33:34] span._.long_form = doc[0:5] correct.add(span) span = doc[6:7] span._.long_form = doc[0:5] correct.add(span) span = doc[29:30] span._.long_form = doc[26:28] correct.add(span) correct_long = {x._.long_form for x in correct} assert set(doc2._.abbreviations) == correct assert {x._.long_form for x in doc2._.abbreviations} == correct_long def test_find(self): doc = self.nlp(self.text) long, shorts = self.detector.find(doc[6:7], doc) assert long.string == "Spinal and bulbar muscular atrophy " assert len(shorts) == 2 assert {x.string for x in shorts} == {"SBMA", "SBMA "} long, shorts = self.detector.find(doc[7:13], doc) assert shorts == set() def test_issue_158(self): text = "The PVO observations showed that the total transterminator flux "\ "was 23% of that at solar maximum and that the largest reductions in the "\ "number of ions transported antisunward occurred at the highest altitudes "\ "(Spenner et al., 1995)." doc = self.nlp(text) doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 0
class TestAbbreviationDetector(unittest.TestCase): def setUp(self): super().setUp() self.nlp = spacy.load("en_core_web_sm") self.detector = AbbreviationDetector(self.nlp) self.text = "Spinal and bulbar muscular atrophy (SBMA) is an \ inherited motor neuron disease caused by the expansion \ of a polyglutamine tract within the androgen receptor (AR). \ SBMA can be caused by this easily." def test_find_abbreviation(self): # Basic case doc = self.nlp("abbreviation (abbrn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "abbreviation" # Hypenation and numbers within abbreviation doc = self.nlp("abbreviation (ab-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "abbreviation" # No match doc = self.nlp("abbreviation (aeb-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form is None # First letter must match start of word. doc = self.nlp("aaaabbreviation (ab-b9rn)") long = doc[0:1] short = doc[2:3] _, long_form = find_abbreviation(long, short) assert long_form.text == "aaaabbreviation" # Matching is greedy for first letter (are is not included). doc = self.nlp("more words are considered aaaabbreviation (ab-b9rn)") long = doc[0:5] short = doc[6:7] _, long_form = find_abbreviation(long, short) assert long_form.text == "aaaabbreviation" def test_filter_matches(self): doc = self.nlp(self.text) matches = self.detector.matcher(doc) matches_no_brackets = [(x[0], x[1] + 1, x[2] - 1) for x in matches] filtered = filter_matches(matches_no_brackets, doc) assert len(filtered) == 2 long, short = filtered[0] assert long.string == "Spinal and bulbar muscular atrophy " assert short.string == "SBMA" long, short = filtered[1] assert long.string == "within the androgen receptor " assert short.string == "AR" def test_abbreviation_detection(self): # Attribute should be registered. doc = self.nlp(self.text) assert doc._.abbreviations == [] doc2 = self.detector(doc) assert len(doc2._.abbreviations) == 2 long, shorts = doc2._.abbreviations[0] assert long.string == "Spinal and bulbar muscular atrophy " assert len(shorts) == 2 assert {x.string for x in shorts} == {"SBMA", "SBMA "} long, shorts = doc2._.abbreviations[1] assert long.string == "androgen receptor " assert len(shorts) == 1 assert shorts.pop().string == "AR" def test_find(self): doc = self.nlp(self.text) long, shorts = self.detector.find(doc[6:7], doc) assert long.string == "Spinal and bulbar muscular atrophy " assert len(shorts) == 2 assert {x.string for x in shorts} == {"SBMA", "SBMA "} long, shorts = self.detector.find(doc[7:13], doc) assert shorts == set()