def test_issue4313(): """ This should not crash or exit with some strange error code """ beam_width = 16 beam_density = 0.0001 nlp = English() ner = EntityRecognizer(nlp.vocab) ner.add_label("SOME_LABEL") ner.begin_training([]) nlp.add_pipe(ner) # add a new label to the doc doc = nlp("What do you think about Apple ?") assert len(ner.labels) == 1 assert "SOME_LABEL" in ner.labels apple_ent = Span(doc, 5, 6, label="MY_ORG") doc.ents = list(doc.ents) + [apple_ent] # ensure the beam_parse still works with the new label docs = [doc] beams = nlp.entity.beam_parse(docs, beam_width=beam_width, beam_density=beam_density) for doc, beam in zip(docs, beams): entity_scores = defaultdict(float) for score, ents in nlp.entity.moves.get_beam_parses(beam): for start, end, label in ents: entity_scores[(start, end, label)] += score
def test_issue3345(): """Test case where preset entity crosses sentence boundary.""" nlp = English() doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) doc[4].is_sent_start = True ruler = EntityRuler(nlp, patterns=[{ "label": "GPE", "pattern": "New York" }]) config = { "learn_tokens": False, "min_action_freq": 30, "update_with_oracle_cut_size": 100, } cfg = {"model": DEFAULT_NER_MODEL} model = registry.resolve(cfg, validate=True)["model"] ner = EntityRecognizer(doc.vocab, model, **config) # Add the OUT action. I wouldn't have thought this would be necessary... ner.moves.add_action(5, "") ner.add_label("GPE") doc = ruler(doc) # Get into the state just before "New" state = ner.moves.init_batch([doc])[0] ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") # Check that B-GPE is valid. assert ner.moves.is_valid(state, "B-GPE")
def test_add_label_deserializes_correctly(): ner1 = EntityRecognizer(Vocab()) ner1.add_label("C") ner1.add_label("B") ner1.add_label("A") ner1.begin_training([]) ner2 = EntityRecognizer(Vocab()).from_bytes(ner1.to_bytes()) assert ner1.moves.n_moves == ner2.moves.n_moves for i in range(ner1.moves.n_moves): assert ner1.moves.get_class_name(i) == ner2.moves.get_class_name(i)
def test_add_label_deserializes_correctly(): ner1 = EntityRecognizer(Vocab()) ner1.add_label("C") ner1.add_label("B") ner1.add_label("A") ner1.begin_training([]) ner2 = EntityRecognizer(Vocab()).from_bytes(ner1.to_bytes()) assert ner1.moves.n_moves == ner2.moves.n_moves for i in range(ner1.moves.n_moves): assert ner1.moves.get_class_name(i) == ner2.moves.get_class_name(i)
def test_add_label_deserializes_correctly(): cfg = {"model": DEFAULT_NER_MODEL} model = registry.resolve(cfg, validate=True)["model"] ner1 = EntityRecognizer(Vocab(), model) ner1.add_label("C") ner1.add_label("B") ner1.add_label("A") ner1.initialize(lambda: [_ner_example(ner1)]) ner2 = EntityRecognizer(Vocab(), model) # the second model needs to be resized before we can call from_bytes ner2.model.attrs["resize_output"](ner2.model, ner1.moves.n_moves) ner2.from_bytes(ner1.to_bytes()) assert ner1.moves.n_moves == ner2.moves.n_moves for i in range(ner1.moves.n_moves): assert ner1.moves.get_class_name(i) == ner2.moves.get_class_name(i)
def test_accept_blocked_token(): """Test succesful blocking of tokens to be in an entity.""" # 1. test normal behaviour nlp1 = English() doc1 = nlp1("I live in New York") ner1 = EntityRecognizer(doc1.vocab) assert [token.ent_iob_ for token in doc1] == ["", "", "", "", ""] assert [token.ent_type_ for token in doc1] == ["", "", "", "", ""] # Add the OUT action ner1.moves.add_action(5, "") ner1.add_label("GPE") # Get into the state just before "New" state1 = ner1.moves.init_batch([doc1])[0] ner1.moves.apply_transition(state1, "O") ner1.moves.apply_transition(state1, "O") ner1.moves.apply_transition(state1, "O") # Check that B-GPE is valid. assert ner1.moves.is_valid(state1, "B-GPE") # 2. test blocking behaviour nlp2 = English() doc2 = nlp2("I live in New York") ner2 = EntityRecognizer(doc2.vocab) # set "New York" to a blocked entity doc2.ents = [(0, 3, 5)] assert [token.ent_iob_ for token in doc2] == ["", "", "", "B", "B"] assert [token.ent_type_ for token in doc2] == ["", "", "", "", ""] # Check that B-GPE is now invalid. ner2.moves.add_action(4, "") ner2.moves.add_action(5, "") ner2.add_label("GPE") state2 = ner2.moves.init_batch([doc2])[0] ner2.moves.apply_transition(state2, "O") ner2.moves.apply_transition(state2, "O") ner2.moves.apply_transition(state2, "O") # we can only use U- for "New" assert not ner2.moves.is_valid(state2, "B-GPE") assert ner2.moves.is_valid(state2, "U-") ner2.moves.apply_transition(state2, "U-") # we can only use U- for "York" assert not ner2.moves.is_valid(state2, "B-GPE") assert ner2.moves.is_valid(state2, "U-")
def test_issue3345(): """Test case where preset entity crosses sentence boundary.""" nlp = English() doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) doc[4].is_sent_start = True ruler = EntityRuler(nlp, patterns=[{"label": "GPE", "pattern": "New York"}]) ner = EntityRecognizer(doc.vocab) # Add the OUT action. I wouldn't have thought this would be necessary... ner.moves.add_action(5, "") ner.add_label("GPE") doc = ruler(doc) # Get into the state just before "New" state = ner.moves.init_batch([doc])[0] ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") # Check that B-GPE is valid. assert ner.moves.is_valid(state, "B-GPE")
def test_issue3345(): """Test case where preset entity crosses sentence boundary.""" nlp = English() doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) doc[4].is_sent_start = True ruler = EntityRuler(nlp, patterns=[{"label": "GPE", "pattern": "New York"}]) ner = EntityRecognizer(doc.vocab) # Add the OUT action. I wouldn't have thought this would be necessary... ner.moves.add_action(5, "") ner.add_label("GPE") doc = ruler(doc) # Get into the state just before "New" state = ner.moves.init_batch([doc])[0] ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") # Check that B-GPE is valid. assert ner.moves.is_valid(state, "B-GPE")
def test_overwrite_token(): nlp = English() ner1 = nlp.create_pipe("ner") nlp.add_pipe(ner1, name="ner") nlp.begin_training() # The untrained NER will predict O for each token doc = nlp("I live in New York") assert [token.ent_iob_ for token in doc] == ["O", "O", "O", "O", "O"] assert [token.ent_type_ for token in doc] == ["", "", "", "", ""] # Check that a new ner can overwrite O ner2 = EntityRecognizer(doc.vocab) ner2.moves.add_action(5, "") ner2.add_label("GPE") state = ner2.moves.init_batch([doc])[0] assert ner2.moves.is_valid(state, "B-GPE") assert ner2.moves.is_valid(state, "U-GPE") ner2.moves.apply_transition(state, "B-GPE") assert ner2.moves.is_valid(state, "I-GPE") assert ner2.moves.is_valid(state, "L-GPE")
def test_add_label_deserializes_correctly(): config = { "learn_tokens": False, "min_action_freq": 30, "update_with_oracle_cut_size": 100, } cfg = {"model": DEFAULT_NER_MODEL} model = registry.resolve(cfg, validate=True)["model"] ner1 = EntityRecognizer(Vocab(), model, **config) ner1.add_label("C") ner1.add_label("B") ner1.add_label("A") ner1.initialize(lambda: [_ner_example(ner1)]) ner2 = EntityRecognizer(Vocab(), model, **config) # the second model needs to be resized before we can call from_bytes ner2.model.attrs["resize_output"](ner2.model, ner1.moves.n_moves) ner2.from_bytes(ner1.to_bytes()) assert ner1.moves.n_moves == ner2.moves.n_moves for i in range(ner1.moves.n_moves): assert ner1.moves.get_class_name(i) == ner2.moves.get_class_name(i)
def test_issue3345(entity_ruler_factory): """Test case where preset entity crosses sentence boundary.""" nlp = English() doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) doc[4].is_sent_start = True ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns([{"label": "GPE", "pattern": "New York"}]) cfg = {"model": DEFAULT_NER_MODEL} model = registry.resolve(cfg, validate=True)["model"] ner = EntityRecognizer(doc.vocab, model) # Add the OUT action. I wouldn't have thought this would be necessary... ner.moves.add_action(5, "") ner.add_label("GPE") doc = ruler(doc) # Get into the state just before "New" state = ner.moves.init_batch([doc])[0] ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") ner.moves.apply_transition(state, "O") # Check that B-GPE is valid. assert ner.moves.is_valid(state, "B-GPE")
filtered_ents = [ent for ent in list(doc.ents) if ent.label_ in ENT_LIST] return doc, filtered_ents ################################################# nlp = spacy.load('en') matcher = Matcher(nlp.vocab) ner = EntityRecognizer(nlp.vocab) ents_to_add = ['IP', 'PHONE', 'ADDRESS', 'EMAIL', 'URL', 'ACCT'] for label in ents_to_add: ner.add_label(label) setup_patterns_v19() #use with Spacy v1.9 #setup_patterns_v20() #use with Spacy v2.0+ IP_ID = nlp.vocab.strings['IP'] PHONE_ID = nlp.vocab.strings['PHONE'] ADDRESS_ID = nlp.vocab.strings['ADDRESS'] EMAIL_ID = nlp.vocab.strings['EMAIL'] URL_ID = nlp.vocab.strings['URL'] ACCT_ID = nlp.vocab.strings['ACCT'] ENT_LIST = [ 'PERSON', 'NORP', 'ORG', 'GPE', 'PRODUCT', 'FAC', 'MONEY', 'LANGUAGE', 'DATE', 'TIME', 'EMAIL', 'URL', 'IP', 'ACCT', 'PHONE', 'ADDRESS' ]