person_string = nlp.vocab.strings[person_hash]
print(person_string)
########## as you might observe nlp object is universal (not dependent on text), its the doc object that carries
# info about our particular use case
# so, hashes generated are universal, here even if we searched for hash of a very common english word "ball" it
# returns a value even though its not in doc text
# any new words encountered will be added to dictionary when we try to lookup for their hash
#### creating Doc and Span objects from built in classes
from spacy.tokens import Doc, Span
# suppose the text is same as in previous example
words = ["David", "Bowie", "is", 'a', 'PERSON', '!']
spaces_after_all_tokens = [True, True, True, True, False, False]
doc = Doc(nlp.vocab, words, spaces_after_all_tokens)
print(doc.text)
span = Span(doc, 0, 2, label="PERSON")
print(span.text, span.label_)
type(doc.ents)
doc.ents = [span]
# Print entities' text and labels
print([(ent.text, ent.label_) for ent in doc.ents])
############# Spacy's word vectors (300d) and similarities ##############
# you'll need large or medium models for this
# Load the en_core_web_md model
nlp = spacy.load('en_core_web_md')
def test_issue912(en_vocab, text, tag, lemma):
"""Test base-forms are preserved."""
doc = Doc(en_vocab, words=[text])
doc[0].tag_ = tag
assert doc[0].lemma_ == lemma
def main(json_loc: Path,
train_file: Path,
dev_file: Path,
test_file: Path,
test_split=0.189,
train_split=0.709):
"""Creating the corpus from the Prodigy annotations."""
Doc.set_extension("rel", default={})
vocab = Vocab()
docs = {"train": [], "dev": [], "test": []}
ids = {"train": set(), "dev": set(), "test": set()}
count_all = {"train": 0, "dev": 0, "test": 0}
count_pos = {"train": 0, "dev": 0, "test": 0}
long_rel_count = 0 #how many relations are longer
error_count_rel = 0 #how often is something different than ARGO, ARG1, ARG
with json_loc.open("r", encoding="utf8") as jsonfile:
length_training_data = len([
True for line in jsonfile if json.loads(line)["answer"] == "accept"
])
msg.info(f"Number of accepted recipes: {length_training_data}")
with json_loc.open("r", encoding="utf8") as jsonfile:
for line in jsonfile:
example = json.loads(line) #one recipe
span_starts = set()
if example["answer"] == "accept":
neg = 0
pos = 0
try:
# Parse the tokens -> example["tokens"] = list of dicts
words = [t["text"] for t in example["tokens"]
] #list containing all words
spaces = [
t["ws"] for t in example["tokens"]
] #list containing ws is behind word (ws = True/False)
doc = Doc(vocab, words=words, spaces=spaces)
# Parse the entities
spans = example[
"spans"] #list of dicts containing entities
entities = []
span_end_to_start = {}
ents_dict = {}
for span in spans: #every detected span
entity = doc.char_span(
span["start"], span["end"], label=span["label"]
) #"start" = wievielter character ist start character des spans im doc
span_end_to_start[span["token_end"]] = span[
"token_start"] #end_token of span as key for start_token (start token = wievielter token in doc)
entities.append(entity) #appended to list
span_starts.add(span["token_start"]) #added to set
ents_dict[span["token_start"]] = (span["label"],
span["token_start"])
doc.ents = entities #entity list assigned as doc entites
# Parse the relations
rels = {}
# create token combinations
for x1 in span_starts:
#VERBS_TO_OTHER 1a
if VERBS_TO_OTHER == True:
if ents_dict[x1][0] == "V": #filter entity type
for x2 in span_starts:
if ents_dict[x2][0] in [
"Z", "TOOL", "ATTR", "TEMP",
"DAUER", "ZEITP", "PRÄP"
]: #filter entity type
#DIFF_FRONT_BACK 1a
if DIFF_FRONT_BACK == True:
if ((x1 - x2) >= 0 and
(x1 - x2) <= BACK) or (
(x1 - x2) < 0 and
(x1 - x2) >= FRONT * -1):
rels[(x1, x2)] = {}
else:
pass
#DIFF_FRONT_BACK 1b
else:
if abs(
ents_dict[x1][1] -
ents_dict[x2][1]
) <= TOKEN_LENGTH: #filter token distance (match with config?)
rels[(x1, x2)] = {
} #every possible span combination becomes key for individual dict (1,1), (1,2) ...
#VERBS_TO_OTHER 1b
else:
for x2 in span_starts:
#DIFF_FRONT_BACK 2a
if DIFF_FRONT_BACK == True:
if ((x1 - x2) >= 0 and
(x1 - x2) <= BACK) or (
(x1 - x2) < 0 and
(x1 - x2) >= FRONT * -1):
rels[(x1, x2)] = {}
else:
pass
#DIFF_FRONT_BACK 2b
else:
if abs(
ents_dict[x1][1] - ents_dict[x2][1]
) <= TOKEN_LENGTH: #filter token distance (match with config?)
rels[(x1, x2)] = {
} #every possible span combination becomes key for individual dict (1,1), (1,2) ...
relations = example[
"relations"] #relations is list of dict
for relation in relations:
# the 'head' and 'child' annotations refer to the end token in the span
# but we want the first token
start = span_end_to_start[relation[
"head"]] #wievielter token ist start token des head
end = span_end_to_start[relation[
"child"]] #wievielter token ist start token des child
label = relation["label"]
#DETAILED_ARGS 1a
if DETAILED_ARGS == True:
if label == "ARG0":
if ents_dict[end][0] not in ["Z", "TOOL"]:
label = MAP_LABELS_ARG[ents_dict[end][0]]
else:
label = MAP_LABELS_ARG0[ents_dict[end][
0]] #assign new label based on span type
elif label == "ARG1":
if ents_dict[end][0] not in ["Z", "TOOL"]:
label = MAP_LABELS_ARG[ents_dict[end][0]]
else:
label = MAP_LABELS_ARG1[ents_dict[end][0]]
elif label == "ARG":
if ents_dict[end][0] in ["Z", "TOOL"]:
if ents_dict[end][0] == "Z":
label = "Arg0Z"
elif ents_dict[end][0] == "TOOL":
label = "Arg1Tool"
else:
label = MAP_LABELS_ARG[ents_dict[end][0]]
else:
error_count_rel += 1
#DETAILED_ARGS 1b
else:
label = MAP_LABELS_STANDARD[
label] #MAP_LABELS = dict containing label as key
# Positive relations are being added
try:
if label not in rels[(
start, end
)]: #check if label already exists for token combination
rels[(
start, end
)][label] = 1.0 #initialize label as new key with value 1.0
pos += 1 #positive case
except:
long_rel_count += 1 #error only if relation exists in annotation but isn't a valid token combi (too long/not starting from verb)
pass
# The annotation is complete, so fill in zero's where the data is missing
for x1 in span_starts:
#VERBS_TO_OTHER 2a
if VERBS_TO_OTHER == True:
if ents_dict[x1][0] == "V": #filter entity type
for x2 in span_starts:
if ents_dict[x2][0] in [
"Z", "TOOL", "ATTR", "TEMP",
"DAUER", "ZEITP", "PRÄP"
]: #filter entity type
#DIFF_FRONT_BACK 2a
if DIFF_FRONT_BACK == True:
if ((x1 - x2) >= 0 and
(x1 - x2) <= BACK) or (
(x1 - x2) < 0 and
(x1 - x2) >= FRONT * -1):
#DETAILED_ARGS 2a
if DETAILED_ARGS == True:
merged_labels = list(
MAP_LABELS_ARG0.values(
)) + list(
MAP_LABELS_ARG1.
values()) + list(
MAP_LABELS_ARG.
values())
for label in merged_labels:
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(
x1, x2
)][label] = 0.0
#DETAILED_ARGS 2b
else:
for label in MAP_LABELS_STANDARD.values(
): #for every label
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(
x1, x2
)][label] = 0.0
#DIFF_FRONT_BACK 2b
else:
if abs(
ents_dict[x1][1] -
ents_dict[x2][1]
) <= TOKEN_LENGTH: #filter token distance (match with config?)
#DETAILED_ARGS 3a
if DETAILED_ARGS == True:
merged_labels = list(
MAP_LABELS_ARG0.values(
)) + list(
MAP_LABELS_ARG1.
values()) + list(
MAP_LABELS_ARG.
values())
for label in merged_labels:
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(
x1, x2
)][label] = 0.0
#DETAILED_ARGS 3b
else:
for label in MAP_LABELS_STANDARD.values(
): #for every label
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(
x1, x2
)][label] = 0.0 #span combination with label as key gets 0 as value
#VERBS_TO_OTHER 2b
else:
for x2 in span_starts:
#DIFF_FRONT_BACK 3a
if DIFF_FRONT_BACK == True:
if ((x1 - x2) >= 0 and
(x1 - x2) <= BACK) or (
(x1 - x2) < 0 and
(x1 - x2) >= FRONT * -1):
#DETAILED_ARGS 4a
if DETAILED_ARGS == True:
merged_labels = list(
MAP_LABELS_ARG0.values()
) + list(MAP_LABELS_ARG1.values(
)) + list(MAP_LABELS_ARG.values())
for label in merged_labels:
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(x1, x2)][label] = 0.0
#DETAILED_ARGS 4b
else:
for label in MAP_LABELS_STANDARD.values(
): #for every label
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(x1, x2)][label] = 0.0
#DIFF_FRONT_BACK 3b
else:
if abs(
ents_dict[x1][1] - ents_dict[x2][1]
) <= TOKEN_LENGTH: #filter token distance (match with config?)
#DETAILED_ARGS 5a
if DETAILED_ARGS == True:
merged_labels = list(
MAP_LABELS_ARG0.values()
) + list(MAP_LABELS_ARG1.values(
)) + list(MAP_LABELS_ARG.values())
for label in merged_labels:
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(x1, x2)][label] = 0.0
#DETAILED_ARGS 5b
else:
for label in MAP_LABELS_STANDARD.values(
): #for every label
if label not in rels[(
x1, x2
)]: #if label isn't assigned to span combination
neg += 1
rels[(x1, x2)][label] = 0.0
#print(rels)
doc._.rel = rels # rels = {(1,1): {Arg0 : 1, Arg1 : 0, Arg : 0}, (1,2): {Arg0 : 0, ...}}
# only keeping documents with at least 1 positive case (if doc isn't annotated relations = empty list)
if pos > 0:
recipe_id = example["_input_hash"]
if len(docs["train"]) < round(
train_split * length_training_data):
ids["train"].add(recipe_id)
docs["train"].append(doc)
count_pos["train"] += pos
count_all["train"] += pos + neg
elif len(docs["test"]) < round(
test_split * length_training_data):
ids["test"].add(recipe_id)
docs["test"].append(doc)
count_pos["test"] += pos
count_all["test"] += pos + neg
else:
ids["dev"].add(recipe_id)
docs["dev"].append(doc)
count_pos["dev"] += pos
count_all["dev"] += pos + neg
except KeyError as e:
msg.fail(
f"Skipping doc because of key error: {e} in {example['_input_hash']}"
)
msg.info(
f"{long_rel_count} relations have been cut because tokens are too far apart."
)
docbin = DocBin(docs=docs["train"], store_user_data=True)
docbin.to_disk(train_file)
msg.info(
f"{len(docs['train'])} training recipes from {len(ids['train'])} unique recipes, "
f"{count_pos['train']}/{count_all['train']} pos instances.")
docbin = DocBin(docs=docs["dev"], store_user_data=True)
docbin.to_disk(dev_file)
msg.info(
f"{len(docs['dev'])} dev recipes from {len(ids['dev'])} unique recipes, "
f"{count_pos['dev']}/{count_all['dev']} pos instances.")
docbin = DocBin(docs=docs["test"], store_user_data=True)
docbin.to_disk(test_file)
msg.info(
f"{len(docs['test'])} test recipes from {len(ids['test'])} unique recipes, "
f"{count_pos['test']}/{count_all['test']} pos instances.")
def __call__(self, text):
words = text.split(" ")
return Doc(self.vocab, words=words)
def test_issue600():
vocab = Vocab(tag_map={"NN": {"pos": "NOUN"}})
doc = Doc(vocab, words=["hello"])
doc[0].tag_ = "NN"
def test_matcher_match_end(matcher):
words = ["I", "like", "java"]
doc = Doc(matcher.vocab, words=words)
assert matcher(doc) == [(doc.vocab.strings["Java"], 2, 3)]
def test_matcher_valid_callback(en_vocab):
"""Test that on_match can only be None or callable."""
matcher = Matcher(en_vocab)
with pytest.raises(ValueError):
matcher.add("TEST", [[{"TEXT": "test"}]], on_match=[])
matcher(Doc(en_vocab, words=["test"]))
def test_vectors_lexeme_doc_similarity(vocab, text):
doc = Doc(vocab, words=text)
lex = vocab[text[0]]
assert lex.similarity(doc) == doc.similarity(lex)
assert -1.0 < lex.similarity(doc) < 1.0
def test_vectors_span_doc_similarity(vocab, text):
doc = Doc(vocab, words=text)
with pytest.warns(UserWarning):
assert doc[0:2].similarity(doc) == doc.similarity(doc[0:2])
assert -1.0 < doc[0:2].similarity(doc) < 1.0
def test_vectors_token_doc_similarity(vocab, text):
doc = Doc(vocab, words=text)
assert doc[0].similarity(doc) == doc.similarity(doc[0])
assert -1.0 < doc[0].similarity(doc) < 1.0
def test_vectors_lexeme_span_similarity(vocab, text):
doc = Doc(vocab, words=text)
lex = vocab[text[0]]
assert lex.similarity(doc[1:3]) == doc[1:3].similarity(lex)
assert -1.0 < doc.similarity(doc[1:3]) < 1.0
def test_vectors_span_vector(vocab, text):
span = Doc(vocab, words=text)[0:2]
assert list(span.vector)
assert span.vector_norm
def test_vectors_doc_vector(vocab, text):
doc = Doc(vocab, words=text)
assert list(doc.vector)
assert doc.vector_norm
def leer_texto(texto):
"""Funcion auxiliar para leer un archivo de texto"""
with open(texto, 'r', encoding="latin-1") as text:
return text.read()
#creamos nuestro <procesador> con la funcion spacy.load y pasamos el nombre de nuestro corpus como parametro, !!!!procesador es un metodo!!!!!
#procesador = spacy.load('es_core_news_lg')
#usamos el metodo leer_texto(), y asignamos el nombre del archivo que deseamos procesar,
#el string resultante lo guardamos en la variable <texto>
#texto = leer_texto("Sirena.txt")
#usamos nuestro procesador y le asignamos el texto
texto_procesado = Doc(Vocab()).from_disk("texto_procesado")
#texto_procesado.to_disk("texto_procesado")
##procesador.to_disk("procesador")
#Ahora tenemos un objeto llamado texto_procesado, el cual le ha asignado a cada palabra del texto una [tokenizacion]
#Podemos valernos de una palabra aleatoria incluida en el tipo [tag] de cada token.
#usamos la siguiente formula para crear el metodo correspondiente:
"""
def NOMBRE_METODO():
return random.choice([token.orth_ for token in texto_procesado if token.tag_ == 'TIPO DE TAG'])
"""
#NOMBRE_METODO = al nombre que queramos asignar para la palabra
#TIPO DE TAG = Un tag que describa el tipo de palabra
# la lista: de tipos :
def test_retokenize_disallow_zero_length(en_vocab):
doc = Doc(en_vocab, words=["hello", "world", "!"])
with pytest.raises(ValueError):
with doc.retokenize() as retokenizer:
retokenizer.merge(doc[1:1])
def test_vectors_doc_doc_similarity(vocab, text1, text2):
doc1 = Doc(vocab, words=text1)
doc2 = Doc(vocab, words=text2)
assert doc1.similarity(doc2) == doc2.similarity(doc1)
assert -1.0 < doc1.similarity(doc2) < 1.0
def read_data(nlp,
conllu_file,
text_file,
raw_text=True,
oracle_segments=False,
max_doc_length=None,
limit=None):
'''Read the CONLLU format into (Doc, GoldParse) tuples. If raw_text=True,
include Doc objects created using nlp.make_doc and then aligned against
the gold-standard sequences. If oracle_segments=True, include Doc objects
created from the gold-standard segments. At least one must be True.'''
if not raw_text and not oracle_segments:
raise ValueError(
"At least one of raw_text or oracle_segments must be True")
paragraphs = split_text(text_file.read())
conllu = read_conllu(conllu_file)
# sd is spacy doc; cd is conllu doc
# cs is conllu sent, ct is conllu token
docs = []
golds = []
for doc_id, (text, cd) in enumerate(zip(paragraphs, conllu)):
sent_annots = []
for cs in cd:
sent = defaultdict(list)
for id_, word, lemma, pos, tag, morph, head, dep, _, space_after in cs:
if '.' in id_:
continue
if '-' in id_:
continue
id_ = int(id_) - 1
head = int(head) - 1 if head != '0' else id_
sent['words'].append(word)
sent['tags'].append(tag)
sent['heads'].append(head)
sent['deps'].append('ROOT' if dep == 'root' else dep)
sent['spaces'].append(space_after == '_')
sent['entities'] = ['-'] * len(sent['words'])
sent['heads'], sent['deps'] = projectivize(sent['heads'],
sent['deps'])
if oracle_segments:
docs.append(
Doc(nlp.vocab, words=sent['words'], spaces=sent['spaces']))
golds.append(GoldParse(docs[-1], **sent))
sent_annots.append(sent)
if raw_text and max_doc_length and len(
sent_annots) >= max_doc_length:
doc, gold = _make_gold(nlp, None, sent_annots)
sent_annots = []
docs.append(doc)
golds.append(gold)
if limit and len(docs) >= limit:
return docs, golds
if raw_text and sent_annots:
doc, gold = _make_gold(nlp, None, sent_annots)
docs.append(doc)
golds.append(gold)
if limit and len(docs) >= limit:
return docs, golds
return docs, golds
def test_vectors_similarity_DD(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors
doc1 = Doc(vocab, words=[word1, word2])
doc2 = Doc(vocab, words=[word2, word1])
assert isinstance(doc1.similarity(doc2), float)
assert doc1.similarity(doc2) == doc2.similarity(doc1)
def test_matcher_match_middle(matcher):
words = ["I", "like", "Google", "Now", "best"]
doc = Doc(matcher.vocab, words=words)
assert matcher(doc) == [(doc.vocab.strings["GoogleNow"], 2, 4)]
def test_vectors_similarity_DS(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors
doc = Doc(vocab, words=[word1, word2])
assert isinstance(doc.similarity(doc[:2]), float)
assert doc.similarity(doc[:2]) == doc[:2].similarity(doc)
def test_matcher_no_match(matcher):
doc = Doc(matcher.vocab, words=["I", "like", "cheese", "."])
assert matcher(doc) == []
if not os.path.isdir(path_to_mlgenome):
print(f"Create Directory {path_to_mlgenome}")
os.mkdir(path_to_mlgenome)
print("Loading Vocab...")
vocab = Vocab().from_disk(os.path.join(path_to_annotations, "spacy.vocab"))
infoDF = pd.read_pickle(os.path.join(path_to_annotations, 'info_db.pandas'))
acronym_dictionary = dict()
entities = set()
lt = LoopTimer(update_after=1, avg_length=1000, target=len(infoDF))
for abstract_id, row in infoDF.iterrows():
file_path = os.path.join(path_to_annotations, f"{abstract_id}.spacy")
doc = Doc(vocab).from_disk(file_path)
for sentence in doc.sents:
for ent in sentence.ents:
entities.add(ent.text.lower())
definition_span = find_definition_candidate(sentence, ent)
if definition_span is not None:
acronym_string = ent.text.lower()
acronym_orth = [token.orth_ for token in ent]
d_string = definition_span.text.lower()
d_orth = [token.orth_.lower() for token in definition_span]
d_orth_with_ws = [token.text_with_ws.lower() for token in definition_span]
d_lemma = [token.lemma_.lower() for token in definition_span]
d_lemma_with_ws = [f"{token.lemma_.lower()}{token.whitespace_}" for token in definition_span]
def test_issue589():
vocab = Vocab()
vocab.strings.set_frozen(True)
doc = Doc(vocab, words=["whata"])
assert doc
def get_spans(word_seqs):
vocab = Vocab()
docs = [Doc(vocab, words=words) for words in word_seqs]
return [doc[:] for doc in docs]
def test_issue743():
doc = Doc(Vocab(), ["hello", "world"])
token = doc[0]
s = set([token])
items = list(s)
assert items[0] is token
def doc(vocab):
return Doc(vocab, words=["Casey", "went", "to", "New", "York", "."])
from spacy.tokens import Doc, Span
from spacy.lang.en import English
nlp = English()
# Create doc
doc = Doc(nlp.vocab,
words=["I", "like", "David", "Bowie"],
spaces=[True, True, True, False])
print(doc)
# create span for David Bowie
span = Span(doc, 2, 4, label=nlp.vocab.strings["PERSON"])
# add span to doc entities
doc.ents = [span]
# print entities' text and labels
print([(ent.text, ent.label_) for ent in doc.ents])
def test_doc_retokenize_spans_entity_merge_iob(en_vocab):
# Test entity IOB stays consistent after merging
words = ["a", "b", "c", "d", "e"]
doc = Doc(Vocab(), words=words)
doc.ents = [
(doc.vocab.strings.add("ent-abc"), 0, 3),
(doc.vocab.strings.add("ent-d"), 3, 4),
]
assert doc[0].ent_iob_ == "B"
assert doc[1].ent_iob_ == "I"
assert doc[2].ent_iob_ == "I"
assert doc[3].ent_iob_ == "B"
with doc.retokenize() as retokenizer:
retokenizer.merge(doc[0:2])
assert len(doc) == len(words) - 1
assert doc[0].ent_iob_ == "B"
assert doc[1].ent_iob_ == "I"
# Test that IOB stays consistent with provided IOB
words = ["a", "b", "c", "d", "e"]
doc = Doc(Vocab(), words=words)
with doc.retokenize() as retokenizer:
attrs = {"ent_type": "ent-abc", "ent_iob": 1}
retokenizer.merge(doc[0:3], attrs=attrs)
retokenizer.merge(doc[3:5], attrs=attrs)
assert doc[0].ent_iob_ == "B"
assert doc[1].ent_iob_ == "I"
# if no parse/heads, the first word in the span is the root and provides
# default values
words = ["a", "b", "c", "d", "e", "f", "g", "h", "i"]
doc = Doc(Vocab(), words=words)
doc.ents = [
(doc.vocab.strings.add("ent-de"), 3, 5),
(doc.vocab.strings.add("ent-fg"), 5, 7),
]
assert doc[3].ent_iob_ == "B"
assert doc[4].ent_iob_ == "I"
assert doc[5].ent_iob_ == "B"
assert doc[6].ent_iob_ == "I"
with doc.retokenize() as retokenizer:
retokenizer.merge(doc[2:4])
retokenizer.merge(doc[4:6])
retokenizer.merge(doc[7:9])
assert len(doc) == 6
assert doc[3].ent_iob_ == "B"
assert doc[3].ent_type_ == "ent-de"
assert doc[4].ent_iob_ == "B"
assert doc[4].ent_type_ == "ent-fg"
# if there is a parse, span.root provides default values
words = ["a", "b", "c", "d", "e", "f", "g", "h", "i"]
heads = [0, 0, 3, 0, 0, 0, 5, 0, 0]
ents = ["O"] * len(words)
ents[3] = "B-ent-de"
ents[4] = "I-ent-de"
ents[5] = "B-ent-fg"
ents[6] = "I-ent-fg"
deps = ["dep"] * len(words)
en_vocab.strings.add("ent-de")
en_vocab.strings.add("ent-fg")
en_vocab.strings.add("dep")
doc = Doc(en_vocab, words=words, heads=heads, deps=deps, ents=ents)
assert doc[2:4].root == doc[3] # root of 'c d' is d
assert doc[4:6].root == doc[4] # root is 'e f' is e
with doc.retokenize() as retokenizer:
retokenizer.merge(doc[2:4])
retokenizer.merge(doc[4:6])
retokenizer.merge(doc[7:9])
assert len(doc) == 6
assert doc[2].ent_iob_ == "B"
assert doc[2].ent_type_ == "ent-de"
assert doc[3].ent_iob_ == "I"
assert doc[3].ent_type_ == "ent-de"
assert doc[4].ent_iob_ == "B"
assert doc[4].ent_type_ == "ent-fg"
# check that B is preserved if span[start] is B
words = ["a", "b", "c", "d", "e", "f", "g", "h", "i"]
heads = [0, 0, 3, 4, 0, 0, 5, 0, 0]
ents = ["O"] * len(words)
ents[3] = "B-ent-de"
ents[4] = "I-ent-de"
ents[5] = "B-ent-de"
ents[6] = "I-ent-de"
deps = ["dep"] * len(words)
doc = Doc(en_vocab, words=words, heads=heads, deps=deps, ents=ents)
with doc.retokenize() as retokenizer:
retokenizer.merge(doc[3:5])
retokenizer.merge(doc[5:7])
assert len(doc) == 7
assert doc[3].ent_iob_ == "B"
assert doc[3].ent_type_ == "ent-de"
assert doc[4].ent_iob_ == "B"
assert doc[4].ent_type_ == "ent-de"
def test_matcher_match_start(matcher):
doc = Doc(matcher.vocab, words=["JavaScript", "is", "good"])
assert matcher(doc) == [(matcher.vocab.strings["JS"], 0, 1)]
def __call__(self, text):
words = text.split()
# All tokens 'own' a subsequent space character in this tokenizer
spaces = [True] * len(words)
return Doc(self.vocab, words=words, spaces=spaces)