def convert_file( input_path: Path = typer.Argument(..., exists=True, dir_okay=False), output_path: Path = typer.Argument(..., dir_okay=False), ): nlp = spacy.blank("en") doc_bin = DocBin(attrs=["ENT_IOB", "ENT_TYPE"]) header = True with open(input_path, "r") as in_f, open(output_path, "w") as out_f: for line in tqdm(in_f): if header: header = False continue sentence, tokens = pd.read_csv(StringIO(line), header=None, usecols=[0, 1]).values[0] tokens = eval(tokens) dict_line = line_to_dict(sentence, tokens) eg = dict_line if eg["answer"] != "accept": continue tokens = [token["text"] for token in eg["tokens"]] words, spaces = get_words_and_spaces(tokens, eg["text"]) doc = Doc(nlp.vocab, words=words, spaces=spaces) doc.ents = [ doc.char_span(s["start"], s["end"], label=s["label"]) for s in eg.get("spans", []) ] doc_bin.add(doc) doc_bin.to_disk(output_path) print(f"Processed {len(doc_bin)} documents: {output_path}")
def test_create_from_words_and_text(vocab): # no whitespace in words words = ["'", "dogs", "'", "run"] text = " 'dogs'\n\nrun " (words, spaces) = util.get_words_and_spaces(words, text) doc = Doc(vocab, words=words, spaces=spaces) assert [t.text for t in doc] == [" ", "'", "dogs", "'", "\n\n", "run", " "] assert [t.whitespace_ for t in doc] == ["", "", "", "", "", " ", ""] assert doc.text == text assert [t.text for t in doc if not t.text.isspace()] == [ word for word in words if not word.isspace() ] # partial whitespace in words words = [" ", "'", "dogs", "'", "\n\n", "run", " "] text = " 'dogs'\n\nrun " (words, spaces) = util.get_words_and_spaces(words, text) doc = Doc(vocab, words=words, spaces=spaces) assert [t.text for t in doc] == [" ", "'", "dogs", "'", "\n\n", "run", " "] assert [t.whitespace_ for t in doc] == ["", "", "", "", "", " ", ""] assert doc.text == text assert [t.text for t in doc if not t.text.isspace()] == [ word for word in words if not word.isspace() ] # non-standard whitespace tokens words = [" ", " ", "'", "dogs", "'", "\n\n", "run"] text = " 'dogs'\n\nrun " (words, spaces) = util.get_words_and_spaces(words, text) doc = Doc(vocab, words=words, spaces=spaces) assert [t.text for t in doc] == [" ", "'", "dogs", "'", "\n\n", "run", " "] assert [t.whitespace_ for t in doc] == ["", "", "", "", "", " ", ""] assert doc.text == text assert [t.text for t in doc if not t.text.isspace()] == [ word for word in words if not word.isspace() ] # mismatch between words and text with pytest.raises(ValueError): words = [" ", " ", "'", "dogs", "'", "\n\n", "run"] text = " 'dogs'\n\nrun " (words, spaces) = util.get_words_and_spaces(words + ["away"], text)
def test_gold_biluo_additional_whitespace(en_vocab, en_tokenizer): # additional whitespace tokens in GoldParse words words, spaces = get_words_and_spaces( ["I", "flew", "to", "San Francisco", "Valley", "."], "I flew to San Francisco Valley.", ) doc = Doc(en_vocab, words=words, spaces=spaces) prefix = "I flew to " entities = [(len(prefix), len(prefix + "San Francisco Valley"), "LOC")] gold_words = ["I", "flew", " ", "to", "San Francisco Valley", "."] gold_spaces = [True, True, False, True, False, False] example = Example.from_dict( doc, {"words": gold_words, "spaces": gold_spaces, "entities": entities} ) ner_tags = example.get_aligned_ner() assert ner_tags == ["O", "O", "O", "O", "B-LOC", "L-LOC", "O"]
def convert_file( input_path: Path = typer.Argument(..., exists=True, dir_okay=False), output_path: Path = typer.Argument(..., dir_okay=False), ): nlp = spacy.blank("en") doc_bin = DocBin(attrs=["ENT_IOB", "ENT_TYPE"]) for eg in tqdm(srsly.read_jsonl(input_path)): if eg["answer"] != "accept": continue tokens = [token["text"] for token in eg["tokens"]] words, spaces = get_words_and_spaces(tokens, eg["text"]) doc = Doc(nlp.vocab, words=words, spaces=spaces) doc.ents = [ doc.char_span(s["start"], s["end"], label=s["label"]) for s in eg.get("spans", []) ] doc_bin.add(doc) doc_bin.to_disk(output_path) print(f"Processed {len(doc_bin)} documents: {output_path.name}")
def main(input_path: Path = typer.Argument(..., exists=True, dir_okay=False)): print("Read params.yaml...") with open("params.yaml", "r") as fd: params = yaml.safe_load(fd) dev_size = params["train"]["corpora"]["dev_size"] shuffle_seed = params["train"]["corpora"]["shuffle_seed"] print(f"...read dev_size={dev_size}, shuffle_seed={shuffle_seed}") print("Read annotations...") corpus = list(srsly.read_jsonl(input_path)) print(f"...read {len(corpus)} texts") print("Convert into documents...") docs = [] nlp = spacy.blank("en") for eg in corpus: if eg["answer"] != "accept": continue tokens = [token["text"] for token in eg["tokens"]] words, spaces = get_words_and_spaces(tokens, eg["text"]) doc = Doc(nlp.vocab, words=words, spaces=spaces) doc.ents = [ doc.char_span(s["start"], s["end"], label=s["label"]) for s in eg.get("spans", []) ] docs.append(doc) print(f"...converted {len(docs)} documents") print("Split into train and dev...") train, dev = train_test_split(docs, test_size=dev_size, random_state=shuffle_seed, shuffle=True) print(f"...split into {len(train)} train and {len(dev)} dev documents") print("Write serialized documents...") for split, data in [("train", train), ("dev", dev)]: output_path = input_path.with_suffix(f".{split}.spacy") doc_bin = DocBin(attrs=["ENT_IOB", "ENT_TYPE"], docs=data) doc_bin.to_disk(output_path) print(f"...wrote {output_path}")
def test_gold_biluo_different_tokenization(en_vocab, en_tokenizer): # one-to-many words = ["I", "flew to", "San Francisco Valley", "."] spaces = [True, True, False, False] doc = Doc(en_vocab, words=words, spaces=spaces) entities = [(len("I flew to "), len("I flew to San Francisco Valley"), "LOC")] gp = GoldParse( doc, words=["I", "flew", "to", "San", "Francisco", "Valley", "."], entities=entities, ) assert gp.ner == ["O", "O", "U-LOC", "O"] # many-to-one words = ["I", "flew", "to", "San", "Francisco", "Valley", "."] spaces = [True, True, True, True, True, False, False] doc = Doc(en_vocab, words=words, spaces=spaces) entities = [(len("I flew to "), len("I flew to San Francisco Valley"), "LOC")] gp = GoldParse(doc, words=["I", "flew to", "San Francisco Valley", "."], entities=entities) assert gp.ner == ["O", "O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] # misaligned words = ["I flew", "to", "San Francisco", "Valley", "."] spaces = [True, True, True, False, False] doc = Doc(en_vocab, words=words, spaces=spaces) entities = [(len("I flew to "), len("I flew to San Francisco Valley"), "LOC")] gp = GoldParse( doc, words=["I", "flew to", "San", "Francisco Valley", "."], entities=entities, ) assert gp.ner == ["O", "O", "B-LOC", "L-LOC", "O"] # additional whitespace tokens in GoldParse words words, spaces = get_words_and_spaces( ["I", "flew", "to", "San Francisco", "Valley", "."], "I flew to San Francisco Valley.", ) doc = Doc(en_vocab, words=words, spaces=spaces) entities = [(len("I flew to "), len("I flew to San Francisco Valley"), "LOC")] gp = GoldParse( doc, words=["I", "flew", " ", "to", "San Francisco Valley", "."], entities=entities, ) assert gp.ner == ["O", "O", "O", "O", "B-LOC", "L-LOC", "O"] # from issue #4791 data = ( "I'll return the ₹54 amount", { "words": [ "I", "'ll", "return", "the", "₹", "54", "amount", ], "entities": [(16, 19, "MONEY")], }, ) gp = GoldParse(en_tokenizer(data[0]), **data[1]) assert gp.ner == ["O", "O", "O", "O", "U-MONEY", "O"] data = ( "I'll return the $54 amount", { "words": [ "I", "'ll", "return", "the", "$", "54", "amount", ], "entities": [(16, 19, "MONEY")], }, ) gp = GoldParse(en_tokenizer(data[0]), **data[1]) assert gp.ner == ["O", "O", "O", "O", "B-MONEY", "L-MONEY", "O"]