def test_en_parser_depset(NLP, test_file):
"""Check that no tags outside the tagset are produced."""
gold_deps = set([
"ROOT", "acl", "acomp", "advcl", "advmod", "agent", "amod", "appos",
"attr", "aux", "auxpass", "case", "cc", "ccomp", "compound", "conj",
"csubj", "csubjpass", "dative", "dep", "det", "dobj", "expl", "intj",
"mark", "meta", "neg", "nmod", "npadvmod", "nsubj", "nsubjpass",
"nummod", "oprd", "parataxis", "pcomp", "pobj", "poss", "preconj",
"predet", "prep", "prt", "punct", "quantmod", "relcl", "root", "xcomp"
])
data_path = os.path.join(TEST_FILES_DIR, test_file)
data_path = util.ensure_path(data_path)
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
pred_deps = set()
parser = NLP.get_pipe('parser')
for doc, _ in dev_docs:
parser(doc)
pred_deps = pred_deps.union(set([t.dep_ for t in doc]))
print(pred_deps - gold_deps)
assert len(pred_deps - gold_deps) == 0
def test_de_tagger_tagset(NLP, test_file):
"""Check that no tags outside the tagset are used."""
gold_tags = set([
"$(", "$,", "$.", "ADJA", "ADJD", "ADV", "APPO", "APPR", "APPRART",
"APZR", "ART", "CARD", "FM", "ITJ", "KOKOM", "KON", "KOUI", "KOUS",
"NE", "NN", "NNE", "PDAT", "PDS", "PIAT", "PIS", "PPER", "PPOSAT",
"PPOSS", "PRELAT", "PRELS", "PRF", "PROAV", "PTKA", "PTKANT", "PTKNEG",
"PTKVZ", "PTKZU", "PWAT", "PWAV", "PWS", "TRUNC", "VAFIN", "VAIMP",
"VAINF", "VAPP", "VMFIN", "VMINF", "VMPP", "VVFIN", "VVIMP", "VVINF",
"VVIZU", "VVPP", "XY"
])
data_path = os.path.join(TEST_FILES_DIR, test_file)
data_path = util.ensure_path(data_path)
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
pred_tags = set()
tagger = NLP.get_pipe('tagger')
for doc, _ in dev_docs:
tagger(doc)
pred_tags = pred_tags.union(set([t.tag_ for t in doc]))
assert len(pred_tags - gold_tags) == 0
def test_nb_ner_corpus(NLP, test_file, ents_f_threshold):
data_path = TEST_FILES_DIR / test_file
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
scorer = NLP.evaluate(dev_docs)
assert scorer.ents_f > ents_f_threshold
def test_ja_segmenter_corpus(NLP, test_file, accuracy_threshold):
data_path = TEST_FILES_DIR / test_file
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
scorer = NLP.evaluate(dev_docs)
assert scorer.token_acc > accuracy_threshold
def test_de_tagger_corpus(NLP, test_file, accuracy_threshold):
data_path = os.path.join(TEST_FILES_DIR, test_file)
data_path = util.ensure_path(data_path)
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
scorer = NLP.evaluate(dev_docs)
assert scorer.tags_acc > accuracy_threshold
def test_en_parser_depset(NLP, test_file):
"""Check that no tags outside the tagset are produced."""
gold_deps = set(NLP.get_pipe("parser").labels)
data_path = TEST_FILES_DIR / test_file
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
pred_deps = set()
parser = NLP.get_pipe("parser")
for doc, _ in dev_docs:
parser(doc)
pred_deps = pred_deps.union(set([t.dep_ for t in doc]))
assert len(pred_deps - gold_deps) == 0
def test_de_parser_depset(NLP, test_file):
"""Check that no tags outside the tagset are used."""
# fmt: off
gold_deps = set([
"ROOT", "ac", "adc", "ag", "ams", "app", "avc", "cc", "cd", "cj", "cm",
"cp", "cvc", "da", "dm", "ep", "ju", "mnr", "mo", "ng", "nk", "nmc",
"oa", "oc", "og", "op", "par", "pd", "pg", "ph", "pm", "pnc", "punct",
"rc", "re", "rs", "sb", "sbp", "sp", "svp", "uc", "vo"
])
# fmt: on
data_path = TEST_FILES_DIR / test_file
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
pred_deps = set()
parser = NLP.get_pipe("parser")
for doc, _ in dev_docs:
parser(doc)
pred_deps = pred_deps.union(set([t.dep_ for t in doc]))
assert len(pred_deps - gold_deps) == 0
def train_parser_and_tagger(train_json_path: str,
dev_json_path: str,
test_json_path: str,
model_output_dir: str,
model_path: str = None,
ontonotes_path: str = None,
ontonotes_train_percent: float = 0.0):
"""Function to train the spacy parser and tagger from a blank model, with the default, en_core_web_sm vocab.
Training setup is mostly copied from the spacy cli train command.
@param train_json_path: path to the conll formatted training data
@param dev_json_path: path to the conll formatted dev data
@param test_json_path: path to the conll formatted test data
@param model_output_dir: path to the output directory for the trained models
@param model_path: path to the model to load
@param ontonotes_path: path to the directory containnig ontonotes in spacy format (optional)
@param ontonotes_train_percent: percentage of the ontonotes training data to use (optional)
"""
msg = Printer()
train_json_path = cached_path(train_json_path)
dev_json_path = cached_path(dev_json_path)
test_json_path = cached_path(test_json_path)
if model_path is not None:
nlp = spacy.load(model_path)
else:
lang_class = util.get_lang_class('en')
nlp = lang_class()
if 'tagger' not in nlp.pipe_names:
tagger = nlp.create_pipe('tagger')
nlp.add_pipe(tagger, first=True)
else:
tagger = nlp.get_pipe('tagger')
if 'parser' not in nlp.pipe_names:
parser = nlp.create_pipe('parser')
nlp.add_pipe(parser)
else:
parser = nlp.get_pipe('parser')
train_corpus = GoldCorpus(train_json_path, dev_json_path)
test_corpus = GoldCorpus(train_json_path, test_json_path)
if ontonotes_path:
onto_train_path = os.path.join(ontonotes_path, "train")
onto_dev_path = os.path.join(ontonotes_path, "dev")
onto_test_path = os.path.join(ontonotes_path, "test")
onto_train_corpus = GoldCorpus(onto_train_path, onto_dev_path)
onto_test_corpus = GoldCorpus(onto_train_path, onto_test_path)
dropout_rates = util.decaying(0.2, 0.2, 0.0)
batch_sizes = util.compounding(1., 16., 1.001)
if model_path is not None:
meta = nlp.meta
else:
meta = {}
meta["lang"] = "en"
meta["pipeline"] = ["tagger", "parser"]
meta["name"] = "scispacy_core_web_sm"
meta["license"] = "CC BY-SA 3.0"
meta["author"] = "Allen Institute for Artificial Intelligence"
meta["url"] = "allenai.org"
meta["sources"] = ["OntoNotes 5", "Common Crawl", "GENIA 1.0"]
meta["version"] = "1.0.0"
meta["spacy_version"] = ">=2.2.1"
meta["parent_package"] = "spacy"
meta["email"] = "*****@*****.**"
n_train_words = train_corpus.count_train()
other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in ['tagger', 'parser']]
with nlp.disable_pipes(*other_pipes):
if ontonotes_path:
optimizer = nlp.begin_training(lambda: itertools.chain(train_corpus.train_tuples, onto_train_corpus.train_tuples))
else:
optimizer = nlp.begin_training(lambda: train_corpus.train_tuples)
nlp._optimizer = None
train_docs = train_corpus.train_docs(nlp)
train_docs = list(train_docs)
train_mixture = train_docs
if ontonotes_path:
onto_train_docs = onto_train_corpus.train_docs(nlp)
onto_train_docs = list(onto_train_docs)
num_onto_docs = int(float(ontonotes_train_percent)*len(onto_train_docs))
randomly_sampled_onto = random.sample(onto_train_docs, num_onto_docs)
train_mixture += randomly_sampled_onto
row_head, output_stats = _configure_training_output(nlp.pipe_names, -1, False)
row_widths = [len(w) for w in row_head]
row_settings = {"widths": row_widths, "aligns": tuple(["r" for i in row_head]), "spacing": 2}
print("")
msg.row(row_head, **row_settings)
msg.row(["-" * width for width in row_settings["widths"]], **row_settings)
best_epoch = 0
best_epoch_uas = 0.0
for i in range(20):
random.shuffle(train_mixture)
with nlp.disable_pipes(*other_pipes):
with tqdm(total=n_train_words, leave=False) as pbar:
losses = {}
minibatches = list(util.minibatch(train_docs, size=batch_sizes))
for batch in minibatches:
docs, golds = zip(*batch)
nlp.update(docs, golds, sgd=optimizer,
drop=next(dropout_rates), losses=losses)
pbar.update(sum(len(doc) for doc in docs))
# save intermediate model and output results on the dev set
with nlp.use_params(optimizer.averages):
epoch_model_path = os.path.join(model_output_dir, "model"+str(i))
os.makedirs(epoch_model_path, exist_ok=True)
nlp.to_disk(epoch_model_path)
with open(os.path.join(model_output_dir, "model"+str(i), "meta.json"), "w") as meta_fp:
meta_fp.write(json.dumps(meta))
nlp_loaded = util.load_model_from_path(epoch_model_path)
dev_docs = train_corpus.dev_docs(nlp_loaded)
dev_docs = list(dev_docs)
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs)
end_time = timer()
gpu_wps = None
cpu_wps = nwords/(end_time-start_time)
if ontonotes_path:
onto_dev_docs = list(onto_train_corpus.dev_docs(nlp_loaded))
onto_scorer = nlp_loaded.evaluate(onto_dev_docs)
if scorer.scores["uas"] > best_epoch_uas:
best_epoch_uas = scorer.scores["uas"]
best_epoch = i
progress = _get_progress(
i, losses, scorer.scores, output_stats, cpu_wps=cpu_wps, gpu_wps=gpu_wps
)
msg.row(progress, **row_settings)
if ontonotes_path:
progress = _get_progress(
i, losses, onto_scorer.scores, output_stats, cpu_wps=cpu_wps, gpu_wps=gpu_wps
)
msg.row(progress, **row_settings)
# save final model and output results on the test set
final_model_path = os.path.join(model_output_dir, "best")
if os.path.exists(final_model_path):
shutil.rmtree(final_model_path)
shutil.copytree(os.path.join(model_output_dir, "model" + str(best_epoch)),
final_model_path)
nlp_loaded = util.load_model_from_path(final_model_path)
start_time = timer()
test_docs = test_corpus.dev_docs(nlp_loaded)
test_docs = list(test_docs)
nwords = sum(len(doc_gold[0]) for doc_gold in test_docs)
scorer = nlp_loaded.evaluate(test_docs)
end_time = timer()
gpu_wps = None
cpu_wps = nwords/(end_time-start_time)
meta["speed"] = {"gpu": None, "nwords": nwords, "cpu": cpu_wps}
print("Retrained genia evaluation")
print("Test results:")
print("UAS:", scorer.uas)
print("LAS:", scorer.las)
print("Tag %:", scorer.tags_acc)
print("Token acc:", scorer.token_acc)
with open(os.path.join(model_output_dir, "genia_test.json"), "w+") as metric_file:
json.dump(scorer.scores, metric_file)
with open(os.path.join(model_output_dir, "best", "meta.json"), "w") as meta_fp:
meta_fp.write(json.dumps(meta))
if ontonotes_path:
onto_test_docs = list(onto_test_corpus.dev_docs(nlp_loaded))
print("Retrained ontonotes evaluation")
scorer_onto_retrained = nlp_loaded.evaluate(onto_test_docs)
print("Test results:")
print("UAS:", scorer_onto_retrained.uas)
print("LAS:", scorer_onto_retrained.las)
print("Tag %:", scorer_onto_retrained.tags_acc)
print("Token acc:", scorer_onto_retrained.token_acc)
with open(os.path.join(model_output_dir, "ontonotes_test.json"), "w+") as metric_file:
json.dump(scorer_onto_retrained.scores, metric_file)
def main(
model="./zh_vectors_web_ud_lg/model-final",
new_model_name="zh_vectors_web_ud_clue_lg",
output_dir="./zh_vectors_web_ud_clue_lg",
train_path="./clue_spacy_train.jsonl",
dev_path="./clue_spacy_dev.jsonl",
meta_path="./meta.json",
use_gpu=0,
n_iter=50
):
import tqdm
"""Set up the pipeline and entity recognizer, and train the new entity."""
random.seed(0)
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
# Add entity recognizer to model if it's not in the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if "ner" not in nlp.pipe_names:
ner = nlp.create_pipe("ner")
nlp.add_pipe(ner)
# otherwise, get it, so we can add labels to it
else:
ner = nlp.get_pipe("ner")
for label in LABEL:
if label not in ner.labels:
ner.add_label(label) # add new entity label to entity recognizer
train_path = ensure_path(train_path)
dev_path = ensure_path(dev_path)
if not train_path or not train_path.exists():
msg.fail("Training data not found", train_path, exits=1)
if not dev_path or not dev_path.exists():
msg.fail("Development data not found", dev_path, exits=1)
if output_dir.exists() and [p for p in output_dir.iterdir() if p.is_dir()]:
msg.warn(
"Output directory is not empty",
"This can lead to unintended side effects when saving the model. "
"Please use an empty directory or a different path instead. If "
"the specified output path doesn't exist, the directory will be "
"created for you.",
)
if not output_dir.exists():
output_dir.mkdir()
meta = srsly.read_json(meta_path) if meta_path else {}
# Prepare training corpus
msg.text("Counting training words (limit={})".format(0))
corpus = GoldCorpus(train_path, dev_path, limit=0)
n_train_words = corpus.count_train()
if model is None:
optimizer = nlp.begin_training(lambda: corpus.train_tuples, device=use_gpu)
else:
optimizer = create_default_optimizer(Model.ops)
# Todo: gpu train?
dropout_rates = decaying( 0.2, 0.2, 0.0)
batch_sizes = compounding( 100.0, 1000.0 , 1.001)
# get names of other pipes to disable them during training
pipe_exceptions = ["ner", "trf_wordpiecer", "trf_tok2vec"]
other_pipes = [
pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]
# UnboundLocalError: local variable 'has_beam_widths' referenced before assignment
# fmt: off
eval_beam_widths=""
if not eval_beam_widths:
eval_beam_widths = [1]
else:
eval_beam_widths = [int(bw) for bw in eval_beam_widths.split(",")]
if 1 not in eval_beam_widths:
eval_beam_widths.append(1)
eval_beam_widths.sort()
has_beam_widths = eval_beam_widths != [1]
row_head, output_stats = _configure_training_output(["ner"], use_gpu, has_beam_widths)
row_widths = [len(w) for w in row_head]
row_settings = {"widths": row_widths, "aligns": tuple(["r" for i in row_head]), "spacing": 2}
# fmt: on
print("")
msg.row(row_head, **row_settings)
msg.row(["-" * width for width in row_settings["widths"]], **row_settings)
try:
noise_level = 0.0
orth_variant_level = 0.0
gold_preproc = False
verbose = False
best_score = 0.0
with nlp.disable_pipes(*other_pipes): # only train NER
for itn in range(n_iter):
train_docs = corpus.train_docs(
nlp,
noise_level=noise_level,
orth_variant_level=orth_variant_level,
gold_preproc=gold_preproc,
max_length=0,
ignore_misaligned=True,
)
words_seen = 0
with tqdm.tqdm(total=n_train_words, leave=False) as pbar:
losses = {}
for batch in minibatch_by_words(train_docs, size=batch_sizes):
if not batch:
continue
docs, golds = zip(*batch)
nlp.update(
docs,
golds,
sgd=optimizer,
drop=next(dropout_rates),
losses=losses,
)
if not int(os.environ.get("LOG_FRIENDLY", 0)):
pbar.update(sum(len(doc) for doc in docs))
words_seen += sum(len(doc) for doc in docs)
with nlp.use_params(optimizer.averages):
set_env_log(False)
epoch_model_path = output_dir / ("model%d" % itn)
nlp.to_disk(epoch_model_path)
nlp_loaded = load_model_from_path(epoch_model_path)
for beam_width in eval_beam_widths:
for name, component in nlp_loaded.pipeline:
if hasattr(component, "cfg"):
component.cfg["beam_width"] = beam_width
dev_docs = list(
corpus.dev_docs(
nlp_loaded,
gold_preproc=gold_preproc,
ignore_misaligned=True,
)
)
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs, verbose=verbose)
end_time = timer()
if use_gpu < 0:
gpu_wps = None
cpu_wps = nwords / (end_time - start_time)
else:
gpu_wps = nwords / (end_time - start_time)
with Model.use_device("cpu"):
nlp_loaded = load_model_from_path(epoch_model_path)
for name, component in nlp_loaded.pipeline:
if hasattr(component, "cfg"):
component.cfg["beam_width"] = beam_width
dev_docs = list(
corpus.dev_docs(
nlp_loaded,
gold_preproc=gold_preproc,
ignore_misaligned=True,
)
)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs, verbose=verbose)
end_time = timer()
cpu_wps = nwords / (end_time - start_time)
acc_loc = output_dir / ("model%d" % itn) / "accuracy.json"
srsly.write_json(acc_loc, scorer.scores)
# Update model meta.json
meta["lang"] = nlp.lang
meta["pipeline"] = nlp.pipe_names
meta["spacy_version"] = ">=%s" % spacy.__version__
if beam_width == 1:
meta["speed"] = {
"nwords": nwords,
"cpu": cpu_wps,
"gpu": gpu_wps,
}
meta["accuracy"] = scorer.scores
else:
meta.setdefault("beam_accuracy", {})
meta.setdefault("beam_speed", {})
meta["beam_accuracy"][beam_width] = scorer.scores
meta["beam_speed"][beam_width] = {
"nwords": nwords,
"cpu": cpu_wps,
"gpu": gpu_wps,
}
meta["vectors"] = {
"width": nlp.vocab.vectors_length,
"vectors": len(nlp.vocab.vectors),
"keys": nlp.vocab.vectors.n_keys,
"name": nlp.vocab.vectors.name,
}
meta.setdefault("name", "model%d" % itn)
meta.setdefault("version", "0.0.1")
meta["labels"] = nlp.meta["labels"]
meta_loc = output_dir / ("model%d" % itn) / "meta.json"
srsly.write_json(meta_loc, meta)
set_env_log(verbose)
progress = _get_progress(
itn,
losses,
scorer.scores,
output_stats,
beam_width=beam_width if has_beam_widths else None,
cpu_wps=cpu_wps,
gpu_wps=gpu_wps,
)
msg.row(progress, **row_settings)
finally:
with nlp.use_params(optimizer.averages):
final_model_path = output_dir / "model-final"
nlp.to_disk(final_model_path)
msg.good("Saved model to output directory", final_model_path)
meta["pipeline"] = nlp.pipe_names
meta["labels"] = nlp.meta["labels"]
meta["factories"] = nlp.meta["factories"]
with msg.loading("Creating best model..."):
best_model_path = _collate_best_model(meta, output_dir, nlp.pipe_names)
msg.good("Created best model", best_model_path)
def custom_train(
lang,
output_path,
train_path,
dev_path,
raw_text=None,
base_model=None,
pipeline="tagger,parser,ner",
vectors=None,
n_iter=30,
n_early_stopping=None,
n_examples=0,
use_gpu=-1,
version="0.0.0",
meta_path=None,
init_tok2vec=None,
parser_multitasks="",
entity_multitasks="",
noise_level=0.0,
orth_variant_level=0.0,
eval_beam_widths="",
gold_preproc=False,
learn_tokens=False,
textcat_multilabel=False,
textcat_arch="bow",
textcat_positive_label=None,
verbose=False,
debug=False,
):
"""
Train or update a spaCy model. Requires data to be formatted in spaCy's
JSON format. To convert data from other formats, use the `spacy convert`
command.
"""
# temp fix to avoid import issues cf https://github.com/explosion/spaCy/issues/4200
import tqdm
msg = Printer()
util.fix_random_seed()
util.set_env_log(verbose)
# Make sure all files and paths exists if they are needed
train_path = util.ensure_path(train_path)
dev_path = util.ensure_path(dev_path)
meta_path = util.ensure_path(meta_path)
output_path = util.ensure_path(output_path)
if raw_text is not None:
raw_text = list(srsly.read_jsonl(raw_text))
if not train_path or not train_path.exists():
msg.fail("Training data not found", train_path, exits=1)
if not dev_path or not dev_path.exists():
msg.fail("Development data not found", dev_path, exits=1)
if meta_path is not None and not meta_path.exists():
msg.fail("Can't find model meta.json", meta_path, exits=1)
meta = srsly.read_json(meta_path) if meta_path else {}
if output_path.exists() and [
p for p in output_path.iterdir() if p.is_dir()
]:
msg.warn(
"Output directory is not empty",
"This can lead to unintended side effects when saving the model. "
"Please use an empty directory or a different path instead. If "
"the specified output path doesn't exist, the directory will be "
"created for you.",
)
if not output_path.exists():
output_path.mkdir()
# Take dropout and batch size as generators of values -- dropout
# starts high and decays sharply, to force the optimizer to explore.
# Batch size starts at 1 and grows, so that we make updates quickly
# at the beginning of training.
dropout_rates = util.decaying(
util.env_opt("dropout_from", 0.2),
util.env_opt("dropout_to", 0.2),
util.env_opt("dropout_decay", 0.0),
)
batch_sizes = util.compounding(
util.env_opt("batch_from", 100.0),
util.env_opt("batch_to", 1000.0),
util.env_opt("batch_compound", 1.001),
)
if not eval_beam_widths:
eval_beam_widths = [1]
else:
eval_beam_widths = [int(bw) for bw in eval_beam_widths.split(",")]
if 1 not in eval_beam_widths:
eval_beam_widths.append(1)
eval_beam_widths.sort()
has_beam_widths = eval_beam_widths != [1]
# Set up the base model and pipeline. If a base model is specified, load
# the model and make sure the pipeline matches the pipeline setting. If
# training starts from a blank model, intitalize the language class.
pipeline = [p.strip() for p in pipeline.split(",")]
msg.text("Training pipeline: {}".format(pipeline))
if base_model:
msg.text("Starting with base model '{}'".format(base_model))
nlp = util.load_model(base_model)
if nlp.lang != lang:
msg.fail(
"Model language ('{}') doesn't match language specified as "
"`lang` argument ('{}') ".format(nlp.lang, lang),
exits=1,
)
nlp.disable_pipes([p for p in nlp.pipe_names if p not in pipeline])
for pipe in pipeline:
if pipe not in nlp.pipe_names:
if pipe == "parser":
pipe_cfg = {"learn_tokens": learn_tokens}
elif pipe == "textcat":
pipe_cfg = {
"exclusive_classes": not textcat_multilabel,
"architecture": textcat_arch,
"positive_label": textcat_positive_label,
}
else:
pipe_cfg = {}
nlp.add_pipe(nlp.create_pipe(pipe, config=pipe_cfg))
else:
if pipe == "textcat":
textcat_cfg = nlp.get_pipe("textcat").cfg
base_cfg = {
"exclusive_classes": textcat_cfg["exclusive_classes"],
"architecture": textcat_cfg["architecture"],
"positive_label": textcat_cfg["positive_label"],
}
pipe_cfg = {
"exclusive_classes": not textcat_multilabel,
"architecture": textcat_arch,
"positive_label": textcat_positive_label,
}
if base_cfg != pipe_cfg:
msg.fail(
"The base textcat model configuration does"
"not match the provided training options. "
"Existing cfg: {}, provided cfg: {}".format(
base_cfg, pipe_cfg),
exits=1,
)
else:
msg.text("Starting with blank model '{}'".format(lang))
lang_cls = util.get_lang_class(lang)
### Here are our modifications:
lang_cls.Defaults.tag_map = custom_tag_map
nlp = lang_cls()
assert nlp.vocab.morphology.n_tags == 36
###
for pipe in pipeline:
if pipe == "parser":
pipe_cfg = {"learn_tokens": learn_tokens}
elif pipe == "textcat":
pipe_cfg = {
"exclusive_classes": not textcat_multilabel,
"architecture": textcat_arch,
"positive_label": textcat_positive_label,
}
else:
pipe_cfg = {}
nlp.add_pipe(nlp.create_pipe(pipe, config=pipe_cfg))
if vectors:
msg.text("Loading vector from model '{}'".format(vectors))
_load_vectors(nlp, vectors)
# Multitask objectives
multitask_options = [("parser", parser_multitasks),
("ner", entity_multitasks)]
for pipe_name, multitasks in multitask_options:
if multitasks:
if pipe_name not in pipeline:
msg.fail("Can't use multitask objective without '{}' in the "
"pipeline".format(pipe_name))
pipe = nlp.get_pipe(pipe_name)
for objective in multitasks.split(","):
pipe.add_multitask_objective(objective)
# Prepare training corpus
msg.text("Counting training words (limit={})".format(n_examples))
corpus = GoldCorpus(train_path, dev_path, limit=n_examples)
n_train_words = corpus.count_train()
if base_model:
# Start with an existing model, use default optimizer
optimizer = create_default_optimizer(Model.ops)
else:
# Start with a blank model, call begin_training
optimizer = nlp.begin_training(lambda: corpus.train_tuples,
device=use_gpu)
nlp._optimizer = None
# Load in pretrained weights
if init_tok2vec is not None:
components = _load_pretrained_tok2vec(nlp, init_tok2vec)
msg.text("Loaded pretrained tok2vec for: {}".format(components))
# Verify textcat config
if "textcat" in pipeline:
textcat_labels = nlp.get_pipe("textcat").cfg["labels"]
if textcat_positive_label and textcat_positive_label not in textcat_labels:
msg.fail(
"The textcat_positive_label (tpl) '{}' does not match any "
"label in the training data.".format(textcat_positive_label),
exits=1,
)
if textcat_positive_label and len(textcat_labels) != 2:
msg.fail(
"A textcat_positive_label (tpl) '{}' was provided for training "
"data that does not appear to be a binary classification "
"problem with two labels.".format(textcat_positive_label),
exits=1,
)
train_docs = corpus.train_docs(
nlp,
noise_level=noise_level,
gold_preproc=gold_preproc,
max_length=0,
ignore_misaligned=True,
)
train_labels = set()
if textcat_multilabel:
multilabel_found = False
for text, gold in train_docs:
train_labels.update(gold.cats.keys())
if list(gold.cats.values()).count(1.0) != 1:
multilabel_found = True
if not multilabel_found and not base_model:
msg.warn("The textcat training instances look like they have "
"mutually-exclusive classes. Remove the flag "
"'--textcat-multilabel' to train a classifier with "
"mutually-exclusive classes.")
if not textcat_multilabel:
for text, gold in train_docs:
train_labels.update(gold.cats.keys())
if list(gold.cats.values()).count(1.0) != 1 and not base_model:
msg.warn(
"Some textcat training instances do not have exactly "
"one positive label. Modifying training options to "
"include the flag '--textcat-multilabel' for classes "
"that are not mutually exclusive.")
nlp.get_pipe("textcat").cfg["exclusive_classes"] = False
textcat_multilabel = True
break
if base_model and set(textcat_labels) != train_labels:
msg.fail(
"Cannot extend textcat model using data with different "
"labels. Base model labels: {}, training data labels: "
"{}.".format(textcat_labels, list(train_labels)),
exits=1,
)
if textcat_multilabel:
msg.text(
"Textcat evaluation score: ROC AUC score macro-averaged across "
"the labels '{}'".format(", ".join(textcat_labels)))
elif textcat_positive_label and len(textcat_labels) == 2:
msg.text("Textcat evaluation score: F1-score for the "
"label '{}'".format(textcat_positive_label))
elif len(textcat_labels) > 1:
if len(textcat_labels) == 2:
msg.warn(
"If the textcat component is a binary classifier with "
"exclusive classes, provide '--textcat_positive_label' for "
"an evaluation on the positive class.")
msg.text(
"Textcat evaluation score: F1-score macro-averaged across "
"the labels '{}'".format(", ".join(textcat_labels)))
else:
msg.fail(
"Unsupported textcat configuration. Use `spacy debug-data` "
"for more information.")
# fmt: off
row_head, output_stats = _configure_training_output(
pipeline, use_gpu, has_beam_widths)
row_widths = [len(w) for w in row_head]
row_settings = {
"widths": row_widths,
"aligns": tuple(["r" for i in row_head]),
"spacing": 2
}
# fmt: on
print("")
msg.row(row_head, **row_settings)
msg.row(["-" * width for width in row_settings["widths"]], **row_settings)
try:
iter_since_best = 0
best_score = 0.0
for i in range(n_iter):
train_docs = corpus.train_docs(
nlp,
noise_level=noise_level,
orth_variant_level=orth_variant_level,
gold_preproc=gold_preproc,
max_length=0,
ignore_misaligned=True,
)
if raw_text:
random.shuffle(raw_text)
raw_batches = util.minibatch(
(nlp.make_doc(rt["text"]) for rt in raw_text), size=8)
words_seen = 0
with tqdm.tqdm(total=n_train_words, leave=False) as pbar:
losses = {}
for batch in util.minibatch_by_words(train_docs,
size=batch_sizes):
if not batch:
continue
docs, golds = zip(*batch)
nlp.update(
docs,
golds,
sgd=optimizer,
drop=next(dropout_rates),
losses=losses,
)
if raw_text:
# If raw text is available, perform 'rehearsal' updates,
# which use unlabelled data to reduce overfitting.
raw_batch = list(next(raw_batches))
nlp.rehearse(raw_batch, sgd=optimizer, losses=losses)
if not int(os.environ.get("LOG_FRIENDLY", 0)):
pbar.update(sum(len(doc) for doc in docs))
words_seen += sum(len(doc) for doc in docs)
with nlp.use_params(optimizer.averages):
util.set_env_log(False)
epoch_model_path = output_path / ("model%d" % i)
nlp.to_disk(epoch_model_path)
nlp_loaded = util.load_model_from_path(epoch_model_path)
for beam_width in eval_beam_widths:
for name, component in nlp_loaded.pipeline:
if hasattr(component, "cfg"):
component.cfg["beam_width"] = beam_width
dev_docs = list(
corpus.dev_docs(
nlp_loaded,
gold_preproc=gold_preproc,
ignore_misaligned=True,
))
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs, verbose=verbose)
end_time = timer()
if use_gpu < 0:
gpu_wps = None
cpu_wps = nwords / (end_time - start_time)
else:
gpu_wps = nwords / (end_time - start_time)
with Model.use_device("cpu"):
nlp_loaded = util.load_model_from_path(
epoch_model_path)
for name, component in nlp_loaded.pipeline:
if hasattr(component, "cfg"):
component.cfg["beam_width"] = beam_width
dev_docs = list(
corpus.dev_docs(
nlp_loaded,
gold_preproc=gold_preproc,
ignore_misaligned=True,
))
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs,
verbose=verbose)
end_time = timer()
cpu_wps = nwords / (end_time - start_time)
acc_loc = output_path / ("model%d" % i) / "accuracy.json"
srsly.write_json(acc_loc, scorer.scores)
# Update model meta.json
meta["lang"] = nlp.lang
meta["pipeline"] = nlp.pipe_names
meta["spacy_version"] = ">=%s" % about.__version__
if beam_width == 1:
meta["speed"] = {
"nwords": nwords,
"cpu": cpu_wps,
"gpu": gpu_wps,
}
meta["accuracy"] = scorer.scores
else:
meta.setdefault("beam_accuracy", {})
meta.setdefault("beam_speed", {})
meta["beam_accuracy"][beam_width] = scorer.scores
meta["beam_speed"][beam_width] = {
"nwords": nwords,
"cpu": cpu_wps,
"gpu": gpu_wps,
}
meta["vectors"] = {
"width": nlp.vocab.vectors_length,
"vectors": len(nlp.vocab.vectors),
"keys": nlp.vocab.vectors.n_keys,
"name": nlp.vocab.vectors.name,
}
meta.setdefault("name", "model%d" % i)
meta.setdefault("version", version)
meta["labels"] = nlp.meta["labels"]
meta_loc = output_path / ("model%d" % i) / "meta.json"
srsly.write_json(meta_loc, meta)
util.set_env_log(verbose)
progress = _get_progress(
i,
losses,
scorer.scores,
output_stats,
beam_width=beam_width if has_beam_widths else None,
cpu_wps=cpu_wps,
gpu_wps=gpu_wps,
)
if i == 0 and "textcat" in pipeline:
textcats_per_cat = scorer.scores.get(
"textcats_per_cat", {})
for cat, cat_score in textcats_per_cat.items():
if cat_score.get("roc_auc_score", 0) < 0:
msg.warn(
"Textcat ROC AUC score is undefined due to "
"only one value in label '{}'.".format(
cat))
msg.row(progress, **row_settings)
# Early stopping
if n_early_stopping is not None:
current_score = _score_for_model(meta)
if current_score < best_score:
iter_since_best += 1
else:
iter_since_best = 0
best_score = current_score
if iter_since_best >= n_early_stopping:
msg.text("Early stopping, best iteration "
"is: {}".format(i - iter_since_best))
msg.text("Best score = {}; Final iteration "
"score = {}".format(best_score,
current_score))
break
finally:
with nlp.use_params(optimizer.averages):
final_model_path = output_path / "model-final"
nlp.to_disk(final_model_path)
msg.good("Saved model to output directory", final_model_path)
with msg.loading("Creating best model..."):
best_model_path = _collate_best_model(meta, output_path,
nlp.pipe_names)
msg.good("Created best model", best_model_path)
def train(pretrained,
output_dir,
train_data,
dev_data,
n_iter=30,
n_sents=0,
parser_multitasks='',
entity_multitasks='',
use_gpu=-1,
no_tagger=False,
no_parser=False,
no_entities=False,
gold_preproc=False,
version="0.0.0",
meta_path=None,
verbose=False):
"""
Re-train a pre-trained model. Expects data in spaCy's JSON
format. This code is based on
https://github.com/explosion/spaCy/blob/master/spacy/cli/train.py.
"""
# There is a bug that prevents me from using the GPU when resuming
# training from a saved model. See
# https://github.com/explosion/spaCy/issues/1806.
if use_gpu >= 0:
msg = "\nWARNING: using GPU may require re-installing thinc. "
msg += "See https://github.com/explosion/spaCy/issues/1806.\n"
print(msg)
util.fix_random_seed()
util.set_env_log(True)
n_sents = n_sents or None
output_path = util.ensure_path(output_dir)
train_path = util.ensure_path(train_data)
dev_path = util.ensure_path(dev_data)
meta_path = util.ensure_path(meta_path)
if not output_path.exists():
output_path.mkdir()
if not train_path.exists():
prints(train_path, title=Messages.M050, exits=1)
if dev_path and not dev_path.exists():
prints(dev_path, title=Messages.M051, exits=1)
if meta_path is not None and not meta_path.exists():
prints(meta_path, title=Messages.M020, exits=1)
meta = util.read_json(meta_path) if meta_path else {}
if not isinstance(meta, dict):
prints(Messages.M053.format(meta_type=type(meta)),
title=Messages.M052,
exits=1)
# Take dropout and batch size as generators of values -- dropout
# starts high and decays sharply, to force the optimizer to explore.
# Batch size starts at 1 and grows, so that we make updates quickly
# at the beginning of training.
dropout_rates = util.decaying(util.env_opt('dropout_from', 0.2),
util.env_opt('dropout_to', 0.2),
util.env_opt('dropout_decay', 0.0))
batch_sizes = util.compounding(util.env_opt('batch_from', 1),
util.env_opt('batch_to', 16),
util.env_opt('batch_compound', 1.001))
max_doc_len = util.env_opt('max_doc_len', 5000)
corpus = GoldCorpus(train_path, dev_path, limit=n_sents)
n_train_words = corpus.count_train()
# Load pre-trained model. Remove components that we are not
# re-training.
nlp = load(pretrained)
if no_tagger and 'tagger' in nlp.pipe_names:
nlp.remove_pipe('tagger')
if no_parser and 'parser' in nlp.pipe_names:
nlp.remove_pipe('parser')
if no_entities and 'ner' in nlp.pipe_names:
nlp.remove_pipe('ner')
meta.setdefault('name', 'unnamed')
meta['pipeline'] = nlp.pipe_names
meta.setdefault('lang', nlp.lang)
nlp.meta.update(meta)
# Add multi-task objectives
if parser_multitasks:
for objective in parser_multitasks.split(','):
nlp.parser.add_multitask_objective(objective)
if entity_multitasks:
for objective in entity_multitasks.split(','):
nlp.entity.add_multitask_objective(objective)
# Get optimizer
optimizer = nlp.begin_training(lambda: corpus.train_tuples, device=use_gpu)
nlp._optimizer = None
print(nlp.pipe_names)
print(nlp.pipeline)
print(
"Itn. Dep Loss NER Loss UAS NER P. NER R. NER F. Tag % Token % CPU WPS GPU WPS"
)
try:
train_docs = corpus.train_docs(nlp,
projectivize=True,
noise_level=0.0,
gold_preproc=gold_preproc,
max_length=0)
train_docs = list(train_docs)
for i in range(n_iter):
with tqdm.tqdm(total=n_train_words, leave=False) as pbar:
losses = {}
for batch in minibatch(train_docs, size=batch_sizes):
batch = [(d, g) for (d, g) in batch
if len(d) < max_doc_len]
if not batch:
continue
docs, golds = zip(*batch)
nlp.update(docs,
golds,
sgd=optimizer,
drop=next(dropout_rates),
losses=losses)
pbar.update(sum(len(doc) for doc in docs))
with nlp.use_params(optimizer.averages):
util.set_env_log(False)
epoch_model_path = output_path / ('model%d' % i)
nlp.to_disk(epoch_model_path)
nlp_loaded = util.load_model_from_path(epoch_model_path)
dev_docs = list(
corpus.dev_docs(nlp_loaded, gold_preproc=gold_preproc))
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs, verbose)
end_time = timer()
if use_gpu < 0:
gpu_wps = None
cpu_wps = nwords / (end_time - start_time)
else:
gpu_wps = nwords / (end_time - start_time)
with Model.use_device('cpu'):
nlp_loaded = util.load_model_from_path(
epoch_model_path)
dev_docs = list(
corpus.dev_docs(nlp_loaded,
gold_preproc=gold_preproc))
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs)
end_time = timer()
cpu_wps = nwords / (end_time - start_time)
acc_loc = (output_path / ('model%d' % i) / 'accuracy.json')
with acc_loc.open('w') as file_:
file_.write(json_dumps(scorer.scores))
meta_loc = output_path / ('model%d' % i) / 'meta.json'
meta['accuracy'] = scorer.scores
meta['speed'] = {
'nwords': nwords,
'cpu': cpu_wps,
'gpu': gpu_wps
}
meta['vectors'] = {
'width': nlp.vocab.vectors_length,
'vectors': len(nlp.vocab.vectors),
'keys': nlp.vocab.vectors.n_keys
}
meta['lang'] = nlp.lang
meta['pipeline'] = nlp.pipe_names
meta['spacy_version'] = '>=%s' % about.__version__
meta.setdefault('name', 'model%d' % i)
meta.setdefault('version', version)
with meta_loc.open('w') as file_:
file_.write(json_dumps(meta))
util.set_env_log(True)
print_progress(i,
losses,
scorer.scores,
cpu_wps=cpu_wps,
gpu_wps=gpu_wps)
finally:
print("Saving model...")
with nlp.use_params(optimizer.averages):
final_model_path = output_path / 'model-final'
nlp.to_disk(final_model_path)
def test_de_parser_depset(NLP, test_file):
"""Check that no tags outside the tagset are used."""
# fmt: off
gold_deps = set([
"ROOT",
"ac",
"adc",
"ag",
"ams",
"app",
"avc",
"cc",
"cd",
"cj",
"cm",
"cp",
"cvc",
"da",
"dm",
"ep",
"ju",
"mnr",
"mo",
"ng",
"nk",
"nmc",
"oa",
"oc",
"og",
"op",
"par",
"pd",
"pg",
"ph",
"pm",
"pnc",
"punct",
"rc",
"re",
"rs",
"sb",
"sbp",
"sp",
"svp",
"uc",
"vo",
# The 'dep' label is introduced by the frequency cutoff
# during projectivisation. So we still need this, even
# though it doesn't occur in the gold data.
"dep"
])
# fmt: on
data_path = TEST_FILES_DIR / test_file
if not data_path.exists():
raise FileNotFoundError("Test corpus not found", data_path)
corpus = GoldCorpus(data_path, data_path)
dev_docs = list(corpus.dev_docs(NLP, gold_preproc=False))
pred_deps = set()
parser = NLP.get_pipe("parser")
for doc, _ in dev_docs:
parser(doc)
pred_deps = pred_deps.union(set([t.dep_ for t in doc]))
unexpected_labels = pred_deps - gold_deps
assert not unexpected_labels