def create_default_optimizer(ops, **cfg): learn_rate = util.env_opt("learn_rate", 0.001) beta1 = util.env_opt("optimizer_B1", 0.9) beta2 = util.env_opt("optimizer_B2", 0.999) eps = util.env_opt("optimizer_eps", 1e-8) L2 = util.env_opt("L2_penalty", 1e-6) max_grad_norm = util.env_opt("grad_norm_clip", 1.0) optimizer = Adam(ops, learn_rate, L2=L2, beta1=beta1, beta2=beta2, eps=eps) optimizer.max_grad_norm = max_grad_norm optimizer.device = ops.device return optimizer
def create_default_optimizer(ops, **cfg): learn_rate = util.env_opt("learn_rate", 0.001) beta1 = util.env_opt("optimizer_B1", 0.9) beta2 = util.env_opt("optimizer_B2", 0.999) eps = util.env_opt("optimizer_eps", 1e-8) L2 = util.env_opt("L2_penalty", 1e-6) max_grad_norm = util.env_opt("grad_norm_clip", 1.0) optimizer = Adam(ops, learn_rate, L2=L2, beta1=beta1, beta2=beta2, eps=eps) optimizer.max_grad_norm = max_grad_norm optimizer.device = ops.device return optimizer
def create_default_optimizer(ops, **cfg): learn_rate = util.env_opt('learn_rate', 0.001) beta1 = util.env_opt('optimizer_B1', 0.9) beta2 = util.env_opt('optimizer_B2', 0.999) eps = util.env_opt('optimizer_eps', 1e-08) L2 = util.env_opt('L2_penalty', 1e-6) max_grad_norm = util.env_opt('grad_norm_clip', 1.) optimizer = Adam(ops, learn_rate, L2=L2, beta1=beta1, beta2=beta2, eps=eps) optimizer.max_grad_norm = max_grad_norm optimizer.device = ops.device return optimizer
def create_default_optimizer(ops, **cfg): learn_rate = util.env_opt('learn_rate', 0.001) beta1 = util.env_opt('optimizer_B1', 0.9) beta2 = util.env_opt('optimizer_B2', 0.999) eps = util.env_opt('optimizer_eps', 1e-08) L2 = util.env_opt('L2_penalty', 1e-6) max_grad_norm = util.env_opt('grad_norm_clip', 1.) optimizer = Adam(ops, learn_rate, L2=L2, beta1=beta1, beta2=beta2, eps=eps) optimizer.max_grad_norm = max_grad_norm optimizer.device = ops.device return optimizer
def test_add_label(parser): doc = Doc(parser.vocab, words=['a', 'b', 'c', 'd']) doc = parser(doc) assert doc[0].head.i == 1 assert doc[0].dep_ == 'left' assert doc[1].head.i == 1 assert doc[2].head.i == 3 assert doc[2].head.i == 3 parser.add_label('right') doc = Doc(parser.vocab, words=['a', 'b', 'c', 'd']) doc = parser(doc) assert doc[0].head.i == 1 assert doc[0].dep_ == 'left' assert doc[1].head.i == 1 assert doc[2].head.i == 3 assert doc[2].head.i == 3 sgd = Adam(NumpyOps(), 0.001) for i in range(10): losses = {} doc = Doc(parser.vocab, words=['a', 'b', 'c', 'd']) gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=['right', 'ROOT', 'left', 'ROOT']) parser.update([doc], [gold], sgd=sgd, losses=losses) doc = Doc(parser.vocab, words=['a', 'b', 'c', 'd']) doc = parser(doc) assert doc[0].dep_ == 'right' assert doc[2].dep_ == 'left'
def test_add_label(parser): doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) doc = parser(doc) assert doc[0].head.i == 1 assert doc[0].dep_ == "left" assert doc[1].head.i == 1 assert doc[2].head.i == 3 assert doc[2].head.i == 3 parser.add_label("right") doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) doc = parser(doc) assert doc[0].head.i == 1 assert doc[0].dep_ == "left" assert doc[1].head.i == 1 assert doc[2].head.i == 3 assert doc[2].head.i == 3 sgd = Adam(NumpyOps(), 0.001) for i in range(10): losses = {} doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=["right", "ROOT", "left", "ROOT"]) parser.update([doc], [gold], sgd=sgd, losses=losses) doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) doc = parser(doc) assert doc[0].dep_ == "right" assert doc[2].dep_ == "left"
def test_wrapper_from_pretrained(name, model, ids): outputs, backprop = model.begin_update(ids.reshape((1, -1))) assert outputs.has_lh if outputs.has_po: assert hasattr(outputs.po[0], "shape") optimizer = Adam(model.ops, 0.001) d_outputs = Activations(outputs.lh, [], [], [], is_grad=True) backprop(d_outputs, sgd=optimizer)
def _train_parser(parser): fix_random_seed(1) parser.add_label("left") parser.begin_training([], **parser.cfg) sgd = Adam(NumpyOps(), 0.001) for i in range(5): losses = {} doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=["left", "ROOT", "left", "ROOT"]) parser.update([doc], [gold], sgd=sgd, losses=losses) return parser
def parser(vocab): parser = DependencyParser(vocab) parser.cfg["token_vector_width"] = 4 parser.cfg["hidden_width"] = 32 # parser.add_label('right') parser.add_label("left") parser.begin_training([], **parser.cfg) sgd = Adam(NumpyOps(), 0.001) for i in range(10): losses = {} doc = Doc(vocab, words=["a", "b", "c", "d"]) gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=["left", "ROOT", "left", "ROOT"]) parser.update([doc], [gold], sgd=sgd, losses=losses) return parser
def parser(vocab): parser = DependencyParser(vocab) parser.cfg['token_vector_width'] = 4 parser.cfg['hidden_width'] = 32 #parser.add_label('right') parser.add_label('left') parser.begin_training([], **parser.cfg) sgd = Adam(NumpyOps(), 0.001) for i in range(10): losses = {} doc = Doc(vocab, words=['a', 'b', 'c', 'd']) gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=['left', 'ROOT', 'left', 'ROOT']) parser.update([doc], [gold], sgd=sgd, losses=losses) return parser
def test_issue910(EN, train_data, additional_entity_types): '''Test that adding entities and resuming training works passably OK. There are two issues here: 1) We have to readd labels. This isn't very nice. 2) There's no way to set the learning rate for the weight update, so we end up out-of-scale, causing it to learn too fast. ''' nlp = EN doc = nlp(u"I am looking for a restaurant in Berlin") ents_before_train = [(ent.label_, ent.text) for ent in doc.ents] # Fine tune the ner model for entity_type in additional_entity_types: nlp.entity.add_label(entity_type) sgd = Adam(nlp.entity.model[0].ops, 0.001) for itn in range(10): random.shuffle(train_data) for raw_text, entity_offsets in train_data: doc = nlp.make_doc(raw_text) nlp.tagger(doc) nlp.tensorizer(doc) gold = GoldParse(doc, entities=entity_offsets) loss = nlp.entity.update(doc, gold, sgd=sgd, drop=0.5) with temp_save_model(nlp.entity) as model_dir: # Load the fine tuned model loaded_ner = EntityRecognizer(nlp.vocab) loaded_ner.from_disk(model_dir) for raw_text, entity_offsets in train_data: doc = nlp.make_doc(raw_text) nlp.tagger(doc) loaded_ner(doc) ents = {(ent.start_char, ent.end_char): ent.label_ for ent in doc.ents} for start, end, label in entity_offsets: assert ents[(start, end)] == label
def test_wrapper_from_pretrained(name, model, inputs): outputs, backprop = model.begin_update(inputs) assert outputs.has_lh optimizer = Adam(model.ops, 0.001) d_outputs = Activations(outputs.lh, RaggedArray.blank()) backprop(d_outputs, sgd=optimizer)