def test_KPI(self): metrics = bu.model(['-t', 'deeppavlov.tasks.insults.agents:FullTeacher', '-m', 'deeppavlov.agents.insults.insults_agents:EnsembleInsultsAgent', '--model_file', './build/insults/insults_ensemble', '--model_files', './build/insults/cnn_word_0', './build/insults/cnn_word_1', './build/insults/cnn_word_2', '--model_names', 'cnn_word', 'cnn_word', 'cnn_word', '--model_coefs', '0.3333333', '0.3333333', '0.3333334', '--datatype', 'test', '--batchsize', '64', '--display-examples', 'False', '--raw-dataset-path', './build/insults/', '--max_sequence_length', '100', '--filters_cnn', '256', '--kernel_sizes_cnn', '1 2 3', '--embedding_dim', '100', '--dense_dim', '100', '--fasttext_model', './build/insults/reddit_fasttext_model.bin' ]) TestModels.report_score("insults", metrics["auc"], self.expected_score) self.assertTrue(metrics['auc'] > self.expected_score, 'KPI for insults is not satisfied. \ Got {}, expected more than {}'.format(metrics['auc'], self.expected_score))
def train_squad(project): create_dir('squad') if project.get_property('idle_train') == 'True': val_time = '600' time_limit = '900' else: val_time = '1800' time_limit = '86400' metrics = bu.model([ '-t', 'squad', '-m', 'deeppavlov.agents.squad.squad:SquadAgent', '--batchsize', '64', '--display-examples', 'False', '--num-epochs', '-1', '--max-train-time', time_limit, '--log-every-n-secs', '60', '--log-every-n-epochs', '-1', '--validation-every-n-secs', val_time, '--validation-every-n-epochs', '-1', '--chosen-metrics', 'f1', '--validation-patience', '5', '--type', 'fastqa_default', '--lr', '0.001', '--lr_drop', '0.3', '--linear_dropout', '0.25', '--embedding_dropout', '0.5', '--rnn_dropout', '0.25', '--recurrent_dropout', '0.0', '--input_dropout', '0.0', '--output_dropout', '0.0', '--context_enc_layers', '1', '--question_enc_layers', '1', '--encoder_hidden_dim', '300', '--projection_dim', '300', '--pointer_dim', '300', '--model-file', './build/squad/squad1', '--embedding_file', './build/squad/glove.840B.300d.txt' ]) return metrics
def test_KPI(self): metrics = bu.model(['-t', 'squad', '-m', 'deeppavlov.agents.squad.squad:SquadAgent', '--batchsize', '64', '--display-examples', 'False', '--num-epochs', '-1', '--log-every-n-secs', '60', '--log-every-n-epochs', '-1', '--validation-every-n-secs', '1800', '--validation-every-n-epochs', '-1', '--chosen-metrics', 'f1', '--validation-patience', '5', '--type', 'fastqa_default', '--linear_dropout', '0.0', '--embedding_dropout', '0.5', '--rnn_dropout', '0.0', '--recurrent_dropout', '0.0', '--input_dropout', '0.0', '--output_dropout', '0.0', '--context_enc_layers', '1', '--question_enc_layers', '1', '--encoder_hidden_dim', '300', '--projection_dim', '300', '--pointer_dim', '300', '--model-file', './build/squad/squad1', '--embedding_file', './build/squad/glove.840B.300d.txt', '--pretrained_model', './build/squad/squad1', '--datatype', 'test' ]) TestModels.report_score("SQuAD", metrics["f1"], self.expected_score) self.assertTrue(metrics['f1'] > self.expected_score, 'KPI for SQuAD is not satisfied. \ Got {}, expected more than {}'.format(metrics['f1'], self.expected_score))
def train_ner(project): create_dir('ner') metrics = bu.model([ '-t', 'deeppavlov.tasks.ner.agents', '-m', 'deeppavlov.agents.ner.ner:NERAgent', '-mf', './build/ner', '-dt', 'train:ordered', '--dict-file', './build/ner/dict', '--learning_rate', '0.01', '--batchsize', '2', '--display-examples', 'False', '--validation-every-n-epochs', '5', '--log-every-n-epochs', '1', '--log-every-n-secs', '-1', '--chosen-metrics', 'f1' ]) return metrics
def train_coreference_scorer_model(project): create_dir('coref') metrics = bu.model([ '-t', 'deeppavlov.tasks.coreference_scorer_model.agents:CoreferenceTeacher', '-m', 'deeppavlov.agents.coreference_scorer_model.agents:CoreferenceAgent', '--display-examples', 'False', '--num-epochs', '20', '--log-every-n-secs', '-1', '--log-every-n-epochs', '1', '--validation-every-n-epochs', '1', '--chosen-metrics', 'f1', '--validation-patience', '5', '--model-file', './build/coref', '--embeddings_path', './build/coref/fasttext_embdgs.bin' ]) return metrics
def test_ner(self): expected_KPI = 70 metrics = bu.model([ '-t', 'deeppavlov.tasks.ner.agents', '-m', 'deeppavlov.agents.ner.ner:NERAgent', '-mf', './build/ner/ner', '-dt', 'test', '--batchsize', '2', '--display-examples', 'False', '--validation-every-n-epochs', '5', '--log-every-n-epochs', '1', '--log-every-n-secs', '-1', '--pretrained-model', './build/ner/ner', '--chosen-metrics', 'f1' ]) self.assertTrue( metrics['f1'] > expected_KPI, 'KPI for NER is not satisfied. \ Got {}, expected more than {}'.format( metrics['f1'], expected_KPI))
def train_coreference(project): create_dir('coreference') mf = './build/coreference/' compile_coreference(mf) metrics = bu.model([ '-t', 'deeppavlov.tasks.coreference.agents', '-m', 'deeppavlov.agents.coreference.agents:CoreferenceAgent', '-mf', mf, '--language', 'russian', '--name', 'main', '--pretrained_model', 'True', '-dt', 'train:ordered', '--batchsize', '1', '--display-examples', 'False', '--max-train-time', '-1', '--validation-every-n-epochs', '1000', '--nitr', '1000', '--log-every-n-epochs', '1', '--log-every-n-secs', '-1', '--chosen-metric', 'f1' ]) return metrics
def train_paraphraser(project): create_dir('paraphraser') metrics = bu.model([ '-t', 'deeppavlov.tasks.paraphrases.agents', '-m', 'deeppavlov.agents.paraphraser.paraphraser:ParaphraserAgent', '-mf', './build/paraphraser/paraphraser', '--datatype', 'train:ordered', '--batchsize', '256', '--display-examples', 'False', '--num-epochs', '-1', '--log-every-n-secs', '-1', '--log-every-n-epochs', '1', '--learning_rate', '0.0001', '--hidden_dim', '200', '--validation-every-n-epochs', '5', '--fasttext_embeddings_dict', './build/paraphraser/paraphraser.emb', '--fasttext_model', './build/paraphraser/ft_0.8.3_nltk_yalen_sg_300.bin', '--teacher-random-seed', '50', '--bagging-folds-number', '5', '--validation-patience', '3', '--chosen-metrics', 'f1' ]) return metrics
def test_paraphraser(self): expected_KPI = 0.8 metrics = bu.model([ '-t', 'deeppavlov.tasks.paraphrases.agents', '-m', 'deeppavlov.agents.paraphraser.paraphraser:EnsembleParaphraserAgent', '-mf', './build/paraphraser/paraphraser', '--model_files', './build/paraphraser/paraphraser', '--datatype', 'test', '--batchsize', '256', '--display-examples', 'False', '--fasttext_embeddings_dict', './build/paraphraser/paraphraser.emb', '--fasttext_model', './build/paraphraser/ft_0.8.3_nltk_yalen_sg_300.bin', '--bagging-folds-number', '5', '--chosen-metrics', 'f1' ]) self.assertTrue( metrics['f1'] > expected_KPI, 'KPI for paraphraser is not satisfied. \ Got {}, expected more than {}'.format( metrics['f1'], expected_KPI))
def train_coreference(project): create_dir('coreference') mf = './build/coreference/' compile_coreference(mf) num_epochs = '20' if project.get_property( 'idle_train') == 'True' else '500' metrics = bu.model([ '-t', 'deeppavlov.tasks.coreference.agents', '-m', 'deeppavlov.agents.coreference.agents:CoreferenceAgent', '-mf', mf, '--language', 'russian', '--name', 'gold_main', '--pretrained_model', 'False', '-dt', 'train:ordered', '--batchsize', '1', '--display-examples', 'False', '--num-epochs', num_epochs, '--validation-every-n-epochs', '10', '--nitr', '500', '--log-every-n-epochs', '1', '--log-every-n-secs', '-1', '--chosen-metric', 'conll-F-1', '--validation-patience', '15', '--train_on_gold', 'True', '--random_seed', '5', '--emb_format', 'vec', '--embedding_size', '100' ]) return metrics
def train_insults(project): create_dir('insults') metrics = bu.model([ '-t', 'deeppavlov.tasks.insults.agents', '-m', 'deeppavlov.agents.insults.insults_agents:InsultsAgent', '--model_file', './build/insults/cnn_word', '-dt', 'train:ordered', '--model_name', 'cnn_word', '--log-every-n-secs', '60', '--raw-dataset-path', './build/insults/', '--batchsize', '64', '--display-examples', 'False', '--num-epochs', '1000', '--max_sequence_length', '100', '--learning_rate', '0.01', '--learning_decay', '0.1', '--filters_cnn', '256', '--embedding_dim', '100', '--kernel_sizes_cnn', '1 2 3', '--regul_coef_conv', '0.001', '--regul_coef_dense', '0.01', '--dropout_rate', '0.5', '--dense_dim', '100', '--fasttext_model', './build/insults/reddit_fasttext_model.bin', '--fasttext_embeddings_dict', './build/insults/emb_dict.emb', '--bagging-folds-number', '3', '-ve', '10', '-vp', '5', '--chosen-metric', 'auc' ]) return metrics
def test_KPI(self): metrics = bu.model(['-t', 'deeppavlov.tasks.ner.agents', '-m', 'deeppavlov.agents.ner.ner:NERAgent', '-mf', './build/ner', '-dt', 'test', '--dict-file', './build/ner/dict', '--batchsize', '2', '--display-examples', 'False', '--validation-every-n-epochs', '5', '--log-every-n-epochs', '1', '--log-every-n-secs', '-1', '--pretrained-model', './build/ner', '--chosen-metrics', 'f1' ]) TestModels.report_score("ner", metrics["f1"], self.expected_score) self.assertTrue(metrics['f1'] > self.expected_score, 'KPI for NER is not satisfied. \ Got {}, expected more than {}'.format(metrics['f1'], self.expected_score))
def test_KPI(self): metrics = bu.model(['-t', 'deeppavlov.tasks.coreference_scorer_model.agents:CoreferenceTeacher', '-m', 'deeppavlov.agents.coreference_scorer_model.agents:CoreferenceAgent', '--display-examples', 'False', '--num-epochs', '-1', '--log-every-n-secs', '-1', '--log-every-n-epochs', '1', '--validation-every-n-epochs', '-1', '--chosen-metrics', 'f1', '--datatype', 'test', '--model-file', './build/coref', '--pretrained_model', './build/coref', '--embeddings_path', './build/coref/fasttext_embdgs.bin', ]) TestModels.report_score("Coreference Scorer Model", metrics["f1"], self.expected_score) self.assertTrue(metrics['f1'] > self.expected_score, 'KPI for Coreference resolution is not satisfied. \ Got {}, expected more than {}'.format(metrics['f1'], self.expected_score))
def test_KPI(self): metrics = bu.model(['-t', 'deeppavlov.tasks.coreference.agents', '-m', 'deeppavlov.agents.coreference.agents:CoreferenceAgent', '-mf', './build/coreference/', '--language', 'russian', '--name', 'gold_main', '--pretrained_model', 'True', '--datatype', 'test:stream', '--batchsize', '1', '--display-examples', 'False', '--chosen-metric', 'conll-F-1', '--train_on_gold', 'True', '--random_seed', '5' ]) TestModels.report_score("Coreference", metrics["conll-F-1"], self.expected_score) self.assertTrue(metrics['conll-F-1'] > self.expected_score, 'KPI for Coreference resolution is not satisfied. \ Got {}, expected more than {}'.format(metrics['conll-F-1'], self.expected_score))