def test_KPI(self):
metrics = bu.model(['-t', 'deeppavlov.tasks.insults.agents:FullTeacher',
'-m', 'deeppavlov.agents.insults.insults_agents:EnsembleInsultsAgent',
'--model_file', './build/insults/insults_ensemble',
'--model_files', './build/insults/cnn_word_0',
'./build/insults/cnn_word_1',
'./build/insults/cnn_word_2',
'--model_names', 'cnn_word',
'cnn_word', 'cnn_word',
'--model_coefs', '0.3333333',
'0.3333333', '0.3333334',
'--datatype', 'test',
'--batchsize', '64',
'--display-examples', 'False',
'--raw-dataset-path', './build/insults/',
'--max_sequence_length', '100',
'--filters_cnn', '256',
'--kernel_sizes_cnn', '1 2 3',
'--embedding_dim', '100',
'--dense_dim', '100',
'--fasttext_model', './build/insults/reddit_fasttext_model.bin'
])
TestModels.report_score("insults", metrics["auc"], self.expected_score)
self.assertTrue(metrics['auc'] > self.expected_score,
'KPI for insults is not satisfied. \
Got {}, expected more than {}'.format(metrics['auc'], self.expected_score))
def train_squad(project):
create_dir('squad')
if project.get_property('idle_train') == 'True':
val_time = '600'
time_limit = '900'
else:
val_time = '1800'
time_limit = '86400'
metrics = bu.model([
'-t', 'squad', '-m', 'deeppavlov.agents.squad.squad:SquadAgent',
'--batchsize', '64', '--display-examples', 'False', '--num-epochs',
'-1', '--max-train-time', time_limit, '--log-every-n-secs', '60',
'--log-every-n-epochs', '-1', '--validation-every-n-secs', val_time,
'--validation-every-n-epochs', '-1', '--chosen-metrics', 'f1',
'--validation-patience', '5', '--type', 'fastqa_default', '--lr',
'0.001', '--lr_drop', '0.3', '--linear_dropout', '0.25',
'--embedding_dropout', '0.5', '--rnn_dropout', '0.25',
'--recurrent_dropout', '0.0', '--input_dropout', '0.0',
'--output_dropout', '0.0', '--context_enc_layers', '1',
'--question_enc_layers', '1', '--encoder_hidden_dim', '300',
'--projection_dim', '300', '--pointer_dim', '300', '--model-file',
'./build/squad/squad1', '--embedding_file',
'./build/squad/glove.840B.300d.txt'
])
return metrics
def test_KPI(self):
metrics = bu.model(['-t', 'squad',
'-m', 'deeppavlov.agents.squad.squad:SquadAgent',
'--batchsize', '64',
'--display-examples', 'False',
'--num-epochs', '-1',
'--log-every-n-secs', '60',
'--log-every-n-epochs', '-1',
'--validation-every-n-secs', '1800',
'--validation-every-n-epochs', '-1',
'--chosen-metrics', 'f1',
'--validation-patience', '5',
'--type', 'fastqa_default',
'--linear_dropout', '0.0',
'--embedding_dropout', '0.5',
'--rnn_dropout', '0.0',
'--recurrent_dropout', '0.0',
'--input_dropout', '0.0',
'--output_dropout', '0.0',
'--context_enc_layers', '1',
'--question_enc_layers', '1',
'--encoder_hidden_dim', '300',
'--projection_dim', '300',
'--pointer_dim', '300',
'--model-file', './build/squad/squad1',
'--embedding_file', './build/squad/glove.840B.300d.txt',
'--pretrained_model', './build/squad/squad1',
'--datatype', 'test'
])
TestModels.report_score("SQuAD", metrics["f1"], self.expected_score)
self.assertTrue(metrics['f1'] > self.expected_score,
'KPI for SQuAD is not satisfied. \
Got {}, expected more than {}'.format(metrics['f1'], self.expected_score))
def train_ner(project):
create_dir('ner')
metrics = bu.model([
'-t', 'deeppavlov.tasks.ner.agents', '-m',
'deeppavlov.agents.ner.ner:NERAgent', '-mf', './build/ner', '-dt',
'train:ordered', '--dict-file', './build/ner/dict', '--learning_rate',
'0.01', '--batchsize', '2', '--display-examples', 'False',
'--validation-every-n-epochs', '5', '--log-every-n-epochs', '1',
'--log-every-n-secs', '-1', '--chosen-metrics', 'f1'
])
return metrics
def train_coreference_scorer_model(project):
create_dir('coref')
metrics = bu.model([
'-t',
'deeppavlov.tasks.coreference_scorer_model.agents:CoreferenceTeacher',
'-m',
'deeppavlov.agents.coreference_scorer_model.agents:CoreferenceAgent',
'--display-examples', 'False', '--num-epochs', '20',
'--log-every-n-secs', '-1', '--log-every-n-epochs', '1',
'--validation-every-n-epochs', '1', '--chosen-metrics', 'f1',
'--validation-patience', '5', '--model-file', './build/coref',
'--embeddings_path', './build/coref/fasttext_embdgs.bin'
])
return metrics
def test_ner(self):
expected_KPI = 70
metrics = bu.model([
'-t', 'deeppavlov.tasks.ner.agents', '-m',
'deeppavlov.agents.ner.ner:NERAgent', '-mf', './build/ner/ner',
'-dt', 'test', '--batchsize', '2', '--display-examples', 'False',
'--validation-every-n-epochs', '5', '--log-every-n-epochs', '1',
'--log-every-n-secs', '-1', '--pretrained-model',
'./build/ner/ner', '--chosen-metrics', 'f1'
])
self.assertTrue(
metrics['f1'] > expected_KPI, 'KPI for NER is not satisfied. \
Got {}, expected more than {}'.format(
metrics['f1'], expected_KPI))
def train_coreference(project):
create_dir('coreference')
mf = './build/coreference/'
compile_coreference(mf)
metrics = bu.model([
'-t', 'deeppavlov.tasks.coreference.agents', '-m',
'deeppavlov.agents.coreference.agents:CoreferenceAgent', '-mf', mf,
'--language', 'russian', '--name', 'main', '--pretrained_model',
'True', '-dt', 'train:ordered', '--batchsize', '1',
'--display-examples', 'False', '--max-train-time', '-1',
'--validation-every-n-epochs', '1000', '--nitr', '1000',
'--log-every-n-epochs', '1', '--log-every-n-secs', '-1',
'--chosen-metric', 'f1'
])
return metrics
def train_paraphraser(project):
create_dir('paraphraser')
metrics = bu.model([
'-t', 'deeppavlov.tasks.paraphrases.agents', '-m',
'deeppavlov.agents.paraphraser.paraphraser:ParaphraserAgent', '-mf',
'./build/paraphraser/paraphraser', '--datatype', 'train:ordered',
'--batchsize', '256', '--display-examples', 'False', '--num-epochs',
'-1', '--log-every-n-secs', '-1', '--log-every-n-epochs', '1',
'--learning_rate', '0.0001', '--hidden_dim', '200',
'--validation-every-n-epochs', '5', '--fasttext_embeddings_dict',
'./build/paraphraser/paraphraser.emb', '--fasttext_model',
'./build/paraphraser/ft_0.8.3_nltk_yalen_sg_300.bin',
'--teacher-random-seed', '50', '--bagging-folds-number', '5',
'--validation-patience', '3', '--chosen-metrics', 'f1'
])
return metrics
def test_paraphraser(self):
expected_KPI = 0.8
metrics = bu.model([
'-t', 'deeppavlov.tasks.paraphrases.agents', '-m',
'deeppavlov.agents.paraphraser.paraphraser:EnsembleParaphraserAgent',
'-mf', './build/paraphraser/paraphraser', '--model_files',
'./build/paraphraser/paraphraser', '--datatype', 'test',
'--batchsize', '256', '--display-examples', 'False',
'--fasttext_embeddings_dict',
'./build/paraphraser/paraphraser.emb', '--fasttext_model',
'./build/paraphraser/ft_0.8.3_nltk_yalen_sg_300.bin',
'--bagging-folds-number', '5', '--chosen-metrics', 'f1'
])
self.assertTrue(
metrics['f1'] > expected_KPI,
'KPI for paraphraser is not satisfied. \
Got {}, expected more than {}'.format(
metrics['f1'], expected_KPI))
def train_coreference(project):
create_dir('coreference')
mf = './build/coreference/'
compile_coreference(mf)
num_epochs = '20' if project.get_property(
'idle_train') == 'True' else '500'
metrics = bu.model([
'-t', 'deeppavlov.tasks.coreference.agents', '-m',
'deeppavlov.agents.coreference.agents:CoreferenceAgent', '-mf', mf,
'--language', 'russian', '--name', 'gold_main', '--pretrained_model',
'False', '-dt', 'train:ordered', '--batchsize', '1',
'--display-examples', 'False', '--num-epochs', num_epochs,
'--validation-every-n-epochs', '10', '--nitr', '500',
'--log-every-n-epochs', '1', '--log-every-n-secs', '-1',
'--chosen-metric', 'conll-F-1', '--validation-patience', '15',
'--train_on_gold', 'True', '--random_seed', '5', '--emb_format', 'vec',
'--embedding_size', '100'
])
return metrics
def train_insults(project):
create_dir('insults')
metrics = bu.model([
'-t', 'deeppavlov.tasks.insults.agents', '-m',
'deeppavlov.agents.insults.insults_agents:InsultsAgent',
'--model_file', './build/insults/cnn_word', '-dt', 'train:ordered',
'--model_name', 'cnn_word', '--log-every-n-secs', '60',
'--raw-dataset-path', './build/insults/', '--batchsize', '64',
'--display-examples', 'False', '--num-epochs', '1000',
'--max_sequence_length', '100', '--learning_rate', '0.01',
'--learning_decay', '0.1', '--filters_cnn', '256', '--embedding_dim',
'100', '--kernel_sizes_cnn', '1 2 3', '--regul_coef_conv', '0.001',
'--regul_coef_dense', '0.01', '--dropout_rate', '0.5', '--dense_dim',
'100', '--fasttext_model', './build/insults/reddit_fasttext_model.bin',
'--fasttext_embeddings_dict', './build/insults/emb_dict.emb',
'--bagging-folds-number', '3', '-ve', '10', '-vp', '5',
'--chosen-metric', 'auc'
])
return metrics
def test_KPI(self):
metrics = bu.model(['-t', 'deeppavlov.tasks.ner.agents',
'-m', 'deeppavlov.agents.ner.ner:NERAgent',
'-mf', './build/ner',
'-dt', 'test',
'--dict-file', './build/ner/dict',
'--batchsize', '2',
'--display-examples', 'False',
'--validation-every-n-epochs', '5',
'--log-every-n-epochs', '1',
'--log-every-n-secs', '-1',
'--pretrained-model', './build/ner',
'--chosen-metrics', 'f1'
])
TestModels.report_score("ner", metrics["f1"], self.expected_score)
self.assertTrue(metrics['f1'] > self.expected_score,
'KPI for NER is not satisfied. \
Got {}, expected more than {}'.format(metrics['f1'], self.expected_score))
def test_KPI(self):
metrics = bu.model(['-t', 'deeppavlov.tasks.coreference_scorer_model.agents:CoreferenceTeacher',
'-m', 'deeppavlov.agents.coreference_scorer_model.agents:CoreferenceAgent',
'--display-examples', 'False',
'--num-epochs', '-1',
'--log-every-n-secs', '-1',
'--log-every-n-epochs', '1',
'--validation-every-n-epochs', '-1',
'--chosen-metrics', 'f1',
'--datatype', 'test',
'--model-file', './build/coref',
'--pretrained_model', './build/coref',
'--embeddings_path', './build/coref/fasttext_embdgs.bin',
])
TestModels.report_score("Coreference Scorer Model", metrics["f1"], self.expected_score)
self.assertTrue(metrics['f1'] > self.expected_score,
'KPI for Coreference resolution is not satisfied. \
Got {}, expected more than {}'.format(metrics['f1'], self.expected_score))
def test_KPI(self):
metrics = bu.model(['-t', 'deeppavlov.tasks.coreference.agents',
'-m', 'deeppavlov.agents.coreference.agents:CoreferenceAgent',
'-mf', './build/coreference/',
'--language', 'russian',
'--name', 'gold_main',
'--pretrained_model', 'True',
'--datatype', 'test:stream',
'--batchsize', '1',
'--display-examples', 'False',
'--chosen-metric', 'conll-F-1',
'--train_on_gold', 'True',
'--random_seed', '5'
])
TestModels.report_score("Coreference", metrics["conll-F-1"], self.expected_score)
self.assertTrue(metrics['conll-F-1'] > self.expected_score,
'KPI for Coreference resolution is not satisfied. \
Got {}, expected more than {}'.format(metrics['conll-F-1'], self.expected_score))
