コード例 #1
0
 def interview(self):
     model = BLSTMModel.load_model("../健康管理师分字BERT-model")
     x_items, train_y = self.read_message('../data/健康管理师分类数据集/test.txt')
     model.evaluate(x_items, train_y)
     results_string: str = ''
     train_string: str = ''
     for i in x_items:
         results_string += model.predict(i)
     for j in train_y:
         train_string += j
     if len(results_string) == len(train_string):
         print('预测结果', results_string, '正确结果', train_string)
         print('五个五个去判断 全等就是做对了不全等就是做错了')
         a = len(train_string)
         b: float = a / 5
         right: int = 0
         for i in range(int(b)):
             var = results_string[5 * i:5 * (i + 1)]
             result = train_string[5 * i:5 * (i + 1)]
             if var == result:
                 print('做对了')
                 right += 1
             else:
                 print('做错了', var, result)
         acc = b - right
         print('正确答案', right, '错误答案', acc)
         print('准确率', right / b)
コード例 #2
0
 def interview(self):
     model = BLSTMModel.load_model("../西药执业药师-model")
     x_items, train_y = self.read_message('../data/西药执业药师/test.txt')
     model.evaluate(x_items, train_y)
     results_string: str = ''
     train_string: str = ''
     for i in x_items:
         results_string += model.predict(i)
     for j in train_y:
         train_string += j
     if len(results_string) == len(train_string):
         print('五个五个去判断 全等就是做对了不全等就是做错了')
         a = len(train_string)
         b: int = int(a / 5)
         print('验证数据集长度', b)
         right: int = 0
         for i in range(int(b)):
             single = x_items[b * 5:b * 5 + 1]
             var = results_string[5 * i:5 * (i + 1)]
             result = train_string[5 * i:5 * (i + 1)]
             if var == result:
                 print('做对了', single)
                 right += 1
             else:
                 print('做错了', var, result, single)
         acc = b - right
         print('正确答案', right, '错误答案', acc)
         print('准确率', right / b)
コード例 #3
0
 def test_save_and_load(self):
     self.test_fit()
     model_path = tempfile.gettempdir()
     self.model.save(model_path)
     new_model = BLSTMModel.load_model(model_path)
     self.assertIsNotNone(new_model)
     sentence = list('语言学包含了几种分支领域。')
     result = new_model.predict(sentence)
     self.assertTrue(isinstance(result, str))
コード例 #4
0
 def test_save_and_load(self):
     self.test_fit()
     model_path = os.path.join(tempfile.gettempdir(), 'kashgari_model',
                               str(time.time()))
     self.model.save(model_path)
     new_model = BLSTMModel.load_model(model_path)
     assert new_model is not None
     sentence = list('语言学包含了几种分支领域。')
     result = new_model.predict(sentence)
     assert isinstance(result, str)
コード例 #5
0
    def interview(self):
        model = BLSTMModel.load_model("../健康管理师单选分字BERT-model")
        x_items, train_y = self.read_message('../data/yingyangshi/test.txt')
        x_full = self.full_message('../data/yingyangshi/test.txt')
        model.evaluate(x_items, train_y)
        results_string: str = ''
        train_string: str = ''
        right_predict: list = []
        wrong_predict: list = []
        for i in x_items:
            results_string += model.predict(i)
        for j in train_y:
            train_string += j
        if len(results_string) == len(train_string):
            print('预测结果', results_string, '正确结果', train_string)
            print('五个五个去判断 全等就是做对了不全等就是做错了')
            a = len(train_string)
            b: int = int(a / 5)
            print('验证数据集长度', b)
            right: int = 0
            for i in range(b):
                start_single: int = i * 5
                end_single: int = (i + 1) * 5
                single = x_full[start_single:end_single]
                var = results_string[start_single:end_single]
                result = train_string[start_single:end_single]
                if var == result:
                    print('做对了')
                    right_predict.append(single)
                    for varey in single:
                        print(varey)

                    right += 1
                else:
                    print('做错了', var, result)
                    wrong_predict.append(single)
                    for varey in single:
                        print(varey)
            acc = b - right
            with open('wrong single.csv', 'w', newline='',
                      encoding='utf-8') as csv_file:
                csv_writer = csv.writer(csv_file)
                for wrong_list in wrong_predict:
                    for message in wrong_list:
                        wrong_list = message.split('\t')
                        csv_writer.writerow(wrong_list)
            with open('right single.csv', 'w', newline='',
                      encoding='utf-8') as csv_file:
                csv_writer = csv.writer(csv_file)
                for right_list in right_predict:
                    for message in right_list:
                        message = message.split('\t')
                        csv_writer.writerow(message)
            print('正确答案', right, '错误答案', acc)
            print('准确率', right / b)
コード例 #6
0
ファイル: 3_build_model.py プロジェクト: t3ls/news-recommand
def test_dataset(model_dir: str) -> list:
	# 从数据库中获取正文并使用模型进行预测分类,
	# 预测结果写回数据库
	conn = pymysql.connect(host=DB_HOST,
                        port=int(DB_PORT),
                        user=DB_USER,
                        password=DB_PASS,
                        db=DB_NAME,
                        charset=DB_CHARSET
                        )
	cursor = conn.cursor()
	cursor.execute("""
		SELECT `page_text`,`page_title`,`category`,`hash` FROM `webpage_text`
		WHERE `%s_predict` IS NULL ORDER BY `time` desc
		""" % model_dir.split('.model')[0].split('/')[-1] 
		)
	all_text = []
	data = cursor.fetchall()
	# 判断预测使用的模型
	if 'cnn.model' in model_dir:
		model = CNNModel.load_model(model_dir)
	elif 'cnnlstm.model' in model_dir:
		model = CNNLSTMModel.load_model(model_dir)
	elif 'blstm.model' in model_dir:
		model = BLSTMModel.load_model(model_dir)
	for i in tqdm.tqdm(data):
		label = i[2]
		# 将文章分词,拼接标题与正文
		content = strip_stopwords(list(jieba.cut(i[0] + '。' + i[1])))
		all_text += content
		predict = model.predict(content)
		cursor.execute(
			'UPDATE `webpage_text` SET {model}_predict="{predict}"'.format(model=model_dir.split('.model')[0].split('/')[-1],predict=predict)+
			'WHERE hash="%s"' % i[3]
			)
		conn.commit()
		# print('[+] Predict:'+predict+', Label:'+label+', Title:'+i[1])

	# 计算词频并将排行前100的热词写入数据库
	c = Counter(all_text)
	i = 1
	cursor.execute(
		'DELETE FROM `hot_key` WHERE 1=1'
		)
	conn.commit()
	for k,v in c.most_common(100):
		if len(k) == 1:
			continue
		cursor.execute(
			'INSERT INTO `hot_key` VALUES ({0}, "{1}", {2})'.format(i, k, v)
			)
		conn.commit()
		i += 1
	print('[+] Success')
コード例 #7
0
    def test_multi_label_model(self):
        multi_label_model = self.model_class(multi_label=True)
        multi_label_model.fit(train_x,
                              train_multi_y,
                              eval_x,
                              eval_multi_y,
                              epochs=2)
        assert isinstance(multi_label_model.predict(train_x[0]), tuple)

        model_path = os.path.join(tempfile.gettempdir(), 'kashgari_model',
                                  str(time.time()))
        multi_label_model.save(model_path)
        new_model = BLSTMModel.load_model(model_path)
        assert new_model is not None
        sentence = list('语言学包含了几种分支领域。')
        result = new_model.predict(sentence)
        assert isinstance(result, tuple)
コード例 #8
0
 def interview(self):
     model = BLSTMModel.load_model(
         "../model/health_manager_multi_bert-model")
     x_items, train_y = read_message(
         '../data/health_manager_v2/multiple-choice.csv')
     model.evaluate(x_items, train_y)
     results_string: str = ''
     train_string: str = ''
     right_predict: list = []
     wrong_predict: list = []
     for i in x_items:
         results_string += model.predict(i)
     for j in train_y:
         train_string += j
     if len(results_string) == len(train_string):
         print('预测结果', results_string, '正确结果', train_string)
         print('五个五个去判断 全等就是做对了不全等就是做错了')
         a = len(train_string)
         b: int = int(a / 5)
         print('验证数据集长度', b)
         right: int = 0
         for i in range(b):
             start_single: int = i * 5
             end_single: int = (i + 1) * 5
             var = results_string[start_single:end_single]
             result = train_string[start_single:end_single]
             if var == result:
                 print('做对了')
         acc = b - right
         with open('wrong single.csv', 'w', newline='',
                   encoding='utf-8') as csv_file:
             csv_writer = csv.writer(csv_file)
             for wrong_list in wrong_predict:
                 for message in wrong_list:
                     wrong_list = message.split('\t')
                     csv_writer.writerow(wrong_list)
         with open('right single.csv', 'w', newline='',
                   encoding='utf-8') as csv_file:
             csv_writer = csv.writer(csv_file)
             for right_list in right_predict:
                 for message in right_list:
                     message = message.split('\t')
                     csv_writer.writerow(message)
         print('正确答案', right, '错误答案', acc)
         print('准确率', right / b)
コード例 #9
0
def predict_each_line(args, model):
    import codecs
    fout = codecs.open(args.output_file, 'w')
    test_x, test_y = fetch_data_set(args.test_set_path)
    for line, y in zip(test_x, test_y):
       result = model.predict(text_processor(''.join(line)), batch_size=1, debug_info=False)
       if result != ''.join(y):
           str_message = ''.join(line) + "\t" + ''.join(y) +"\t" + result
           print(str_message)
           fout.write(str_message+'\n')
    fout.close()

if __name__ == '__main__':
    # initialize parameter
    args = params_setup()
    logging.basicConfig(filename=args.log_path, level=logging.DEBUG)

    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id

    bert_embedding = BERTEmbedding('bert-base-chinese', sequence_length=30)

    model = BLSTMModel(bert_embedding)
    model = model.load_model(args.model_path)

    if (args.predict_mode == "from_input"):
        predict_from_user_input(model)
    else:
        predict_from_test_set(args, model)

 def pre_train(self):
     model = BLSTMModel.load_model("../model/中医执业药师classification-model")
     x_items, train_y = self.read_message('../data/Chinese medicine licensed pharmacist/test.txt')
     model.evaluate(x_items, train_y)
コード例 #11
0
 def pre_evaluate(self):
     model = BLSTMModel.load_model("../健康管理师分字-model")
     x_items, train_y = self.read_message(
         '../data/health_manager_v4/test.txt')
     model.evaluate(x_items, train_y)
コード例 #12
0
 def pre_evaluate(self):
     model = BLSTMModel.load_model(
         "../model/health_manager_multi_bert-model")
     result = model.predict("")
コード例 #13
0
 def pre_train(self):
     bilstm_model = BLSTMModel.load_model('../classification-model')
     x_items, _ = self.read_message('../data/西药执业药师/dev.txt')
     for i in x_items:
         result = bilstm_model.predict(i)
         print("\n" + result)