async def do_get_answer_api(question: str): if not question: return {'status': False, 'msg': 'Please enter the query.'} if question: try: # user_id = 'qa_' + user_id conn = connect_postgres_server() cursor = conn.cursor() results = get_result(question, conn, cursor) if results: return {'status': True, 'msg': results[0][0]} else: return { 'status': False, 'msg': 'There is no answer to this question in the database' } except Exception as e: print('get answer faild: ', e) return { 'status': False, 'msg': 'Failed to search, please try again.' } finally: cursor.close() conn.close() return {'status': False, 'msg': 'Failed to search, please try again.'}
def import_to_pg(table_name,ids,answer_file): conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER, PG_PASSWORD, PG_DATABASE) cur = conn.cursor() pg_operating.create_pg_table(conn, cur, table_name) pg_operating.record_txt(ids,answer_file) pg_operating.copy_data_to_pg(conn, cur, table_name) pg_operating.build_pg_index(conn, cur, table_name)
async def do_get_question_api(question: str): if not question: return {'status': False, 'msg': 'Please enter the query.'} if question: try: # user_id = 'qa_' + user_id conn = connect_postgres_server() cursor = conn.cursor() client = milvus_client() bc = BertClient(ip=BERT_HOST, port=BERT_PORT, check_length=False) output = get_similar_question(question, client, conn, cursor, bc) if output: return {'status': True, 'msg': output} else: return { 'status': False, 'msg': 'No similar questions in the database' } except Exception as e: print('search faild: ', e) return { 'status': False, 'msg': 'Failed to search, please try again.' } finally: cursor.close() conn.close() bc.close() return {'status': False, 'msg': 'Failed to search, please try again.'}
async def do_load_api(file: UploadFile = File(...)): try: text = await file.read() fname = file.filename dirs = "QA_data/" if not os.path.exists(dirs): os.makedirs(dirs) fname_path = dirs + "/" + fname with open(fname_path, 'wb') as f: f.write(text) except Exception as e: return {'status': False, 'msg': 'Failed to load data.'} try: conn = connect_postgres_server() cursor = conn.cursor() client = milvus_client() bc = BertClient(ip=BERT_HOST, port=BERT_PORT, check_length=False) status, message = load_data(fname_path, client, conn, cursor, bc) return {'status': status, 'msg': message} except Exception as e: print("load data faild: ", e) return {'status': False, 'msg': 'Failed to load data.'} finally: cursor.close() conn.close() bc.close()
def search_in_milvus(collection_name, query_sentence, bc): logging.info("start test process ...") query_data = [query_sentence] try: vectors = bc.encode(query_data) logging.info("get query vector!") except Exception as e: info = "bert Error: " + e logging.info(info) return info query_list = normaliz_vec(vectors.tolist()) #connect_milvus_server() try: milvus = Milvus(host=MILVUS_HOST, port=MILVUS_PORT) #logging.info(status) except Exception as e: info = "Milvus connect error: " + e logging.info(info) return info try: logging.info("start search in milvus...") search_params = {'nprobe': 64} status, results = milvus.search(collection_name=collection_name, query_records=query_list, top_k=1, params=search_params) if not results: return "there is no data in milvus" if results[0][0].distance < 0.9: return "对不起,我暂时无法为您解答该问题" except Exception as e: info = "Milvus search error: " + e logging.info(info) return info try: conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER, PG_PASSWORD, PG_DATABASE) cur = conn.cursor() except Exception as e: info = "postgres service connection failed" + e logging.info(info) return info try: logging.info("start search in pg ...") rows = pg_operating.search_in_pg(conn, cur, results[0][0].id, collection_name) out_put = rows[0][1] return out_put except Exception as e: info = "dearch in postgres error: " + e logging.info(info) return info finally: conn.close()
def search_in_milvus(collection_name, query_sentence): logging.info("start test process ...") query_data = [query_sentence] try: vectors = bc.encode(query_data) except: return "bert service disconnect" query_list = normaliz_vec(vectors.tolist()) #connect_milvus_server() try: milvus = Milvus(host=MILVUS_HOST, port=MILVUS_PORT) #logging.info(status) except: return "milvus service connection failed" try: logging.info("start search in milvus...") search_params = {'nprobe': 64} status, results = milvus.search(collection_name=collection_name, query_records=query_list, top_k=9, params=search_params) if results[0][0].distance < 0.8: return "对不起,无法搜索到相似文本" except: return "milvus service disconnect" try: conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER, PG_PASSWORD, PG_DATABASE) cur = conn.cursor() except: return "postgres service connection failed" try: logging.info("start search in pg ...") out_put = [] # print(results) for a in range(9): rows = pg_operating.search_in_pg(conn, cur, results[0][a].id, collection_name) if rows and len(rows[0]) > 0: #print(rows) out_put.append(rows[0][1:]) # print(out_put) return out_put except: return "postgres service disconnect" finally: conn.close()
def search_in_milvus(collection_name, query_sentence): logging.info("start test process ...") query_data = [query_sentence] try: vectors = bc.encode(query_data) except: return "bert service disconnect" query_list = normaliz_vec(vectors.tolist()) #connect_milvus_server() try: milvus = Milvus(host=MILVUS_HOST, port=MILVUS_PORT) #logging.info(status) except: return "milvus service connection failed" try: logging.info("start search in milvus...") search_params = {'nprobe': 64} status, results = milvus.search(collection_name=collection_name, query_records=query_list, top_k=1, params=search_params) if results[0][0].distance < 0.9: return "对不起,我暂时无法为您解答该问题" except: return "milvus service disconnect" try: conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER, PG_PASSWORD, PG_DATABASE) cur = conn.cursor() except: return "postgres service connection failed" try: logging.info("start search in pg ...") rows = pg_operating.search_in_pg(conn, cur, results[0][0].id, collection_name) out_put = rows[0][1] return out_put except: return "postgres service disconnect" finally: if milvus: milvus.disconnect() conn.close()