async def do_get_answer_api(question: str):
if not question:
return {'status': False, 'msg': 'Please enter the query.'}
if question:
try:
# user_id = 'qa_' + user_id
conn = connect_postgres_server()
cursor = conn.cursor()
results = get_result(question, conn, cursor)
if results:
return {'status': True, 'msg': results[0][0]}
else:
return {
'status': False,
'msg':
'There is no answer to this question in the database'
}
except Exception as e:
print('get answer faild: ', e)
return {
'status': False,
'msg': 'Failed to search, please try again.'
}
finally:
cursor.close()
conn.close()
return {'status': False, 'msg': 'Failed to search, please try again.'}
def import_to_pg(table_name,ids,answer_file):
conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER, PG_PASSWORD, PG_DATABASE)
cur = conn.cursor()
pg_operating.create_pg_table(conn, cur, table_name)
pg_operating.record_txt(ids,answer_file)
pg_operating.copy_data_to_pg(conn, cur, table_name)
pg_operating.build_pg_index(conn, cur, table_name)
async def do_get_question_api(question: str):
if not question:
return {'status': False, 'msg': 'Please enter the query.'}
if question:
try:
# user_id = 'qa_' + user_id
conn = connect_postgres_server()
cursor = conn.cursor()
client = milvus_client()
bc = BertClient(ip=BERT_HOST, port=BERT_PORT, check_length=False)
output = get_similar_question(question, client, conn, cursor, bc)
if output:
return {'status': True, 'msg': output}
else:
return {
'status': False,
'msg': 'No similar questions in the database'
}
except Exception as e:
print('search faild: ', e)
return {
'status': False,
'msg': 'Failed to search, please try again.'
}
finally:
cursor.close()
conn.close()
bc.close()
return {'status': False, 'msg': 'Failed to search, please try again.'}
async def do_load_api(file: UploadFile = File(...)):
try:
text = await file.read()
fname = file.filename
dirs = "QA_data/"
if not os.path.exists(dirs):
os.makedirs(dirs)
fname_path = dirs + "/" + fname
with open(fname_path, 'wb') as f:
f.write(text)
except Exception as e:
return {'status': False, 'msg': 'Failed to load data.'}
try:
conn = connect_postgres_server()
cursor = conn.cursor()
client = milvus_client()
bc = BertClient(ip=BERT_HOST, port=BERT_PORT, check_length=False)
status, message = load_data(fname_path, client, conn, cursor, bc)
return {'status': status, 'msg': message}
except Exception as e:
print("load data faild: ", e)
return {'status': False, 'msg': 'Failed to load data.'}
finally:
cursor.close()
conn.close()
bc.close()
def search_in_milvus(collection_name, query_sentence, bc):
logging.info("start test process ...")
query_data = [query_sentence]
try:
vectors = bc.encode(query_data)
logging.info("get query vector!")
except Exception as e:
info = "bert Error: " + e
logging.info(info)
return info
query_list = normaliz_vec(vectors.tolist())
#connect_milvus_server()
try:
milvus = Milvus(host=MILVUS_HOST, port=MILVUS_PORT)
#logging.info(status)
except Exception as e:
info = "Milvus connect error: " + e
logging.info(info)
return info
try:
logging.info("start search in milvus...")
search_params = {'nprobe': 64}
status, results = milvus.search(collection_name=collection_name,
query_records=query_list,
top_k=1,
params=search_params)
if not results:
return "there is no data in milvus"
if results[0][0].distance < 0.9:
return "对不起，我暂时无法为您解答该问题"
except Exception as e:
info = "Milvus search error: " + e
logging.info(info)
return info
try:
conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER,
PG_PASSWORD, PG_DATABASE)
cur = conn.cursor()
except Exception as e:
info = "postgres service connection failed" + e
logging.info(info)
return info
try:
logging.info("start search in pg ...")
rows = pg_operating.search_in_pg(conn, cur, results[0][0].id,
collection_name)
out_put = rows[0][1]
return out_put
except Exception as e:
info = "dearch in postgres error: " + e
logging.info(info)
return info
finally:
conn.close()
def search_in_milvus(collection_name, query_sentence):
logging.info("start test process ...")
query_data = [query_sentence]
try:
vectors = bc.encode(query_data)
except:
return "bert service disconnect"
query_list = normaliz_vec(vectors.tolist())
#connect_milvus_server()
try:
milvus = Milvus(host=MILVUS_HOST, port=MILVUS_PORT)
#logging.info(status)
except:
return "milvus service connection failed"
try:
logging.info("start search in milvus...")
search_params = {'nprobe': 64}
status, results = milvus.search(collection_name=collection_name,
query_records=query_list,
top_k=9,
params=search_params)
if results[0][0].distance < 0.8:
return "对不起，无法搜索到相似文本"
except:
return "milvus service disconnect"
try:
conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER,
PG_PASSWORD, PG_DATABASE)
cur = conn.cursor()
except:
return "postgres service connection failed"
try:
logging.info("start search in pg ...")
out_put = []
# print(results)
for a in range(9):
rows = pg_operating.search_in_pg(conn, cur, results[0][a].id,
collection_name)
if rows and len(rows[0]) > 0:
#print(rows)
out_put.append(rows[0][1:])
# print(out_put)
return out_put
except:
return "postgres service disconnect"
finally:
conn.close()
def search_in_milvus(collection_name, query_sentence):
logging.info("start test process ...")
query_data = [query_sentence]
try:
vectors = bc.encode(query_data)
except:
return "bert service disconnect"
query_list = normaliz_vec(vectors.tolist())
#connect_milvus_server()
try:
milvus = Milvus(host=MILVUS_HOST, port=MILVUS_PORT)
#logging.info(status)
except:
return "milvus service connection failed"
try:
logging.info("start search in milvus...")
search_params = {'nprobe': 64}
status, results = milvus.search(collection_name=collection_name,
query_records=query_list,
top_k=1,
params=search_params)
if results[0][0].distance < 0.9:
return "对不起，我暂时无法为您解答该问题"
except:
return "milvus service disconnect"
try:
conn = pg_operating.connect_postgres_server(PG_HOST, PG_PORT, PG_USER,
PG_PASSWORD, PG_DATABASE)
cur = conn.cursor()
except:
return "postgres service connection failed"
try:
logging.info("start search in pg ...")
rows = pg_operating.search_in_pg(conn, cur, results[0][0].id,
collection_name)
out_put = rows[0][1]
return out_put
except:
return "postgres service disconnect"
finally:
if milvus:
milvus.disconnect()
conn.close()
