def create_answers(deck_id, query):
decks_collection = db["decks"]
slides_collection = db["slides"]
decks = decks_collection.find({"_id": deck_id})
target_deck = decks[0]
author_id = target_deck["user"]
author_slides = slides_collection.find({"user": author_id})
df = pd.DataFrame()
for slide in author_slides:
revision = slide["revisions"][-1]
usages = revision["usage"]
for usage in usages:
if usage["id"] == deck_id:
content = html2text.html2text(revision["content"])
paragraphs = content.split('\n\n')
df = df.append(
{
"date": revision["timestamp"],
"title": revision["title"],
"category": "Infromation",
"link": "",
"abstract": "",
"paragraphs": paragraphs,
"revision_id": revision["id"],
"slide_id": slide["_id"]
},
ignore_index=True)
break
download_model(model='bert-squad_1.1', dir='./models')
# df = filter_paragraphs(df)
cdqa_pipeline = QAPipeline(reader='models/bert_qa.joblib',
max_df=0.95,
min_df=3)
cdqa_pipeline.fit_retriever(df)
predictions = cdqa_pipeline.predict(
query, n_predictions=5) #retriever_score_weight=0.99
answers = []
i = 1
for prediction in predictions:
slide_id = df.loc[df["title"] == prediction[1]].iloc[0]['slide_id']
revision_id = df.loc[df["title"] ==
prediction[1]].iloc[0]['revision_id']
answers.append({
"slide_id": int(slide_id),
"revision_id": int(revision_id),
"answer": prediction[0],
"title": prediction[1],
"paragraph": prediction[2],
"score": prediction[3]
})
i += 1
return answers
def question(text, query):
print(text)
test = []
for i in sent_tokenize(text):
if len(i) > 2:
test.append(i)
n = 4
# using list comprehension
final = [test[i * n:(i + 1) * n] for i in range((len(test) + n - 1) // n)]
title_s = []
for j in range(len(final)):
title_s.append(f'Title{j}')
data = [title_s, final]
df3 = pd.DataFrame(data=data)
df3 = df3.transpose()
df3.columns = ['title', 'paragraphs']
print(df3)
#st.text('Hold on this will take some time')
from ast import literal_eval
from cdqa.utils.filters import filter_paragraphs
from cdqa.utils.download import download_model, download_bnpp_data
from cdqa.pipeline.cdqa_sklearn import QAPipeline
# Download data and models
#download_bnpp_data(dir='./data/bnpp_newsroom_v1.1/')
#download_model(model='bert-squad_1.1', dir='./models')
# Loading data and filtering / preprocessing the documents
df = pd.read_csv(
'D:/devjams/Machine-Learning-Web-Apps-master/NLPIffy_NLP_Based_SpaCy_Flask_App&_API/cdQA/data/bnpp_newsroom_v1.1/bnpp_newsroom-v1.1.csv',
converters={'paragraphs': literal_eval})
df = filter_paragraphs(df)
#st.text('Please Wait. We are looking for the answer to your question')
# Loading QAPipeline with CPU version of BERT Reader pretrained on SQuAD 1.1
cdqa_pipeline = QAPipeline(
reader=
'D:/devjams/Machine-Learning-Web-Apps-master/NLPIffy_NLP_Based_SpaCy_Flask_App&_API/bert_qa_vGPU-sklearn.joblib'
)
# Fitting the retriever to the list of documents in the dataframe
cdqa_pipeline.fit_retriever(df3)
print(query)
#st.text('Almost done.......')
#query = 'Intellectual Property Rights'
try:
prediction = cdqa_pipeline.predict(query)
except Exception as e:
print(e)
#st.text(prediction[2])
return prediction[2]
def find_answer(question):
# Set your path to pdf directory
df = pdf_converter(directory_path='pdf_folder/')
cdqa_pipeline = QAPipeline(reader='models/bert_qa.joblib')
cdqa_pipeline.fit_retriever(df)
query = question + '?'
prediction = cdqa_pipeline.predict(query)
# print('query: {}\n'.format(query))
# print('answer: {}\n'.format(prediction[0]))
# print('title: {}\n'.format(prediction[1]))
# print('paragraph: {}\n'.format(prediction[2]))
return prediction[0]
def search_view(request):
if request.POST:
question = request.POST.get('question')
for idx, url in enumerate(
search(question, tld="com", num=10, stop=3, pause=2)):
crawl_result(url, idx)
# change path to pdfs folder
df = pdf_converter(directory_path='/path/to/pdfs')
cdqa_pipeline = QAPipeline(reader='models/bert_qa.joblib')
cdqa_pipeline.fit_retriever(df)
prediction = cdqa_pipeline.predict(question)
data = {'answer': prediction[0]}
return JsonResponse(data)
return render(request, 'search.html')
def execute_pipeline(query):
download_bnpp_data('./data/bnpp_newsroom_v1.1/')
download_model('bert-squad_1.1', dir='./models')
df = pd.read_csv('./data/bnpp_newsroom_v1.1/bnpp_newsroom-v1.1.csv',
converters={'paragraphs': literal_eval})
df = filter_paragraphs(df)
cdqa_pipeline = QAPipeline(reader='models/bert_qa_vCPU-sklearn.joblib')
cdqa_pipeline.fit_retriever(X=df)
prediction = cdqa_pipeline.predict(X=query)
result = (prediction[0], prediction[1])
return result
def test_evaluate_pipeline():
download_bnpp_data("./data/bnpp_newsroom_v1.1/")
download_model("bert-squad_1.1", dir="./models")
df = pd.read_csv(
"./data/bnpp_newsroom_v1.1/bnpp_newsroom-v1.1.csv",
converters={"paragraphs": literal_eval},
)
df = filter_paragraphs(df)
test_data = {
"data": [
{
"title": "BNP Paribas’ commitment to universities and schools",
"paragraphs": [
{
"context": "Since January 2016, BNP Paribas has offered an Excellence Program targeting new Master’s level graduates (BAC+5) who show high potential. The aid program lasts 18 months and comprises three assignments of six months each. It serves as a strong career accelerator that enables participants to access high-level management positions at a faster rate. The program allows participants to discover the BNP Paribas Group and its various entities in France and abroad, build an internal and external network by working on different assignments and receive personalized assistance from a mentor and coaching firm at every step along the way.",
"qas": [
{
"answers": [
{"answer_start": 6, "text": "January 2016"},
{"answer_start": 6, "text": "January 2016"},
{"answer_start": 6, "text": "January 2016"},
],
"question": "Since when does the Excellence Program of BNP Paribas exist?",
"id": "56be4db0acb8001400a502ec",
}
],
}
],
}
],
"version": "1.1",
}
with open("./test_data.json", "w") as f:
json.dump(test_data, f)
cdqa_pipeline = QAPipeline(reader="./models/bert_qa_vCPU-sklearn.joblib", n_jobs=-1)
cdqa_pipeline.fit_retriever(X=df)
eval_dict = evaluate_pipeline(cdqa_pipeline, "./test_data.json", output_dir=None)
assert eval_dict["exact_match"] > 0.8
assert eval_dict["f1"] > 0.8
def ask():
name = request.form['btn-input']
#print(name)
f = open('current.txt')
file1 = f.read().rstrip()
f.close()
cdqa_pipeline = QAPipeline(reader='models/bert_qa_vCPU-sklearn.joblib')
row = file_open(file1)
df = pd.DataFrame(row)
df = df.T
df.columns = ['title', 'paragraphs']
#print(df.head())
# Fitting the retriever to the list of documents in the dataframe
cdqa_pipeline.fit_retriever(df)
prediction = cdqa_pipeline.predict(name)
ret = [name, prediction[0], prediction[1], prediction[2]]
speech = ret[1] + "\n\n Related Paragraph" + ret[3]
print('This is error output', speech)
#return speech
return render_template('index.html', value1=name, value2=speech)
def execute_pipeline(query, n_predictions=None):
download_bnpp_data("./data/bnpp_newsroom_v1.1/")
download_model("bert-squad_1.1", dir="./models")
df = pd.read_csv(
"./data/bnpp_newsroom_v1.1/bnpp_newsroom-v1.1.csv",
converters={"paragraphs": literal_eval},
)
df = filter_paragraphs(df)
cdqa_pipeline = QAPipeline(reader="models/bert_qa_vCPU-sklearn.joblib")
cdqa_pipeline.fit_retriever(X=df)
if n_predictions is not None:
predictions = cdqa_pipeline.predict(X=query,
n_predictions=n_predictions)
result = []
for answer, title, paragraph in predictions:
prediction = (answer, title)
result.append(prediction)
return result
else:
prediction = cdqa_pipeline.predict(X=query)
result = (prediction[0], prediction[1])
return result
import pandas as pd
from ast import literal_eval
from cdqa.utils.filters import filter_paragraphs
from cdqa.utils.download import download_model, download_bnpp_data
from cdqa.pipeline.cdqa_sklearn import QAPipeline
# Download data and models
#download_bnpp_data(dir='./data/bnpp_newsroom_v1.1/')
#download_model(model='bert-squad_1.1', dir='./models')
# Loading data and filtering / preprocessing the documents
df = pd.read_csv('data/bnpp_newsroom_v1.1/bnpp_newsroom-v1.1.csv',
converters={'paragraphs': literal_eval})
df = filter_paragraphs(df)
# Loading QAPipeline with CPU version of BERT Reader pretrained on SQuAD 1.1
cdqa_pipeline = QAPipeline(reader='models/bert_qa_vCPU-sklearn.joblib')
# Fitting the retriever to the list of documents in the dataframe
_ = cdqa_pipeline.fit_retriever(df)
# Sending a question to the pipeline and getting prediction
query = 'Since when does the Excellence Program of BNP Paribas exist?'
prediction = cdqa_pipeline.predict(query)
print('query: {}\n'.format(query))
print('answer: {}\n'.format(prediction[0]))
print('title: {}\n'.format(prediction[1]))
print('paragraph: {}\n'.format(prediction[2]))
print('유사도가 있고 리스트를 조회하는 경우\n')
print('{}\n\n{}\n\n{}'.format(df.loc[list(best_idx_scores.keys())[0]]['content'],POS_query,query))
print(''.join(df.loc[list(best_idx_scores.keys())[0]]['paragraphs']))
#best_idx_scores=''
continue#강빈이 코드에서는 필요없을것
#return 강빈이 코드에서 리턴하면 될것
elif list(best_idx_scores.keys())[0]<8 and list(list(retriever.predict(POS_query).values())[0])[0]<1.:
#첫번째 케이스를 대비한것
print(POS_query)
#best_idx_scores=''
continue
print('0~8의 인덱스가 안나온 경우 \n'+df.loc[list(best_idx_scores.keys())[0]]['title'])#테스트를 위한
#if max(retriever_temp.predict(ETRI_POS_Tagging(query)).values())>max(retriever.predict(ETRI_POS_Tagging(query)).values()): pass
#else:
cdqa_pipeline.fit_retriever(df.loc[best_idx_scores.keys()].head(1))
if max(retriever.predict(POS_query).values())<1.5 and max(retriever_temp.predict(POS_query).values())<1.5:
print(ETRI_wiki(query))
continue
kor_query=ETRI_korBERT(' '.join(list(df.loc[best_idx_scores.keys()].head(1)['paragraphs'])[0]),query)
#----------------------------------
# temp_prediction=retriever_temp.predict(kor_query)
# print(para[max(temp_prediction)])
#----------------------------------
prediction=cdqa_pipeline.predict(kor_query)
print('cdqa 유사도 수치 '+str(prediction[3]))
print(prediction[2])
# print('{}\n\n{}\n\n{}\n\n'.format(df.loc[list(best_idx_scores.keys())[0]]['content'],ETRI_POS_Tagging(query),query))
reader.fit(X=(train_examples, train_features))
# Output fine-tuned model
reader.model.to('cpu')
reader.device = torch.device('cpu')
joblib.dump(reader, os.path.join(reader.output_dir, 'bert_tim_qa_vCPU.joblib'))
#%% [markdown]
# ### Training
#%%
from cdqa.pipeline.cdqa_sklearn import QAPipeline
# Load standard model
cdqa_pipeline = QAPipeline(model='./cdqa/bert_qa_vCPU-sklearn.joblib', max_answer_length=60)
cdqa_pipeline.fit_retriever(X=df_X)
#%%
# Evaluate QnA system
from cdqa.utils.evaluation import evaluate_pipeline
evaluate_pipeline(cdqa_pipeline, 'cdqa-v1.1-tim_qna.json')
# Standard pre trained model: {'exact_match': 0.0, 'f1': 5.025362668068075}
# Fine-tuned model: {'exact_match': 0.0, 'f1': 5.684362620078064}
#%% [markdown]
# ### Inference
#%%
prediction = cdqa_pipeline.predict(X='what would be a good gymnastic strength training goal to have?')
configs = json.loads(cfgs)
question = configs['question']
data_directory = '/data/'
models_directory = '/models/'
# download_squad(dir = './' + data_directory)
download_bnpp_data(dir = './' + data_directory)
# download_model('distilbert-squad_1.1', dir = './' + models_directory)
download_model('bert-squad_1.1', dir = './' + models_directory)
df = pandas.read_csv(data_directory + '/bnpp_paribas/-??-.csv', converter = {'paragraphs': ast.literal_evl})
df = filter_paragraphs(df)
cdqa_pipeline = QAPipeline(reader = models_directory + '/bert_qa/bert_qa.joblib')
cdqa_pipeline.fit_retriever(q = df)
# cdqa_pipeline.fit_reader('path to squad like dataset . json')
prediction = cdqa_pipeline.predict(q = question, n_prediction = ?) # ? = predictions
# cdqa_pipeline.dump_reader('path to save . joblib') # save reader model
query = 'query: {}\n'.format(query),
answer = 'answer: {}\n'.format(prediction[0]),
title = 'title: {}\n'.format(prediction[1]),
paragraph = 'paragraph: {}\n'.format(prediction[2])
result = query, answer, title, paragraph
notify2.init('question answer')
notif = notify2.Notification('qa', result)
# notif.set_urgency(notify2.URGENCY_CRITICAL)
notif.show()