def train_squad(train_path=ST_TRAINING_JSON, squad_path=SQUAD_TRAINING_JSON):
trainset = load_trainset(train_path, squad_path=squad_path)
model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad',
args={'reprocess_input_data': True, 'overwrite_output_dir': True})
model.train_model(trainset)
return model
def qa_model(self):
"""
Must be called from question-answering module
:return: Loaded cached QA-MODELS
"""
from utils import model_info
from simpletransformers.question_answering import QuestionAnsweringModel
logger.info(f"Loading QA models...")
models = {}
for model in settings.AVAILABLE_QA_MODELS:
model_data = model_info.get(model)
try:
models[model] = QuestionAnsweringModel(
model_data.get("type"),
model_data.get("dir"),
args=model_data.get("args"),
use_cuda=True,
)
except ValueError as err:
logger.debug(err)
models[model] = QuestionAnsweringModel(
model_data.get("type"),
model_data.get("dir"),
args=model_data.get("args"),
use_cuda=False,
)
return models
def get_data():
if request.method == 'POST':
if (request.get_json() is None):
data = request.form
else:
data = request.get_json()
context = data['context']
question = data['question']
to_predict = [{
'context': context,
'qas': [{
'question': question,
'id': '0'
}]
}]
model_type = "electra"
model_name = "google/electra-base-discriminator"
model = QuestionAnsweringModel(model_type=model_type,
model_name=f"models/{model_type}/",
use_cuda=False)
preds, _ = model.predict(to_predict)
print(preds[0]['answer'][0])
if (preds[0]['answer'][0] == ""):
result = "No answer found"
else:
result = preds[0]['answer'][0]
return jsonify({'output': result})
def __init__(self):
logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)
self.model = QuestionAnsweringModel('distilbert',
'outputs/',
args={
'reprocess_input_data': True,
'overwrite_output_dir': True,
'fp16': False
},
use_cuda=False)
def train_model(model_name):
api = os.environ['DB_API'] # get the request path
path = os.environ[
'MODEL_REPO'] + "/" + model_name # get the model repository location
request_test = requests.get(api) # get the train data from db
train_data = request_test.json()[0][0] # format the data as json
model = None # init model
if model_name == "bert": # if the model selected is bert then
if not os.path.exists(path): # check whether BERT already exsits
# set the arguments for training
train_args = {
'learning_rate': 3e-5,
'num_train_epochs': 2,
'max_seq_length': 384,
'doc_stride': 128,
'overwrite_output_dir': True,
'reprocess_input_data': False,
'train_batch_size': 2,
'gradient_accumulation_steps': 8,
}
#download the bert model
model = QuestionAnsweringModel(
model_name,
"mrm8488/bert-tiny-5-finetuned-squadv2",
args=train_args,
use_cuda=False)
model.save_model(
path, model=model.model) # save the model in the repository
return json.dumps(
{
'message':
'BERT has been downloaded, in order to train request again'
},
sort_keys=False,
indent=4), 200 # return result
else: # if there is a BERT model available
model = QuestionAnsweringModel(model_name,
path + "/",
use_cuda=False) # load the model
model.train_model(
train_data) # train the model on the fetched data
model.save_model(path, model=model.model) # save the updated model
return json.dumps(
{'message': 'Model has been trained on database train data'},
sort_keys=False,
indent=4), 200 # return the result
return json.dumps({'message': model_name + 'has not yet been implemented'},
sort_keys=False,
indent=4), 404 # return the result
def __init__(self):
self.transformer_loggers = []
for name in logging.root.manager.loggerDict:
if len(name) >= 11 and name[:11] in ['transformer', 'simpletrans']:
self.transformer_loggers.append(logging.getLogger(name))
self.transformer_loggers[-1].setLevel(logging.ERROR)
url_str = 'https://totalgood.org/midata/models/bert/cased_simpletransformers.zip'
model_dir = os.path.join(DATA_DIR, 'simple-transformer')
if not os.path.isdir(model_dir):
os.mkdir(model_dir)
if (not os.path.exists(os.path.join(model_dir, 'config.json'))
or not os.path.exists(
os.path.join(model_dir, 'pytorch_model.bin'))
or not os.path.exists(
os.path.join(model_dir, 'special_tokens_map.json'))
or not os.path.exists(
os.path.join(model_dir, 'tokenizer_config.json'))
or not os.path.exists(
os.path.join(model_dir, 'training_args.bin'))
or not os.path.exists(os.path.join(model_dir, 'vocab.txt'))):
zip_local_path = os.path.join(model_dir,
'cased_simpletransformers.zip')
with DownloadProgressBar(unit='B',
unit_scale=True,
miniters=1,
desc=url_str.split('/')[-1]) as t:
urllib.request.urlretrieve(url_str,
filename=zip_local_path,
reporthook=t.update_to)
with zipfile.ZipFile(zip_local_path, 'r') as zip_file:
zip_file.extractall(model_dir)
os.remove(zip_local_path)
process_count = cpu_count() - 2 if cpu_count() > 2 else 1
args = {
'process_count': process_count,
'output_dir': model_dir,
'cache_dir': model_dir,
'no_cache': True,
'use_cached_eval_features': False,
'overwrite_output_dir': False,
'silent': True
}
self.model = QuestionAnsweringModel('bert',
model_dir,
args=args,
use_cuda=USE_CUDA)
class GetCases:
def __init__(self):
logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)
self.model = QuestionAnsweringModel('distilbert',
'outputs/',
args={
'reprocess_input_data': True,
'overwrite_output_dir': True,
'fp16': False
},
use_cuda=False)
def train_model(self):
train_data = []
with open('C:/Users/NathanGrant/Downloads/rona/rona/training_data.json'
) as f:
train_data = json.load(f)
self.model.train_model(train_data)
def predict(self, news, county):
to_predict = []
county = re.sub(", [A-Z]+", " county", county).lower()
temp = {
'context': news,
'qas': [{
'question': 'Total deaths in ' + county,
'id': '0'
}]
}
to_predict.append(temp)
pre = self.model.predict(to_predict)
cases = [prediction['answer'] for prediction in pre]
print(cases)
if len(cases) > 0:
for i in range(len(cases)):
try:
cases[i] = int(cases[i])
except:
cases[i] = w2n.word_to_num(cases[i])
else:
return 0
return cases
def evaluate_model(self, data):
result, text = self.model.eval_model(data)
print(text)
print(result)
def predict(model_name):
path = os.environ['MODEL_REPO'] + "/" + model_name # get the path to BERT
model = QuestionAnsweringModel(model_name, path + "/",
use_cuda=False) # Load the model
question = request.get_json() # get the question and context from input
result = model.predict(question) # predict the result
# format the result
answers = pd.DataFrame(result[0][0]['answer'][0:3], columns=["Answers"])
answers["probabilities"] = result[1][0]['probability'][0:3]
df_no_indices = answers.to_string(index=False)
return json.dumps(df_no_indices, sort_keys=False,
indent=4), 200 # return the result
def main():
print('\nRunning...')
#initialize the twitter bot with your personal twitter developer keys
api = intitilize_twitter_bot()
#get the mentions from your twitter timeline
mentions = api.mentions_timeline()
#a boolean to check if the first mention is being looked at. This will be changed
#in the future when I update the bot to look for tweets in real time.
first_mention = True
for mention in mentions:
if first_mention == True:
#Finds the question from the current mention being looked at
user_data = get_question_from_mentions(mention)
user_id = str(mention.id)
question = str(user_data[1])
question_context = get_context(question)
#if you would like to train a model on a different data-set, use the statement below
#model = QuestionAnsweringModel('bert', 'bert-base-cased', use_cuda=False, args=train_args)
#This statement initializes the model with the distilbert dataset and makes sure that we are not using
#CUDA on our machine. CUDA is a feature that is used with NVIDIA graphics cards in order to compute
#and execute the model at a faster speed. If you do not have a NVIDIA machine or CUDA installed, leave
#use_cuda equal to False.
model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', use_cuda=False)
#Get the prediction from the model
prediction = predict_answer(model, question, question_context)
#Evaulate the information in the prediction datatype
evaluation = evaluate_prediction(user_id, question, prediction)
print(evaluation)
reply_with_answer(api, user_id, evaluation)
first_mention = False
print('\n\nExiting...')
def create_model(model_class, model_type, model_name, num_labels, weight, args,
use_cuda, cuda_device, **kwargs):
if model_class == "ClassificationModel":
return ClassificationModel(model_type, model_name, num_labels, weight,
args, use_cuda, cuda_device, **kwargs)
elif model_class == "MultiLabelClassificationModel":
return MultiLabelClassificationModel(model_type, model_name,
num_labels, weight, args,
use_cuda, cuda_device, **kwargs)
elif model_class == "QuestionAnsweringModel":
return QuestionAnsweringModel(model_type, model_name, args, use_cuda,
cuda_device, **kwargs)
elif model_class == "NERModel":
return NERModel(model_type,
model_name,
args=args,
use_cuda=use_cuda,
cuda_device=cuda_device,
**kwargs)
elif model_class == "T5Model":
args = T5Args()
args.use_multiprocessed_decoding = False
return T5Model(model_type,
model_name,
args=args,
use_cuda=use_cuda,
cuda_device=cuda_device,
**kwargs)
else:
raise ValueError(
"{} is either invalid or not yet implemented.".format(model_class))
def run(self, dispatcher: CollectingDispatcher, tracker: Tracker,
domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:
model = QuestionAnsweringModel(
'distilbert', 'distilbert-base-uncased-distilled-squad')
recent_msg = tracker.latest_message['text']
ans = self.q_to_a(model, recent_msg, n=8)
dispatcher.utter_message(text=ans)
return []
def train():
parser = argparse.ArgumentParser()
parser.add_argument('--job-id', help='Training file name')
parser.add_argument('--model-type',
default='bert',
help='Type of model, eg: BERT, XLM, ROBERTA')
parser.add_argument('--model-name',
default='bert-base-cased',
help='If true will not use cross validation')
arguments, _ = parser.parse_known_args()
bucket = connect_to_storage('question_answering')
download_folder_structure_from_bucket(bucket, arguments.job_id,
arguments.job_id)
with open(posixpath.join(arguments.job_id, 'train_data.json'), 'r') as f:
train_data = json.load(f)
with open(posixpath.join(arguments.job_id, 'eval_data.json'), 'r') as f:
eval_data = json.load(f)
if os.path.isfile(posixpath.join(arguments.job_id, 'config.json')):
with open(posixpath.join(arguments.job_id, 'config.json'), 'r') as f:
args = json.load(f)
else:
args = None
output_dir = posixpath.join(arguments.job_id, 'model')
os.makedirs(output_dir, exist_ok=True)
from simpletransformers.question_answering import QuestionAnsweringModel
model = QuestionAnsweringModel(arguments.model_type,
arguments.model_name,
args=args,
use_cuda=True)
model.train_model(train_data, eval_data=eval_data)
upload_folder_to_bucket(bucket,
'outputs',
output_dir,
recursive_upload=True)
upload_folder_to_bucket(bucket, 'runs', output_dir, recursive_upload=True)
def getModel():
model = ""
train_args = get_train_args()
if modelType == ModelType.TRAINED:
model = QuestionAnsweringModel("bert",
params[Params.TRAINED_MODEL_PATH],
args=train_args,
use_cuda=True)
if modelType == ModelType.NOT_TRAINED:
model = QuestionAnsweringModel("bert",
"bert-base-cased",
args=train_args,
use_cuda=True)
if model == "":
print("Error on getting the model")
exit(1)
return model
def get_model(model_name, epoch, batch_size):
model_base = model_name.split("/")[1].split("-")[0]
model = QuestionAnsweringModel(model_base,
model_name,
args={
"num_train_epochs": epoch,
"train_batch_size": batch_size
},
use_cuda=True)
return model
def load(self, path):
use_cuda = torch.cuda.is_available()
if not use_cuda:
raise UserWarning('CUDA not available.')
try:
import apex
fp16 = True
except ImportError:
fp16 = False
self.model = QuestionAnsweringModel(model_type='albert',
model_name=path,
use_cuda=use_cuda,
args={
'fp16': fp16,
'n_best_size': 1,
})
def answer_questions(questions, answer_text, model_path=None, use_cuda=False):
"""
Takes a `question` dict list and an `answer_text` string (which contains the
answer), and identifies the words within the `answer_text` that are the
answer.
Parameters
----------
questions : list
ex.)
questions = [
{
'id': 1,
'question': 'What are the three sources of European Union law?'
},
...
]
answer_text : str
model_path : str
use_cuda: bool
Return
-------
preds : list
ex.)
[
{
'id': '1',
'answer': 'primary law, secondary law and supplementary law'
},
...
]
"""
if model_path is None:
model_path = os.path.dirname(os.path.abspath(__file__)) + '/outputs'
# model = QuestionAnsweringModel('bert', './outputs/final', use_cuda=use_cuda)
model = QuestionAnsweringModel('bert', model_path, use_cuda=use_cuda)
data_to_predict = [{'context': answer_text, 'qas': questions}]
preds = model.predict(data_to_predict)
return preds
def fine_tuning(model_name, epoch, batch_size):
model_base = model_name.split("/")[1].split("-")[0]
train_args = {
"reprocess_input_data": True,
"overwrite_output_dir": True,
}
filename = "train-v0.1.json"
with open('train_data.txt', 'w') as f:
for item in data_list:
f.write("%s\n" % item)
# for some models seed may cause error
try:
lang_model = LanguageModelingModel(model_base,
model_name,
torch.Generator().manual_seed(42),
use_cuda=True,
args=train_args)
except:
lang_model = LanguageModelingModel(model_base,
model_name,
use_cuda=True,
args=train_args)
lang_output_file_name = "lang_output"
lang_model.train_model("train_data.txt", output_dir=lang_output_file_name)
necessary_files_for_pretrained_model = [
'pytorch_model.bin', 'config.json', 'vocab.txt'
]
lang_model_output_files = files = glob(lang_output_file_name + "/*")
files_to_be_removed = []
for f in lang_model_output_files:
if f.split("/")[1] not in necessary_files_for_pretrained_model:
files_to_be_removed.append(f)
for f in files_to_be_removed:
if os.path.isfile(f):
os.remove(f)
elif os.path.isdir(f):
shutil.rmtree(f)
shutil.rmtree("cache_dir", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
train_args = {
"reprocess_input_data": True,
"overwrite_output_dir": True,
"num_train_epochs": epoch,
"train_batch_size": batch_size
}
qa_model = QuestionAnsweringModel(model_base,
"lang_output",
use_cuda=True,
args=train_args)
return qa_model
def main():
# load data
train_data = load_standard_dataset(standard_train)
# ======================= instantiate model ====================
# models are from https://huggingface.co/models?pipeline_tag=question-answering
# model = QuestionAnsweringModel('roberta', 'csarron/roberta-large-squad-v1', args=train_args)
# model = QuestionAnsweringModel('electra', 'mrm8488/electra-large-finetuned-squadv1', args=train_args)
# model = QuestionAnsweringModel('albert', 'Wikidepia/albert-bahasa-cased-squad', args=train_args)
# model = QuestionAnsweringModel('bert', 'bert-base-cased', args=train_args)
model = QuestionAnsweringModel('distilbert',
'distilbert-base-uncased-distilled-squad',
args=train_args)
# model = QuestionAnsweringModel('bert', 'trained_models/bert', args=train_args) # run eval from pre-trained model
# =========================== train model ======================
model.train_model(train_data)
# ========================= do evaluation ======================
dev_data = load_standard_dataset(standard_dev)
result, texts = model.eval_model(dev_data,
f1=f1_multiclass,
acc=accuracy_score)
print(f'Result: {result}')
# ========================= do predictions =====================
answers, probabilities = model.predict(dev_data, n_best_size=1)
preds = {pred['id']: pred['answer'] for pred in answers}
with open('results/squad_predictions.json', 'w') as f:
for qid, answer in preds.items():
f.write(f'{{"qid": "{qid}", "answer": "{answer}"}}\n')
def main():
# 创建模型
train_args = {
"do_lower_case": True,
"encoding": "utf-8",
"learning_rate": 3e-5,
"max_seq_length": 512,
"doc_stride": 128,
"max_query_length": 10,
"n_best_size": 5,
"max_answer_length": 60
}
use_cuda = torch.cuda.is_available()
print(use_cuda)
model = QuestionAnsweringModel("bert",
"model",
use_cuda=use_cuda,
args=train_args)
# 例子
context = "START4283 中标信息详情 [朝阳]分局处突车防护装备更新成交公告更正公告 2020-10-3013：03：16 一、项目基本情况 原公告的采购项目编号：CYCG_20_1846 原公告的采购项目名称：分局处突车防护装备更新 首次公告日期：2020-10-2916：42地址：http：//www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbjggg/t20201029_1287683.html 二、更正信息 更正事项：采购结果 更正内容： 1、主要标的信息中防弹防刺服数量1更正为195 2、补充公告附件 更正日期：2020-10-3011：00 三、其他补充事宜 / 四、凡对本次公告内容提出询问，请按以下方式联系。 1.采购人信息 名称：北京市公安局朝阳分局机关 地址：北京市朝阳区道家园1号 联系方式：吕德华,85953808 2.采购代理机构信息 名称：北京合信恒盛咨询有限公司 地 址：北京市朝阳区幺家店路常营公园内办公区办公用房 联系方式：李冬，65766188-8013 3.项目联系方式 项目联系人：李冬 电 话： 65766188-8013 更多内容请 下载保标APP 2015- 2019 woyaobid.com.AllRightsReserved世舶科技（武汉）有限公司版权所有 $(document).ready(function(){vardate=newDate；varnewyear=date.getFullYear()；$(\"#new_year\").html(newyear)；$(\"table\").each(function(){$(this).wrap(\"<divclass='margin-tong'></div>\")；})；})；；END"
result = predict(model, context)
print(json.dumps(result, ensure_ascii=False, indent=4))
train_args = {
'output_dir': 'outputs/flaubert_train/',
'fp16': False,
'learning_rate': 3e-5,
'num_train_epochs': 2,
'max_seq_length': 384,
'doc_stride': 128,
'overwrite_output_dir': True,
'reprocess_input_data': False,
'train_batch_size': 2,
'gradient_accumulation_steps': 8,
}
model = QuestionAnsweringModel('camembert',
'camembert-base',
args=train_args,
use_cuda=True)
model.train_model(train_data)
# with open('data/dev-v2.0.json', 'r') as f:
# dev_data = json.load(f)
# dev_data = [item for topic in dev_data['data'] for item in topic['paragraphs']]
# preds = model.predict(dev_data)
# os.makedirs('results', exist_ok=True)
# submission = {pred['id']: pred['answer'] for pred in preds}
pickle.dump(regressor, open('model.pkl','wb'))
model = pickle.load(open('model.pkl','rb'))
print(model.predict([[4, 300, 500]]))
"""
import torch
import tensorflow as tf
import tensorflow_hub as hub
from simpletransformers.question_answering import QuestionAnsweringModel
model = QuestionAnsweringModel('distilbert',
'distilbert-base-uncased-distilled-squad',
args={
'reprocess_input_data': True,
'overwrite_output_dir': True
},
use_cuda=False)
to_predict = [{
'context': 'This is the context used for demonstrating predictions.',
'qas': [{
'question': 'What is this context?',
'id': '0'
}]
}]
print(model.predict(to_predict))
pickle.dump(model, open('model.pkl', 'wb'))
model = pickle.load(open('model.pkl', 'rb'))
def answer_question(item: Item):
print(item)
train_data = [
{
"context":
"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levis Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the golden anniversary with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as Super Bowl L), so that the logo could prominently feature the Arabic numerals 50.",
"qas": [
{
"id": "00001",
"is_impossible": False,
"question": "Who is the new champion?",
"answers": [{
"text": "Denver Broncos",
"answer_start": 178,
}],
},
{
"id": "00002",
"is_impossible": False,
"question": "What year was Super Bowl 50?",
"answers": [{
"text": "2016",
"answer_start": 347,
}],
},
],
},
]
eval_data = [
{
"context":
"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levis Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the golden anniversary with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as Super Bowl L), so that the logo could prominently feature the Arabic numerals 50.",
"qas": [
{
"id": "00001",
"is_impossible": False,
"question": "Who is the new champion?",
"answers": [{
"text": "Denver Broncos",
"answer_start": 178,
}],
},
{
"id": "00002",
"is_impossible": False,
"question": "What year was Super Bowl 50?",
"answers": [{
"text": "2016",
"answer_start": 347,
}],
},
],
},
]
# define model args
model_args = QuestionAnsweringArgs()
model_args.num_train_epochs = 5
model_args.reprocess_input_data = True
model_args.overwrite_output_dir = True
# model_args.use_early_stopping = True
# model_args.early_stopping_delta = 0.01
# model_args.early_stopping_metric = "mcc"
# model_args.early_stopping_metric_minimize = False
# model_args.early_stopping_patience = 5
# model_args.evaluate_during_training_steps = 1000
# Create the QuestionAnsweringModel
model = QuestionAnsweringModel('distilbert',
'distilbert-base-uncased-distilled-squad',
args=model_args,
use_cuda=False)
# model.train_model(train_data, eval_data=eval_data)
# Making predictions using the model.
to_predict = [{
'context': item.context,
'qas': [{
'question': item.question,
'id': '0'
}]
}]
results = model.predict(to_predict)
return results
'evaluate_during_training': True,
'evaluate_during_training_steps': num_steps,
'save_eval_checkpoints': False,
'save_model_every_epoch': False,
'save_steps': -1, #500000
'train_batch_size': batch_size,
'num_train_epochs': epochs
}
###### only train on first 100 samples
x_train = x_train[:100]
x_test = x_test[:100]
x_valid = x_valid[:100]
#############
qa_model = QuestionAnsweringModel(model_family[model], model_exact_id[model], args=train_args, cuda_device=cuda)
qa_model.train_model(x_train, eval_data=x_valid)
qa_model = QuestionAnsweringModel(model_family[model], save_dir + 'best_model/', args=train_args, cuda_device=cuda)
result, text = qa_model.eval_model(x_test)
r = evaluate_results(text)
print (r)
'''
rf = open('results/dailydialog_qa.txt', 'a')
rf.write(str(args) + '\n\n')
rf.write(r + '\n' + '-'*40 + '\n')
rf.close()
rf = open(result_file, 'a')
def test_question_answering(model_type, model_name):
# Create dummy data to use for training.
train_data = [
{
"context": "This is the first context",
"qas": [
{
"id": "00001",
"is_impossible": False,
"question": "Which context is this?",
"answers": [{"text": "the first", "answer_start": 8}],
}
],
},
{
"context": "Other legislation followed, including the Migratory Bird Conservation Act of 1929, a 1937 treaty prohibiting the hunting of right and gray whales,\
and the Bald Eagle Protection Act of 1940. These later laws had a low cost to society—the species were relatively rare—and little opposition was raised",
"qas": [
{
"id": "00002",
"is_impossible": False,
"question": "What was the cost to society?",
"answers": [{"text": "low cost", "answer_start": 225}],
},
{
"id": "00003",
"is_impossible": False,
"question": "What was the name of the 1937 treaty?",
"answers": [{"text": "Bald Eagle Protection Act", "answer_start": 167}],
},
{"id": "00004", "is_impossible": True, "question": "How did Alexandar Hamilton die?", "answers": [],},
],
},
] # noqa
for i in range(4):
train_data.extend(train_data)
# Save as a JSON file
os.makedirs("data", exist_ok=True)
with open("data/train.json", "w") as f:
json.dump(train_data, f)
logging.basicConfig(level=logging.WARNING)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.ERROR)
# Create the QuestionAnsweringModel
model = QuestionAnsweringModel(
model_type,
model_name,
args={"no_save": True, "reprocess_input_data": True, "overwrite_output_dir": True},
use_cuda=False,
)
# Train the model
model.train_model("data/train.json")
# Evaluate the model. (Being lazy and evaluating on the train data itself)
result, text = model.eval_model("data/train.json")
# Making predictions using the model.
to_predict = [
{
"context": "This is the context used for demonstrating predictions.",
"qas": [{"question": "What is this context?", "id": "0"}],
}
]
model.predict(to_predict)
def test_question_answering():
# Create dummy data to use for training.
train_data = [
{
"context":
"This is the first context",
"qas": [{
"id": "00001",
"is_impossible": False,
"question": "Which context is this?",
"answers": [{
"text": "the first",
"answer_start": 8
}],
}],
},
{
"context":
"Other legislation followed, including the Migratory Bird"
" Conservation Act of 1929, a 1937 treaty prohibiting the hunting of"
" right and gray whales, and the Bald Eagle Protection Act of 1940."
" These later laws had a low cost to society—the species were"
" relatively rare—and little opposition was raised",
"qas": [
{
"id": "00002",
"is_impossible": False,
"question": "What was the cost to society?",
"answers": [{
"text": "low cost",
"answer_start": 225
}],
},
{
"id":
"00003",
"is_impossible":
False,
"question":
"What was the name of the 1937 treaty?",
"answers": [{
"text": "Bald Eagle Protection Act",
"answer_start": 167
}],
},
],
},
]
# Save as a JSON file
os.makedirs("data", exist_ok=True)
with open("data/train.json", "w") as f:
json.dump(train_data, f)
# Create the QuestionAnsweringModel
model = QuestionAnsweringModel(
"distilbert",
"distilbert-base-uncased-distilled-squad",
args={
"no_save": True,
"reprocess_input_data": True,
"overwrite_output_dir": True
},
use_cuda=False,
)
# Train the model
model.train_model("data/train.json")
# Evaluate the model. (Being lazy and evaluating on the train data itself)
result, text = model.eval_model("data/train.json")
# Making predictions using the model.
to_predict = [{
"context": "This is the context used for demonstrating predictions.",
"qas": [{
"question": "What is this context?",
"id": "0"
}],
}]
model.predict(to_predict)
import flask
from flask import request, jsonify
from flask_cors import CORS, cross_origin
from simpletransformers.question_answering import QuestionAnsweringModel
model = QuestionAnsweringModel('xlmroberta',
'./trained_model/',
use_cuda=False,
args={'fp16': False})
app = flask.Flask(__name__)
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'
@app.route('/predict', methods=['POST'])
@cross_origin()
def predict():
request_data = request.json
print(request_data)
predictions, raw_outputs = model.predict([{
"context":
request_data.get('context', ''),
"qas": [{
"question": request_data.get('question', ''),
"id": "0",
}],
}])
# "use_early_stopping": True,
"n_best_size": 3,
"fp16": False,
"no_save": True,
"manual_seed": 4,
"max_seq_length": 512,
"no_save": True,
"n_best_size": 10,
"lazy_loading": True,
# "use_multiprocessing": False,
}
# Create the QuestionAnsweringModel
model = QuestionAnsweringModel("bert",
"bert-base-cased",
args=train_args,
use_cuda=True,
cuda_device=0)
# Train the model with JSON file
model.train_model("data/train.jsonl", eval_data="data/train.json")
# Making predictions using the model.
to_predict = [{
"context":
"Other legislation followed, including the Migratory Bird Conservation Act of 1929, a 1937 treaty prohibiting the hunting of right and gray whales,\
and the Bald Eagle Protection Act of 1940. These later laws had a low cost to society—the species were relatively rare—and little opposition was raised",
"qas": [{
"question": "What was the name of the 1937 treaty?",
"id": "0"
}],
]
train_args = {
'learning_rate': 1e-5,
'num_train_epochs': 1,
'max_seq_length': 384,
'doc_stride': 128,
'overwrite_output_dir': True,
'reprocess_input_data': False,
'train_batch_size': 2,
'gradient_accumulation_steps': 8,
'save_model_every_epoch': False
}
model = QuestionAnsweringModel('bert',
'bert-base-cased',
use_cuda=False,
args=train_args)
model.train_model(train_data, output_dir=None)
#Prediction
with open('dev-v2.0.json', 'r') as f:
dev_data = json.load(f)
dev_data = [item for topic in dev_data['data'] for item in topic['paragraphs']]
preds = model.predict(dev_data)
os.makedirs('results', exist_ok=True)
submission = {pred['id']: pred['answer'] for pred in preds}
qa_test
# !pip install seqeval
# !pip install transformers
%%time
from simpletransformers.question_answering import QuestionAnsweringModel
model = QuestionAnsweringModel('distilbert',
'/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/',
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 4,
'max_seq_length': 200,
'doc_stride': 64,
'fp16': False,
},
use_cuda=True)
model.train_model(qa_train)
%%time
preds = model.predict(qa_test)
predic_df = pd.DataFrame.from_dict(preds)
sub_df['selected_text'] = predic_df['answer']
sub_df.to_csv("submission.csv", sep=',', index=False)
sub_df.head()
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
output.append({'context': context.lower(), 'qas': qas})
return output
qa_test = do_qa_test(test)
with open('/content/gdrive/My Drive/data/test.json', 'w') as outfile:
json.dump(qa_test, outfile)
MODEL_PATH = '/content/gdrive/My Drive/model_deeplearning/'
#MODEL_PATH = 'https://drive.google.com/drive/folders/1CkjjRb6GJENfPQqfDJgVnzwipShmy4RE?usp=sharing'
model = QuestionAnsweringModel('distilbert',
MODEL_PATH,
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 3,
'max_seq_length': 192,
'doc_stride': 64,
'fp16': False,
},
use_cuda=True)
model.train_model('/content/gdrive/My Drive/data/train.json')
predictions = model.predict(qa_test)
predictions_df = pd.DataFrame.from_dict(predictions)
sub_df['selected_text'] = predictions_df['answer']
sub_df.to_csv('/content/gdrive/My Drive/sample_submission.csv', index=False)
print("File submitted successfully.")
#test_df.head()