def main(): # load data train_data = load_standard_dataset(standard_train) # ======================= instantiate model ==================== # models are from https://huggingface.co/models?pipeline_tag=question-answering # model = QuestionAnsweringModel('roberta', 'csarron/roberta-large-squad-v1', args=train_args) # model = QuestionAnsweringModel('electra', 'mrm8488/electra-large-finetuned-squadv1', args=train_args) # model = QuestionAnsweringModel('albert', 'Wikidepia/albert-bahasa-cased-squad', args=train_args) # model = QuestionAnsweringModel('bert', 'bert-base-cased', args=train_args) model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', args=train_args) # model = QuestionAnsweringModel('bert', 'trained_models/bert', args=train_args) # run eval from pre-trained model # =========================== train model ====================== model.train_model(train_data) # ========================= do evaluation ====================== dev_data = load_standard_dataset(standard_dev) result, texts = model.eval_model(dev_data, f1=f1_multiclass, acc=accuracy_score) print(f'Result: {result}') # ========================= do predictions ===================== answers, probabilities = model.predict(dev_data, n_best_size=1) preds = {pred['id']: pred['answer'] for pred in answers} with open('results/squad_predictions.json', 'w') as f: for qid, answer in preds.items(): f.write(f'{{"qid": "{qid}", "answer": "{answer}"}}\n')
class GetCases: def __init__(self): logging.basicConfig(level=logging.INFO) transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.WARNING) self.model = QuestionAnsweringModel('distilbert', 'outputs/', args={ 'reprocess_input_data': True, 'overwrite_output_dir': True, 'fp16': False }, use_cuda=False) def train_model(self): train_data = [] with open('C:/Users/NathanGrant/Downloads/rona/rona/training_data.json' ) as f: train_data = json.load(f) self.model.train_model(train_data) def predict(self, news, county): to_predict = [] county = re.sub(", [A-Z]+", " county", county).lower() temp = { 'context': news, 'qas': [{ 'question': 'Total deaths in ' + county, 'id': '0' }] } to_predict.append(temp) pre = self.model.predict(to_predict) cases = [prediction['answer'] for prediction in pre] print(cases) if len(cases) > 0: for i in range(len(cases)): try: cases[i] = int(cases[i]) except: cases[i] = w2n.word_to_num(cases[i]) else: return 0 return cases def evaluate_model(self, data): result, text = self.model.eval_model(data) print(text) print(result)
'save_steps': -1, #500000 'train_batch_size': batch_size, 'num_train_epochs': epochs } ###### only train on first 100 samples x_train = x_train[:100] x_test = x_test[:100] x_valid = x_valid[:100] ############# qa_model = QuestionAnsweringModel(model_family[model], model_exact_id[model], args=train_args, cuda_device=cuda) qa_model.train_model(x_train, eval_data=x_valid) qa_model = QuestionAnsweringModel(model_family[model], save_dir + 'best_model/', args=train_args, cuda_device=cuda) result, text = qa_model.eval_model(x_test) r = evaluate_results(text) print (r) ''' rf = open('results/dailydialog_qa.txt', 'a') rf.write(str(args) + '\n\n') rf.write(r + '\n' + '-'*40 + '\n') rf.close() rf = open(result_file, 'a') rf.write(str(args) + '\n\n') rf.write(r + '\n' + '-'*40 + '\n') rf.close()
def test_question_answering(model_type, model_name): # Create dummy data to use for training. train_data = [ { "context": "This is the first context", "qas": [ { "id": "00001", "is_impossible": False, "question": "Which context is this?", "answers": [{"text": "the first", "answer_start": 8}], } ], }, { "context": "Other legislation followed, including the Migratory Bird Conservation Act of 1929, a 1937 treaty prohibiting the hunting of right and gray whales,\ and the Bald Eagle Protection Act of 1940. These later laws had a low cost to society—the species were relatively rare—and little opposition was raised", "qas": [ { "id": "00002", "is_impossible": False, "question": "What was the cost to society?", "answers": [{"text": "low cost", "answer_start": 225}], }, { "id": "00003", "is_impossible": False, "question": "What was the name of the 1937 treaty?", "answers": [{"text": "Bald Eagle Protection Act", "answer_start": 167}], }, {"id": "00004", "is_impossible": True, "question": "How did Alexandar Hamilton die?", "answers": [],}, ], }, ] # noqa for i in range(4): train_data.extend(train_data) # Save as a JSON file os.makedirs("data", exist_ok=True) with open("data/train.json", "w") as f: json.dump(train_data, f) logging.basicConfig(level=logging.WARNING) transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.ERROR) # Create the QuestionAnsweringModel model = QuestionAnsweringModel( model_type, model_name, args={"no_save": True, "reprocess_input_data": True, "overwrite_output_dir": True}, use_cuda=False, ) # Train the model model.train_model("data/train.json") # Evaluate the model. (Being lazy and evaluating on the train data itself) result, text = model.eval_model("data/train.json") # Making predictions using the model. to_predict = [ { "context": "This is the context used for demonstrating predictions.", "qas": [{"question": "What is this context?", "id": "0"}], } ] model.predict(to_predict)
def test_question_answering(): # Create dummy data to use for training. train_data = [ { "context": "This is the first context", "qas": [{ "id": "00001", "is_impossible": False, "question": "Which context is this?", "answers": [{ "text": "the first", "answer_start": 8 }], }], }, { "context": "Other legislation followed, including the Migratory Bird" " Conservation Act of 1929, a 1937 treaty prohibiting the hunting of" " right and gray whales, and the Bald Eagle Protection Act of 1940." " These later laws had a low cost to society—the species were" " relatively rare—and little opposition was raised", "qas": [ { "id": "00002", "is_impossible": False, "question": "What was the cost to society?", "answers": [{ "text": "low cost", "answer_start": 225 }], }, { "id": "00003", "is_impossible": False, "question": "What was the name of the 1937 treaty?", "answers": [{ "text": "Bald Eagle Protection Act", "answer_start": 167 }], }, ], }, ] # Save as a JSON file os.makedirs("data", exist_ok=True) with open("data/train.json", "w") as f: json.dump(train_data, f) # Create the QuestionAnsweringModel model = QuestionAnsweringModel( "distilbert", "distilbert-base-uncased-distilled-squad", args={ "no_save": True, "reprocess_input_data": True, "overwrite_output_dir": True }, use_cuda=False, ) # Train the model model.train_model("data/train.json") # Evaluate the model. (Being lazy and evaluating on the train data itself) result, text = model.eval_model("data/train.json") # Making predictions using the model. to_predict = [{ "context": "This is the context used for demonstrating predictions.", "qas": [{ "question": "What is this context?", "id": "0" }], }] model.predict(to_predict)
# Create the QuestionAnsweringModel model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', args={ 'reprocess_input_data': True, 'overwrite_output_dir': True }) # Train the model with JSON file model.train_model('data/train.json') # The list can also be used directly # model.train_model(train_data) # Evaluate the model. (Being lazy and evaluating on the train data itself) result, text = model.eval_model('data/train.json') print(result) print(text) print('-------------------') # Making predictions using the model. to_predict = [{ 'context': 'This is the context used for demonstrating predictions.', 'qas': [{ 'question': 'What is this context?', 'id': '0' }] }]
from simpletransformers.question_answering import QuestionAnsweringModel import json import os import logging logging.basicConfig(level=logging.INFO) transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.WARNING) # predictions for each sample will be written to the "output_dir" specified below # see "predictions_test.json" in the specified output_dir for per sample predictions model_args = { "eval_batch_size": 256, 'output_dir': "./output/", 'doc_stride': 128, 'do_lower_case': True, 'max_seq_length': 384 } #below give the folder where your trained model is model = QuestionAnsweringModel('bert', './models/bert-large-squad-docvqa-finetuned/', args=model_args) print(model.args) #import pdb; pdb.set_trace() with open('./data_in_squad_format/docvqa_test_squad_format.json') as f: json_data = json.load(f) model.eval_model(json_data)
# Create the QuestionAnsweringModel model = QuestionAnsweringModel( "distilbert", "distilbert-base-uncased-distilled-squad", args={ "reprocess_input_data": True, "overwrite_output_dir": True }, ) # Train the model model.train_model("data/train.json") # Evaluate the model. (Being lazy and evaluating on the train data itself) result, text = model.eval_model("data/train.json") print(result) print(text) print("-------------------") # Making predictions using the model. to_predict = [{ "context": "This is the context used for demonstrating predictions.", "qas": [{ "question": "What is this context?", "id": "0" }], }]
args.model] modelName = "%s_%s_%s_%s" % (args.model, pretrainFile.replace( "/", "-"), args.train, args.test) # Create the QuestionAnsweringModel model = QuestionAnsweringModel(args.model, pretrainFile, use_cuda=torch.cuda.is_available(), args={ 'reprocess_input_data': True, 'overwrite_output_dir': True }) # Train the model with JSON file model.train_model('data/dialog/%s.json' % args.train) # Evaluate the model. (Being lazy and evaluating on the train data itself) result, out = model.eval_model('data/dialog/%s.json' % args.test) print2file(args.out_dir, modelName, ".res", result, True) # save output to file with open('%s%s.out' % (args.out_dir, modelName), 'w') as f: json.dump(out, f) print('-------------------') # Making predictions using the model. # to_predict = [{'context': 'This is the context used for demonstrating predictions.', 'qas': [{'question': 'What is this context?', 'id': '0'}]}] # # print(model.predict(to_predict))
'bert-base-uncased', use_cuda=args.use_cuda, args={ 'reprocess_input_data': True, 'overwrite_output_dir': True }) modelName = "%s_%s" % (args.data, "bert") # best_acc = 0 for epoch in range(args.epoch_nb): if args.enable_wdc: model.train_model('data/sgd/wdc.json') modelName = "%s_%s_wdc" % (args.data, "bert") model.train_model('data/sgd/%s-train.json' % args.data) # result, out = model.eval_model('data/sgd/sgd-dev-%s.json' % args.eval_mode) result, out = model.eval_model('data/sgd/%s-test.json' % args.data) slot_acc = result['correct'] / (result['correct'] + result['similar'] + result['incorrect']) if slot_acc > best_acc: best_acc = slot_acc print2file(args.out_dir, modelName, ".res", slot_acc, True) # save output to file with open('%s%s.out' % (args.out_dir, modelName), 'w') as f: json.dump(out, f)