def train_squad(train_path=ST_TRAINING_JSON, squad_path=SQUAD_TRAINING_JSON): trainset = load_trainset(train_path, squad_path=squad_path) model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', args={'reprocess_input_data': True, 'overwrite_output_dir': True}) model.train_model(trainset) return model
def qa_model(self): """ Must be called from question-answering module :return: Loaded cached QA-MODELS """ from utils import model_info from simpletransformers.question_answering import QuestionAnsweringModel logger.info(f"Loading QA models...") models = {} for model in settings.AVAILABLE_QA_MODELS: model_data = model_info.get(model) try: models[model] = QuestionAnsweringModel( model_data.get("type"), model_data.get("dir"), args=model_data.get("args"), use_cuda=True, ) except ValueError as err: logger.debug(err) models[model] = QuestionAnsweringModel( model_data.get("type"), model_data.get("dir"), args=model_data.get("args"), use_cuda=False, ) return models
def get_data(): if request.method == 'POST': if (request.get_json() is None): data = request.form else: data = request.get_json() context = data['context'] question = data['question'] to_predict = [{ 'context': context, 'qas': [{ 'question': question, 'id': '0' }] }] model_type = "electra" model_name = "google/electra-base-discriminator" model = QuestionAnsweringModel(model_type=model_type, model_name=f"models/{model_type}/", use_cuda=False) preds, _ = model.predict(to_predict) print(preds[0]['answer'][0]) if (preds[0]['answer'][0] == ""): result = "No answer found" else: result = preds[0]['answer'][0] return jsonify({'output': result})
def __init__(self): logging.basicConfig(level=logging.INFO) transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.WARNING) self.model = QuestionAnsweringModel('distilbert', 'outputs/', args={ 'reprocess_input_data': True, 'overwrite_output_dir': True, 'fp16': False }, use_cuda=False)
def train_model(model_name): api = os.environ['DB_API'] # get the request path path = os.environ[ 'MODEL_REPO'] + "/" + model_name # get the model repository location request_test = requests.get(api) # get the train data from db train_data = request_test.json()[0][0] # format the data as json model = None # init model if model_name == "bert": # if the model selected is bert then if not os.path.exists(path): # check whether BERT already exsits # set the arguments for training train_args = { 'learning_rate': 3e-5, 'num_train_epochs': 2, 'max_seq_length': 384, 'doc_stride': 128, 'overwrite_output_dir': True, 'reprocess_input_data': False, 'train_batch_size': 2, 'gradient_accumulation_steps': 8, } #download the bert model model = QuestionAnsweringModel( model_name, "mrm8488/bert-tiny-5-finetuned-squadv2", args=train_args, use_cuda=False) model.save_model( path, model=model.model) # save the model in the repository return json.dumps( { 'message': 'BERT has been downloaded, in order to train request again' }, sort_keys=False, indent=4), 200 # return result else: # if there is a BERT model available model = QuestionAnsweringModel(model_name, path + "/", use_cuda=False) # load the model model.train_model( train_data) # train the model on the fetched data model.save_model(path, model=model.model) # save the updated model return json.dumps( {'message': 'Model has been trained on database train data'}, sort_keys=False, indent=4), 200 # return the result return json.dumps({'message': model_name + 'has not yet been implemented'}, sort_keys=False, indent=4), 404 # return the result
def __init__(self): self.transformer_loggers = [] for name in logging.root.manager.loggerDict: if len(name) >= 11 and name[:11] in ['transformer', 'simpletrans']: self.transformer_loggers.append(logging.getLogger(name)) self.transformer_loggers[-1].setLevel(logging.ERROR) url_str = 'https://totalgood.org/midata/models/bert/cased_simpletransformers.zip' model_dir = os.path.join(DATA_DIR, 'simple-transformer') if not os.path.isdir(model_dir): os.mkdir(model_dir) if (not os.path.exists(os.path.join(model_dir, 'config.json')) or not os.path.exists( os.path.join(model_dir, 'pytorch_model.bin')) or not os.path.exists( os.path.join(model_dir, 'special_tokens_map.json')) or not os.path.exists( os.path.join(model_dir, 'tokenizer_config.json')) or not os.path.exists( os.path.join(model_dir, 'training_args.bin')) or not os.path.exists(os.path.join(model_dir, 'vocab.txt'))): zip_local_path = os.path.join(model_dir, 'cased_simpletransformers.zip') with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url_str.split('/')[-1]) as t: urllib.request.urlretrieve(url_str, filename=zip_local_path, reporthook=t.update_to) with zipfile.ZipFile(zip_local_path, 'r') as zip_file: zip_file.extractall(model_dir) os.remove(zip_local_path) process_count = cpu_count() - 2 if cpu_count() > 2 else 1 args = { 'process_count': process_count, 'output_dir': model_dir, 'cache_dir': model_dir, 'no_cache': True, 'use_cached_eval_features': False, 'overwrite_output_dir': False, 'silent': True } self.model = QuestionAnsweringModel('bert', model_dir, args=args, use_cuda=USE_CUDA)
class GetCases: def __init__(self): logging.basicConfig(level=logging.INFO) transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.WARNING) self.model = QuestionAnsweringModel('distilbert', 'outputs/', args={ 'reprocess_input_data': True, 'overwrite_output_dir': True, 'fp16': False }, use_cuda=False) def train_model(self): train_data = [] with open('C:/Users/NathanGrant/Downloads/rona/rona/training_data.json' ) as f: train_data = json.load(f) self.model.train_model(train_data) def predict(self, news, county): to_predict = [] county = re.sub(", [A-Z]+", " county", county).lower() temp = { 'context': news, 'qas': [{ 'question': 'Total deaths in ' + county, 'id': '0' }] } to_predict.append(temp) pre = self.model.predict(to_predict) cases = [prediction['answer'] for prediction in pre] print(cases) if len(cases) > 0: for i in range(len(cases)): try: cases[i] = int(cases[i]) except: cases[i] = w2n.word_to_num(cases[i]) else: return 0 return cases def evaluate_model(self, data): result, text = self.model.eval_model(data) print(text) print(result)
def predict(model_name): path = os.environ['MODEL_REPO'] + "/" + model_name # get the path to BERT model = QuestionAnsweringModel(model_name, path + "/", use_cuda=False) # Load the model question = request.get_json() # get the question and context from input result = model.predict(question) # predict the result # format the result answers = pd.DataFrame(result[0][0]['answer'][0:3], columns=["Answers"]) answers["probabilities"] = result[1][0]['probability'][0:3] df_no_indices = answers.to_string(index=False) return json.dumps(df_no_indices, sort_keys=False, indent=4), 200 # return the result
def main(): print('\nRunning...') #initialize the twitter bot with your personal twitter developer keys api = intitilize_twitter_bot() #get the mentions from your twitter timeline mentions = api.mentions_timeline() #a boolean to check if the first mention is being looked at. This will be changed #in the future when I update the bot to look for tweets in real time. first_mention = True for mention in mentions: if first_mention == True: #Finds the question from the current mention being looked at user_data = get_question_from_mentions(mention) user_id = str(mention.id) question = str(user_data[1]) question_context = get_context(question) #if you would like to train a model on a different data-set, use the statement below #model = QuestionAnsweringModel('bert', 'bert-base-cased', use_cuda=False, args=train_args) #This statement initializes the model with the distilbert dataset and makes sure that we are not using #CUDA on our machine. CUDA is a feature that is used with NVIDIA graphics cards in order to compute #and execute the model at a faster speed. If you do not have a NVIDIA machine or CUDA installed, leave #use_cuda equal to False. model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', use_cuda=False) #Get the prediction from the model prediction = predict_answer(model, question, question_context) #Evaulate the information in the prediction datatype evaluation = evaluate_prediction(user_id, question, prediction) print(evaluation) reply_with_answer(api, user_id, evaluation) first_mention = False print('\n\nExiting...')
def create_model(model_class, model_type, model_name, num_labels, weight, args, use_cuda, cuda_device, **kwargs): if model_class == "ClassificationModel": return ClassificationModel(model_type, model_name, num_labels, weight, args, use_cuda, cuda_device, **kwargs) elif model_class == "MultiLabelClassificationModel": return MultiLabelClassificationModel(model_type, model_name, num_labels, weight, args, use_cuda, cuda_device, **kwargs) elif model_class == "QuestionAnsweringModel": return QuestionAnsweringModel(model_type, model_name, args, use_cuda, cuda_device, **kwargs) elif model_class == "NERModel": return NERModel(model_type, model_name, args=args, use_cuda=use_cuda, cuda_device=cuda_device, **kwargs) elif model_class == "T5Model": args = T5Args() args.use_multiprocessed_decoding = False return T5Model(model_type, model_name, args=args, use_cuda=use_cuda, cuda_device=cuda_device, **kwargs) else: raise ValueError( "{} is either invalid or not yet implemented.".format(model_class))
def run(self, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]) -> List[Dict[Text, Any]]: model = QuestionAnsweringModel( 'distilbert', 'distilbert-base-uncased-distilled-squad') recent_msg = tracker.latest_message['text'] ans = self.q_to_a(model, recent_msg, n=8) dispatcher.utter_message(text=ans) return []
def train(): parser = argparse.ArgumentParser() parser.add_argument('--job-id', help='Training file name') parser.add_argument('--model-type', default='bert', help='Type of model, eg: BERT, XLM, ROBERTA') parser.add_argument('--model-name', default='bert-base-cased', help='If true will not use cross validation') arguments, _ = parser.parse_known_args() bucket = connect_to_storage('question_answering') download_folder_structure_from_bucket(bucket, arguments.job_id, arguments.job_id) with open(posixpath.join(arguments.job_id, 'train_data.json'), 'r') as f: train_data = json.load(f) with open(posixpath.join(arguments.job_id, 'eval_data.json'), 'r') as f: eval_data = json.load(f) if os.path.isfile(posixpath.join(arguments.job_id, 'config.json')): with open(posixpath.join(arguments.job_id, 'config.json'), 'r') as f: args = json.load(f) else: args = None output_dir = posixpath.join(arguments.job_id, 'model') os.makedirs(output_dir, exist_ok=True) from simpletransformers.question_answering import QuestionAnsweringModel model = QuestionAnsweringModel(arguments.model_type, arguments.model_name, args=args, use_cuda=True) model.train_model(train_data, eval_data=eval_data) upload_folder_to_bucket(bucket, 'outputs', output_dir, recursive_upload=True) upload_folder_to_bucket(bucket, 'runs', output_dir, recursive_upload=True)
def getModel(): model = "" train_args = get_train_args() if modelType == ModelType.TRAINED: model = QuestionAnsweringModel("bert", params[Params.TRAINED_MODEL_PATH], args=train_args, use_cuda=True) if modelType == ModelType.NOT_TRAINED: model = QuestionAnsweringModel("bert", "bert-base-cased", args=train_args, use_cuda=True) if model == "": print("Error on getting the model") exit(1) return model
def get_model(model_name, epoch, batch_size): model_base = model_name.split("/")[1].split("-")[0] model = QuestionAnsweringModel(model_base, model_name, args={ "num_train_epochs": epoch, "train_batch_size": batch_size }, use_cuda=True) return model
def load(self, path): use_cuda = torch.cuda.is_available() if not use_cuda: raise UserWarning('CUDA not available.') try: import apex fp16 = True except ImportError: fp16 = False self.model = QuestionAnsweringModel(model_type='albert', model_name=path, use_cuda=use_cuda, args={ 'fp16': fp16, 'n_best_size': 1, })
def answer_questions(questions, answer_text, model_path=None, use_cuda=False): """ Takes a `question` dict list and an `answer_text` string (which contains the answer), and identifies the words within the `answer_text` that are the answer. Parameters ---------- questions : list ex.) questions = [ { 'id': 1, 'question': 'What are the three sources of European Union law?' }, ... ] answer_text : str model_path : str use_cuda: bool Return ------- preds : list ex.) [ { 'id': '1', 'answer': 'primary law, secondary law and supplementary law' }, ... ] """ if model_path is None: model_path = os.path.dirname(os.path.abspath(__file__)) + '/outputs' # model = QuestionAnsweringModel('bert', './outputs/final', use_cuda=use_cuda) model = QuestionAnsweringModel('bert', model_path, use_cuda=use_cuda) data_to_predict = [{'context': answer_text, 'qas': questions}] preds = model.predict(data_to_predict) return preds
def fine_tuning(model_name, epoch, batch_size): model_base = model_name.split("/")[1].split("-")[0] train_args = { "reprocess_input_data": True, "overwrite_output_dir": True, } filename = "train-v0.1.json" with open('train_data.txt', 'w') as f: for item in data_list: f.write("%s\n" % item) # for some models seed may cause error try: lang_model = LanguageModelingModel(model_base, model_name, torch.Generator().manual_seed(42), use_cuda=True, args=train_args) except: lang_model = LanguageModelingModel(model_base, model_name, use_cuda=True, args=train_args) lang_output_file_name = "lang_output" lang_model.train_model("train_data.txt", output_dir=lang_output_file_name) necessary_files_for_pretrained_model = [ 'pytorch_model.bin', 'config.json', 'vocab.txt' ] lang_model_output_files = files = glob(lang_output_file_name + "/*") files_to_be_removed = [] for f in lang_model_output_files: if f.split("/")[1] not in necessary_files_for_pretrained_model: files_to_be_removed.append(f) for f in files_to_be_removed: if os.path.isfile(f): os.remove(f) elif os.path.isdir(f): shutil.rmtree(f) shutil.rmtree("cache_dir", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) train_args = { "reprocess_input_data": True, "overwrite_output_dir": True, "num_train_epochs": epoch, "train_batch_size": batch_size } qa_model = QuestionAnsweringModel(model_base, "lang_output", use_cuda=True, args=train_args) return qa_model
def main(): # load data train_data = load_standard_dataset(standard_train) # ======================= instantiate model ==================== # models are from https://huggingface.co/models?pipeline_tag=question-answering # model = QuestionAnsweringModel('roberta', 'csarron/roberta-large-squad-v1', args=train_args) # model = QuestionAnsweringModel('electra', 'mrm8488/electra-large-finetuned-squadv1', args=train_args) # model = QuestionAnsweringModel('albert', 'Wikidepia/albert-bahasa-cased-squad', args=train_args) # model = QuestionAnsweringModel('bert', 'bert-base-cased', args=train_args) model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', args=train_args) # model = QuestionAnsweringModel('bert', 'trained_models/bert', args=train_args) # run eval from pre-trained model # =========================== train model ====================== model.train_model(train_data) # ========================= do evaluation ====================== dev_data = load_standard_dataset(standard_dev) result, texts = model.eval_model(dev_data, f1=f1_multiclass, acc=accuracy_score) print(f'Result: {result}') # ========================= do predictions ===================== answers, probabilities = model.predict(dev_data, n_best_size=1) preds = {pred['id']: pred['answer'] for pred in answers} with open('results/squad_predictions.json', 'w') as f: for qid, answer in preds.items(): f.write(f'{{"qid": "{qid}", "answer": "{answer}"}}\n')
def main(): # 创建模型 train_args = { "do_lower_case": True, "encoding": "utf-8", "learning_rate": 3e-5, "max_seq_length": 512, "doc_stride": 128, "max_query_length": 10, "n_best_size": 5, "max_answer_length": 60 } use_cuda = torch.cuda.is_available() print(use_cuda) model = QuestionAnsweringModel("bert", "model", use_cuda=use_cuda, args=train_args) # 例子 context = "START4283 中标信息详情 [朝阳]分局处突车防护装备更新成交公告更正公告 2020-10-3013:03:16 一、项目基本情况 原公告的采购项目编号:CYCG_20_1846 原公告的采购项目名称:分局处突车防护装备更新 首次公告日期:2020-10-2916:42地址:http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbjggg/t20201029_1287683.html 二、更正信息 更正事项:采购结果 更正内容: 1、主要标的信息中防弹防刺服数量1更正为195 2、补充公告附件 更正日期:2020-10-3011:00 三、其他补充事宜 / 四、凡对本次公告内容提出询问,请按以下方式联系。 1.采购人信息 名称:北京市公安局朝阳分局机关 地址:北京市朝阳区道家园1号 联系方式:吕德华,85953808 2.采购代理机构信息 名称:北京合信恒盛咨询有限公司 地 址:北京市朝阳区幺家店路常营公园内办公区办公用房 联系方式:李冬,65766188-8013 3.项目联系方式 项目联系人:李冬 电 话: 65766188-8013 更多内容请 下载保标APP 2015- 2019 woyaobid.com.AllRightsReserved世舶科技(武汉)有限公司版权所有 $(document).ready(function(){vardate=newDate;varnewyear=date.getFullYear();$(\"#new_year\").html(newyear);$(\"table\").each(function(){$(this).wrap(\"<divclass='margin-tong'></div>\");});});;END" result = predict(model, context) print(json.dumps(result, ensure_ascii=False, indent=4))
train_args = { 'output_dir': 'outputs/flaubert_train/', 'fp16': False, 'learning_rate': 3e-5, 'num_train_epochs': 2, 'max_seq_length': 384, 'doc_stride': 128, 'overwrite_output_dir': True, 'reprocess_input_data': False, 'train_batch_size': 2, 'gradient_accumulation_steps': 8, } model = QuestionAnsweringModel('camembert', 'camembert-base', args=train_args, use_cuda=True) model.train_model(train_data) # with open('data/dev-v2.0.json', 'r') as f: # dev_data = json.load(f) # dev_data = [item for topic in dev_data['data'] for item in topic['paragraphs']] # preds = model.predict(dev_data) # os.makedirs('results', exist_ok=True) # submission = {pred['id']: pred['answer'] for pred in preds}
pickle.dump(regressor, open('model.pkl','wb')) model = pickle.load(open('model.pkl','rb')) print(model.predict([[4, 300, 500]])) """ import torch import tensorflow as tf import tensorflow_hub as hub from simpletransformers.question_answering import QuestionAnsweringModel model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', args={ 'reprocess_input_data': True, 'overwrite_output_dir': True }, use_cuda=False) to_predict = [{ 'context': 'This is the context used for demonstrating predictions.', 'qas': [{ 'question': 'What is this context?', 'id': '0' }] }] print(model.predict(to_predict)) pickle.dump(model, open('model.pkl', 'wb')) model = pickle.load(open('model.pkl', 'rb'))
def answer_question(item: Item): print(item) train_data = [ { "context": "Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levis Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the golden anniversary with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as Super Bowl L), so that the logo could prominently feature the Arabic numerals 50.", "qas": [ { "id": "00001", "is_impossible": False, "question": "Who is the new champion?", "answers": [{ "text": "Denver Broncos", "answer_start": 178, }], }, { "id": "00002", "is_impossible": False, "question": "What year was Super Bowl 50?", "answers": [{ "text": "2016", "answer_start": 347, }], }, ], }, ] eval_data = [ { "context": "Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levis Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the golden anniversary with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as Super Bowl L), so that the logo could prominently feature the Arabic numerals 50.", "qas": [ { "id": "00001", "is_impossible": False, "question": "Who is the new champion?", "answers": [{ "text": "Denver Broncos", "answer_start": 178, }], }, { "id": "00002", "is_impossible": False, "question": "What year was Super Bowl 50?", "answers": [{ "text": "2016", "answer_start": 347, }], }, ], }, ] # define model args model_args = QuestionAnsweringArgs() model_args.num_train_epochs = 5 model_args.reprocess_input_data = True model_args.overwrite_output_dir = True # model_args.use_early_stopping = True # model_args.early_stopping_delta = 0.01 # model_args.early_stopping_metric = "mcc" # model_args.early_stopping_metric_minimize = False # model_args.early_stopping_patience = 5 # model_args.evaluate_during_training_steps = 1000 # Create the QuestionAnsweringModel model = QuestionAnsweringModel('distilbert', 'distilbert-base-uncased-distilled-squad', args=model_args, use_cuda=False) # model.train_model(train_data, eval_data=eval_data) # Making predictions using the model. to_predict = [{ 'context': item.context, 'qas': [{ 'question': item.question, 'id': '0' }] }] results = model.predict(to_predict) return results
'evaluate_during_training': True, 'evaluate_during_training_steps': num_steps, 'save_eval_checkpoints': False, 'save_model_every_epoch': False, 'save_steps': -1, #500000 'train_batch_size': batch_size, 'num_train_epochs': epochs } ###### only train on first 100 samples x_train = x_train[:100] x_test = x_test[:100] x_valid = x_valid[:100] ############# qa_model = QuestionAnsweringModel(model_family[model], model_exact_id[model], args=train_args, cuda_device=cuda) qa_model.train_model(x_train, eval_data=x_valid) qa_model = QuestionAnsweringModel(model_family[model], save_dir + 'best_model/', args=train_args, cuda_device=cuda) result, text = qa_model.eval_model(x_test) r = evaluate_results(text) print (r) ''' rf = open('results/dailydialog_qa.txt', 'a') rf.write(str(args) + '\n\n') rf.write(r + '\n' + '-'*40 + '\n') rf.close() rf = open(result_file, 'a')
def test_question_answering(model_type, model_name): # Create dummy data to use for training. train_data = [ { "context": "This is the first context", "qas": [ { "id": "00001", "is_impossible": False, "question": "Which context is this?", "answers": [{"text": "the first", "answer_start": 8}], } ], }, { "context": "Other legislation followed, including the Migratory Bird Conservation Act of 1929, a 1937 treaty prohibiting the hunting of right and gray whales,\ and the Bald Eagle Protection Act of 1940. These later laws had a low cost to society—the species were relatively rare—and little opposition was raised", "qas": [ { "id": "00002", "is_impossible": False, "question": "What was the cost to society?", "answers": [{"text": "low cost", "answer_start": 225}], }, { "id": "00003", "is_impossible": False, "question": "What was the name of the 1937 treaty?", "answers": [{"text": "Bald Eagle Protection Act", "answer_start": 167}], }, {"id": "00004", "is_impossible": True, "question": "How did Alexandar Hamilton die?", "answers": [],}, ], }, ] # noqa for i in range(4): train_data.extend(train_data) # Save as a JSON file os.makedirs("data", exist_ok=True) with open("data/train.json", "w") as f: json.dump(train_data, f) logging.basicConfig(level=logging.WARNING) transformers_logger = logging.getLogger("transformers") transformers_logger.setLevel(logging.ERROR) # Create the QuestionAnsweringModel model = QuestionAnsweringModel( model_type, model_name, args={"no_save": True, "reprocess_input_data": True, "overwrite_output_dir": True}, use_cuda=False, ) # Train the model model.train_model("data/train.json") # Evaluate the model. (Being lazy and evaluating on the train data itself) result, text = model.eval_model("data/train.json") # Making predictions using the model. to_predict = [ { "context": "This is the context used for demonstrating predictions.", "qas": [{"question": "What is this context?", "id": "0"}], } ] model.predict(to_predict)
def test_question_answering(): # Create dummy data to use for training. train_data = [ { "context": "This is the first context", "qas": [{ "id": "00001", "is_impossible": False, "question": "Which context is this?", "answers": [{ "text": "the first", "answer_start": 8 }], }], }, { "context": "Other legislation followed, including the Migratory Bird" " Conservation Act of 1929, a 1937 treaty prohibiting the hunting of" " right and gray whales, and the Bald Eagle Protection Act of 1940." " These later laws had a low cost to society—the species were" " relatively rare—and little opposition was raised", "qas": [ { "id": "00002", "is_impossible": False, "question": "What was the cost to society?", "answers": [{ "text": "low cost", "answer_start": 225 }], }, { "id": "00003", "is_impossible": False, "question": "What was the name of the 1937 treaty?", "answers": [{ "text": "Bald Eagle Protection Act", "answer_start": 167 }], }, ], }, ] # Save as a JSON file os.makedirs("data", exist_ok=True) with open("data/train.json", "w") as f: json.dump(train_data, f) # Create the QuestionAnsweringModel model = QuestionAnsweringModel( "distilbert", "distilbert-base-uncased-distilled-squad", args={ "no_save": True, "reprocess_input_data": True, "overwrite_output_dir": True }, use_cuda=False, ) # Train the model model.train_model("data/train.json") # Evaluate the model. (Being lazy and evaluating on the train data itself) result, text = model.eval_model("data/train.json") # Making predictions using the model. to_predict = [{ "context": "This is the context used for demonstrating predictions.", "qas": [{ "question": "What is this context?", "id": "0" }], }] model.predict(to_predict)
import flask from flask import request, jsonify from flask_cors import CORS, cross_origin from simpletransformers.question_answering import QuestionAnsweringModel model = QuestionAnsweringModel('xlmroberta', './trained_model/', use_cuda=False, args={'fp16': False}) app = flask.Flask(__name__) cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' @app.route('/predict', methods=['POST']) @cross_origin() def predict(): request_data = request.json print(request_data) predictions, raw_outputs = model.predict([{ "context": request_data.get('context', ''), "qas": [{ "question": request_data.get('question', ''), "id": "0", }], }])
# "use_early_stopping": True, "n_best_size": 3, "fp16": False, "no_save": True, "manual_seed": 4, "max_seq_length": 512, "no_save": True, "n_best_size": 10, "lazy_loading": True, # "use_multiprocessing": False, } # Create the QuestionAnsweringModel model = QuestionAnsweringModel("bert", "bert-base-cased", args=train_args, use_cuda=True, cuda_device=0) # Train the model with JSON file model.train_model("data/train.jsonl", eval_data="data/train.json") # Making predictions using the model. to_predict = [{ "context": "Other legislation followed, including the Migratory Bird Conservation Act of 1929, a 1937 treaty prohibiting the hunting of right and gray whales,\ and the Bald Eagle Protection Act of 1940. These later laws had a low cost to society—the species were relatively rare—and little opposition was raised", "qas": [{ "question": "What was the name of the 1937 treaty?", "id": "0" }],
] train_args = { 'learning_rate': 1e-5, 'num_train_epochs': 1, 'max_seq_length': 384, 'doc_stride': 128, 'overwrite_output_dir': True, 'reprocess_input_data': False, 'train_batch_size': 2, 'gradient_accumulation_steps': 8, 'save_model_every_epoch': False } model = QuestionAnsweringModel('bert', 'bert-base-cased', use_cuda=False, args=train_args) model.train_model(train_data, output_dir=None) #Prediction with open('dev-v2.0.json', 'r') as f: dev_data = json.load(f) dev_data = [item for topic in dev_data['data'] for item in topic['paragraphs']] preds = model.predict(dev_data) os.makedirs('results', exist_ok=True) submission = {pred['id']: pred['answer'] for pred in preds}
qa_test # !pip install seqeval # !pip install transformers %%time from simpletransformers.question_answering import QuestionAnsweringModel model = QuestionAnsweringModel('distilbert', '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/', args={'reprocess_input_data': True, 'overwrite_output_dir': True, 'learning_rate': 5e-5, 'num_train_epochs': 4, 'max_seq_length': 200, 'doc_stride': 64, 'fp16': False, }, use_cuda=True) model.train_model(qa_train) %%time preds = model.predict(qa_test) predic_df = pd.DataFrame.from_dict(preds) sub_df['selected_text'] = predic_df['answer'] sub_df.to_csv("submission.csv", sep=',', index=False) sub_df.head()
answers = [] answers.append({'answer_start': 1000000, 'text': '__None__'}) qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers}) output.append({'context': context.lower(), 'qas': qas}) return output qa_test = do_qa_test(test) with open('/content/gdrive/My Drive/data/test.json', 'w') as outfile: json.dump(qa_test, outfile) MODEL_PATH = '/content/gdrive/My Drive/model_deeplearning/' #MODEL_PATH = 'https://drive.google.com/drive/folders/1CkjjRb6GJENfPQqfDJgVnzwipShmy4RE?usp=sharing' model = QuestionAnsweringModel('distilbert', MODEL_PATH, args={'reprocess_input_data': True, 'overwrite_output_dir': True, 'learning_rate': 5e-5, 'num_train_epochs': 3, 'max_seq_length': 192, 'doc_stride': 64, 'fp16': False, }, use_cuda=True) model.train_model('/content/gdrive/My Drive/data/train.json') predictions = model.predict(qa_test) predictions_df = pd.DataFrame.from_dict(predictions) sub_df['selected_text'] = predictions_df['answer'] sub_df.to_csv('/content/gdrive/My Drive/sample_submission.csv', index=False) print("File submitted successfully.") #test_df.head()