def __init__(self, FLAGS, id2word, word2id, emb_matrix): """ Initializes the QA model. Inputs: FLAGS: the flags passed in from main.py id2word: dictionary mapping word idx (int) to word (string) word2id: dictionary mapping word (string) to word idx (int) emb_matrix: numpy array shape (400002, embedding_size) containing pre-traing GloVe embeddings """ QAModel.__init__(self, FLAGS, id2word, word2id, emb_matrix)
def get_predictions(self, batch): """ Return: """ starts, ends = [], [] for ckpt, FLAGS in zip(self.ckpts, self.flags): qa_model = QAModel(FLAGS, self.id2word, self.word2id, self.emb_matrix, self.id2idf, is_training=False) with tf.Session(config=self.tf_config) as session: qa_model.initialize_from_checkpoint(session, ckpt, True) pred_start_pos, pred_end_pos = qa_model.get_start_end_pos( session, batch) starts.append(pred_start_pos) ends.append(pred_end_pos) del qa_model tf.reset_default_graph() starts, _ = stats.mode(np.array(starts)) ends, _ = stats.mode(np.array(ends)) return (starts[0].astype(np.int), ends[0].astype(np.int))
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) print "qn_uuid_data" print qn_uuid_data print "#" * 100 print "context_token_data" print context_token_data print "#" * 100 print "qn_token_data" print qn_token_data with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or \ os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings timer.start("glove_getter") emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) id2idf = get_idf(os.path.abspath(FLAGS.idf_path), word2id) logger.warn("Get glove embedding of size {} takes {:.2f} s".format( FLAGS.embedding_size, timer.stop("glove_getter"))) # Print out Tensorflow version # print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ ensumble = FLAGS.ensumble print(ensumble) if not ensumble and not FLAGS.attn_layer and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --attn_layer or --train_dir") # Define train_dir if not FLAGS.experiment_name: FLAGS.experiment_name = "A_{}_E_{}_D_{}".format( FLAGS.attn_layer, FLAGS.embedding_size, FLAGS.dropout) checkptr_name = FLAGS.experiment_name + "/glove{}".format( FLAGS.embedding_size) FLAGS.train_dir = FLAGS.train_dir or\ os.path.join(EXPERIMENTS_DIR, checkptr_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True is_training = (FLAGS.mode == "train") if not ensumble: # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2idf, is_training) else: ensumbler = Ensumbler(ensumble, config, id2word, word2id, emb_matrix, id2idf) # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model qa_model.initialize_from_checkpoint(sess, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model qa_model.initialize_from_checkpoint(sess, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores f1, em = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) logger.info("Dev: F1 = {0:.3}, EM = {0:.3}".format(f1, em)) elif FLAGS.mode == "eval": if ensumble: # train train_f1, train_em = ensumbler.check_f1_em(train_context_path, train_qn_path, train_ans_path, "train", FLAGS.n_eval) # dev dev_f1, dev_em = ensumbler.check_f1_em(dev_context_path, dev_qn_path, dev_ans_path, "dev", FLAGS.n_eval) else: with tf.Session(config=config) as sess: # Load best model qa_model.initialize_from_checkpoint(sess, FLAGS.ckpt_load_dir, expect_exists=True) logger.info("Model initialzed from checkpoint") # train train_f1, train_em = qa_model.check_f1_em( sess, train_context_path, train_qn_path, train_ans_path, "train", num_samples=10, print_to_screen=False) # dev dev_f1, dev_em = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=False) logger.error("Train: F1 = {:.3}, EM = {:.3}".format( train_f1, train_em)) logger.error("Dev: F1 = {:.3}, EM = {:.3}".format(dev_f1, dev_em)) elif FLAGS.mode == "official_eval": if not ensumble: if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path" ) if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir" ) # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) if ensumble: answers_dict = ensumbler.generate_answers(qn_uuid_data, context_token_data, qn_token_data) else: with tf.Session(config=config) as sess: # Load model from ckpt_load_dir qa_model.initialize_from_checkpoint(sess, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, id2idf, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
# coding: utf-8 import telebot from qa_model import QAModel from rubert import RuBertWrapper CONFIG_PATH = 'models/rubert_cased_L-12_H-768_A-12_pt' rubert = RuBertWrapper(CONFIG_PATH) QA_PATH = 'questions_answers/qa_small.txt' qai = QAModel(QA_PATH, rubert) TOKEN = '1409080754:AAHRHF35y-BwFPTF6Mb5REuxILvAg0e0gM4' bot = telebot.TeleBot(TOKEN) START_MSG = 'Предлагаем Вам ответить на несколько вопросов по системному анализу.' @bot.message_handler(commands=['start']) def handle_start(message): qai.idx = 0 qai.session_active = True cid = message.chat.id bot.send_message(cid, START_MSG) first_question = qai.questions[qai.idx] bot.send_message(cid, first_question) @bot.message_handler(func=lambda message: True, content_types=['text']) def process_answer(m): if not qai.session_active:
def main(): print("Your TensorFlow version: %s" % tf.__version__) # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR + "/glove.6B/", "glove.6B.{}d.txt".format( FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") global_step = 1 epoch = 0 print("Beginning training loop...") # Initialize model model = QAModel(FLAGS, id2word, word2id, emb_matrix) optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.learning_rate) while FLAGS.num_epochs == 0 or epoch < FLAGS.num_epochs: epoch += 1 epoch_tic = time.time() for batch in get_batch_generator( \ word2id, train_context_path, train_qn_path, \ train_ans_path, FLAGS.batch_size, context_len=FLAGS.context_len, \ question_len=FLAGS.question_len, discard_long=True): # print(batch.ans_span) with tf.GradientTape() as tape: prob_start, prob_end = model([ batch.context_ids, batch.context_mask, batch.qn_ids, batch.qn_mask ]) # prob_start, prob_end = model(batch.context_ids, batch.context_mask, batch.qn_ids, batch.qn_mask) loss_start = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prob_start, labels=batch.ans_span[:, 0]) loss_start = tf.reduce_mean(loss_start) loss_end = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prob_end, labels=batch.ans_span[:, 1]) loss_end = tf.reduce_mean(loss_end) loss = loss_start + loss_end # print("loss %f" % (loss.numpy())) grads = tape.gradient(loss, model.variables) optimizer.apply_gradients( grads_and_vars=zip(grads, model.variables)) if global_step % FLAGS.eval_every == 0: print("==== start evaluating ==== ") dev_f1, dev_em = evaluate(model, word2id, FLAGS, dev_context_path, dev_qn_path, dev_ans_path) print("Epoch %d, Iter %d, Dev F1 score: %f, Dev EM score: %f" % (epoch, global_step, dev_f1, dev_em)) print("==========================") global_step += 1 epoch_toc = time.time() print("End of epoch %i. Time for epoch: %f" % (epoch, epoch_toc - epoch_tic)) sys.stdout.flush()
os.path.abspath(__file__)))) # relative path of the main directory DEFAULT_DATA_DIR = os.path.join(MAIN_DIR, "data") # relative path of data dir EXPERIMENTS_DIR = os.path.join( MAIN_DIR, "experiments") # relative path of experiments dir # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True print "glove_path" FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) print FLAGS.glove_path emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) def initialize_model(session, model, train_dir, expect_exists): """ Initializes model from train_dir. Inputs: session: TensorFlow session model: QAModel train_dir: path to directory where we'll look for checkpoint expect_exists: If True, throw an error if no checkpoint is found. If False, initialize fresh model if no checkpoint is found. """ print "Looking for model at %s..." % train_dir ckpt = tf.train.get_checkpoint_state(train_dir)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory global bestmodel_dir bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings global emb_matrix, word2id, id2word emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Initialize model global qa_model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings global config config = tf.ConfigProto() config.gpu_options.allow_growth = True global sess sess = tf.Session(config=config) global global_context global_context = 'INSOFE has awarded over Rs 3.2 Crores in merit scholarships in the last 2 years alone. INSOFE recognizes top performers and rewards them for demonstrating outstanding achievement at every phase of the program based on their performance and eligibility criteria. At each phase of the program, top performers are awarded rankings based on which scholarship winners are announced. Top performers can potentially win scholarships ranging from Rs 25,000 to entire program fee and this can be attained on the successful completion of the program.' global global_context_list global_context_list = [ 'INSOFE has awarded over Rs 3.2 Crores in merit scholarships in the last 2 years alone. INSOFE recognizes top performers and rewards them for demonstrating outstanding achievement at every phase of the program based on their performance and eligibility criteria. At each phase of the program, top performers are awarded rankings based on which scholarship winners are announced. Top performers can potentially win scholarships ranging from Rs 25,000 to entire program fee and this can be attained on the successful completion of the program.', 'INSOFE is working on developing a video surveillance tool with enhanced smart capabilities. The tool identifies the violation and sends out instant automated response without requiring any manual interference. Since the current process involves manually going through the footage and checking for violations, it is not only a time-consuming process but also requires manual hours and effort. The tool makes the entire process automated with an Embedded Machine Learning chip Question', 'Dr Dakshinamurthy, is the Founder and President of INSOFE. He did his PhD in Materials Science and Engineering from Carnegie Mellon University. He is known for simplifying complex ideas and communicating them clearly and excitingly. Dr Sridhar Pappu is the Executive VP - Academics of INSOFE. He leads the academic administration of the institute and ensures the highest standards in learning for the students. He teaches statistics. He loves data, soo much that he wears two fitness trackers.' ] # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # app.run(host='0.0.0.0', port=443, ssl_context=('/home/gem/.ssh/certificate.pem', '/home/gem/.ssh/private-key.pem')) app.run( host='0.0.0.0', port=443, ssl_context= ('/etc/letsencrypt/live/gem.eastus2.cloudapp.azure.com/fullchain.pem', '/etc/letsencrypt/live/gem.eastus2.cloudapp.azure.com/privkey.pem'))
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_competition_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") if not FLAGS.attention_type or not FLAGS.reduction_type: raise Exception( "You have to specify both --attention_type (dot_product, bidaf, self_attention) and --reduction_type (max, mean) to proceed." ) FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") bestmodel_dir_dev_loss = os.path.join(FLAGS.train_dir, "best_checkpoint_dev_loss") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_headline_path = os.path.join(FLAGS.data_dir, "train.headline") train_body_path = os.path.join(FLAGS.data_dir, "train.body") train_ans_path = os.path.join(FLAGS.data_dir, "train.stance") dev_headline_path = os.path.join(FLAGS.data_dir, "dev.headline") dev_body_path = os.path.join(FLAGS.data_dir, "dev.body") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.stance") test_headline_path = os.path.join(FLAGS.data_dir, "test.headline") test_body_path = os.path.join(FLAGS.data_dir, "test.body") test_ans_path = os.path.join(FLAGS.data_dir, "test.stance") # Initialize model qa_model = QAModel( FLAGS, id2word, word2id, emb_matrix ) # create entire computation graph, add loss, opimizer etc... # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True attention_type = FLAGS.attention_type reduction_type = FLAGS.reduction_type if attention_type == 'dot_product' and reduction_type == 'max': FLAGS.max_gradient_norm = 10.0 if attention_type == 'bidaf': FLAGS.hidden_size = 120 if attention_type == 'self_attention': FLAGS.self_attn_zsize = 60 FLAGS.hidden_size = 70 # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) if not os.path.exists(bestmodel_dir_dev_loss): os.makedirs(bestmodel_dir_dev_loss) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.custom_train(sess, train_body_path, train_headline_path, train_ans_path, dev_headline_path, dev_body_path, dev_ans_path) elif FLAGS.mode == "check_eval": if FLAGS.ckpt_load_dir == "": raise Exception( "For check_eval mode, you need to specify --ckpt_load_dir") with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) dev_score = qa_model.check_score_cm(sess, dev_body_path, dev_headline_path, dev_ans_path, "dev", num_samples=0) print("Dev score:=>", dev_score) test_score = qa_model.check_score_cm(sess, test_body_path, test_headline_path, test_ans_path, "test", num_samples=0) print("Test score:=>", test_score) elif FLAGS.mode == "official_competition_eval": if FLAGS.ckpt_load_dir == "": raise Exception( "For official_competition_eval mode, you need to specify --ckpt_load_dir" ) competition_dataset = DataSet( name="competition_test", path=FLAGS.data_dir) # Dataset competition read from csv. #Retreive list of body/ article ids for competition dataset comp_body_ids = list( competition_dataset.articles.keys()) # get a list of article ids #Retrieve stance for all body ids comp_stances_list = get_stances(competition_dataset, comp_body_ids) # get body and headline tokens body_token_data_list, headline_token_data_list, input_body_id_list, headline_list = get_preprocessed_data( competition_dataset, comp_stances_list, 'competition') with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) #As text not number pred_label_answer_list = get_answers(sess, qa_model, word2id, body_token_data_list, headline_token_data_list) #stance_df = pd.DataFrame() #stance_df['Stance'] = pred_label_answer_list #stance_df.to_csv(os.path.join(FLAGS.result_output_path,"stance.csv"), index=False) np.savetxt(os.path.join(FLAGS.result_output_path, "stance.csv"), pred_label_answer_list, delimiter="\n", fmt='%s') else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): #First check the FLAGS enter correctly (format), python version and tensorflow version #Then check if train_dir or experiment_dir defined #set bestmodel path which named best_checkpoint #set context, question. ans path #read glove #Initialise the model architecture #gpu setting #mode choice if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) if sys.version_info[0] != 3: raise Exception( "ERROR: You must use Python 3 but you are running Python %i" % sys.version_info[0]) print( "This code was developed and tested on TensorFlow 1.8.0. Your TensorFlow version: %s" % tf.__version__) if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) #glove path emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) char2id, id2char = get_char_embed() #path for context, question, ans_span train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") #Initialise the model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2char, char2id) # Some GPU settings config = tf.ConfigProto() #set configuration for sess.run config.gpu_options.allow_growth = True #make gpu storage usage based for condition #different modes if FLAGS.mode == "train": #setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump( FLAGS.flag_values_dict(), fout ) #NoteL changed from FLAGS.__flags to FLAGS.flag_values_dict() after tensorflow 1.5 # Make bestmodel dir if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: #Added tfdbg # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) #Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) """ elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception("For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception("For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print ("Writing predictions to %s..." % FLAGS.json_out_path) with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print ("Wrote predictions to %s" % FLAGS.json_out_path) """ else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Initialize bestmodel directory bestmodel_dir = os.path.join( "F:\Sid\Learnings\Data Scientist\Text Analysis\Machine Comprehension\data\ModelLogs\best_weights", "best_checkpoint") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception("ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception("You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config=tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception("For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception("For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def _init_state(self) -> None: """ Initialize the state and load it from an existing checkpoint if any """ job_env = submitit.JobEnvironment() if job_env.global_rank == 0: # config_path = Path(args.save_folder) / str(job_env.job_id) / 'config.json' os.makedirs(self._train_cfg.output_dir, exist_ok=True) config_path = Path(self._train_cfg.output_dir) / 'config.json' with open(config_path, "w") as g: g.write(json.dumps(self._train_cfg._asdict())) print(f"Setting random seed {self._train_cfg.seed}", flush=True) random.seed(self._train_cfg.seed) np.random.seed(self._train_cfg.seed) torch.manual_seed(self._train_cfg.seed) print("Create data loaders", flush=True) tokenizer = AutoTokenizer.from_pretrained(self._train_cfg.model_name) collate_fc = partial(rank_collate, pad_id=tokenizer.pad_token_id) train_set = RankingDataset(tokenizer, self._train_cfg.train_file, self._train_cfg.max_seq_len, self._train_cfg.max_q_len, train=True) train_sampler = MhopSampler(train_set, num_neg=self._train_cfg.neg_num) batch_size_per_gpu = (1 + self._train_cfg.neg_num) * self._train_cfg.num_q_per_gpu n_gpu = torch.cuda.device_count() print(f"Number of GPUs: {n_gpu}", flush=True) print(f"Batch size per node: {batch_size_per_gpu * n_gpu}", flush=True) self._train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size_per_gpu * n_gpu, num_workers=self._train_cfg.num_workers, collate_fn=collate_fc, sampler=train_sampler) test_set = RankingDataset(tokenizer, self._train_cfg.predict_file, self._train_cfg.max_seq_len, self._train_cfg.max_q_len) self._test_loader = torch.utils.data.DataLoader( test_set, batch_size=self._train_cfg.predict_batch_size, num_workers=self._train_cfg.num_workers, collate_fn=collate_fc ) print("Create model", flush=True) print(f"Local rank {job_env.local_rank}", flush=True) bert_config = AutoConfig.from_pretrained(self._train_cfg.model_name) model = QAModel(bert_config, self._train_cfg) model.cuda(job_env.local_rank) no_decay = ['bias', 'LayerNorm.weight'] optimizer_parameters = [ {'params': [p for n, p in model.named_parameters() if not any( nd in n for nd in no_decay)], 'weight_decay': self._train_cfg.weight_decay}, {'params': [p for n, p in model.named_parameters() if any( nd in n for nd in no_decay)], 'weight_decay': 0.0} ] if self._train_cfg.use_adam: optimizer = optim.Adam(optimizer_parameters, lr=self._train_cfg.learning_rate) else: optimizer = AdamW(optimizer_parameters, lr=self._train_cfg.learning_rate) # lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2) if self._train_cfg.fp16: model, optimizer = amp.initialize( model, optimizer, opt_level=self._train_cfg.fp16_opt_level) t_total = len(self._train_loader) // self._train_cfg.gradient_accumulation_steps * self._train_cfg.num_train_epochs warmup_steps = t_total * self._train_cfg.warmup_ratio lr_scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=warmup_steps, num_training_steps=t_total ) model = torch.nn.DataParallel(model) self._state = TrainerState( epoch=0, model=model, optimizer=optimizer, lr_scheduler=lr_scheduler, global_step=0 ) self.tb_logger = SummaryWriter(self._train_cfg.output_dir.replace("logs", "tflogs")) checkpoint_fn = osp.join(self._train_cfg.output_dir, str(job_env.job_id), "checkpoint.pth") # checkpoint_fn = osp.join(self._train_cfg.output_dir, "checkpoint.pth") if os.path.isfile(checkpoint_fn): print(f"Load existing checkpoint from {checkpoint_fn}", flush=True) self._state = TrainerState.load( checkpoint_fn, default=self._state, gpu=job_env.local_rank)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.graph") train_qn_path = os.path.join(FLAGS.data_dir, "train.instruction") train_ans_path = os.path.join(FLAGS.data_dir, "train.answer") dev_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") dev_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") dev_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") # Create vocabularies of the appropriate sizes for output answer. context_vocab_path = os.path.join( FLAGS.data_dir, "vocab%d.context" % FLAGS.context_vocabulary_size) # ans_vocab_path = os.path.join(FLAGS.data_dir, "vocab%d." % FLAGS.ans_vocabulary_size) # initialize the vocabulary. context_vocab, rev_context_vocab = create_vocabulary( context_vocab_path, train_context_path, FLAGS.context_vocabulary_size) # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, context_vocab, rev_context_vocab, context_vocab) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir # with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: # json.dump(FLAGS(sys.argv), fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": """ To show a few examples without attention map. """ with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph) eval_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") eval_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") eval_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") _, _, _, _ = qa_model.check_f1_em( sess, eval_context_path, eval_qn_path, eval_ans_path, FLAGS.file_in_path, num_samples=FLAGS.print_num, print_to_screen=True) #, summary_writer=summary_writer) # summary_writer.close() elif FLAGS.mode == "show_attention": """ To show a few examples of attention map. """ with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph) eval_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") eval_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") eval_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") qa_model.demo(sess, eval_context_path, eval_qn_path, eval_ans_path, FLAGS.file_in_path, num_samples=FLAGS.print_num, print_to_screen=True, shuffle=False) # , summary_writer=summary_writer) elif FLAGS.mode == "official_eval": with tf.Session(config=config) as sess: if FLAGS.ckpt_load_dir: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) else: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) eval_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") eval_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") eval_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") f1, em, edit_dist, rem = qa_model.check_f1_em( sess, eval_context_path, eval_qn_path, eval_ans_path, FLAGS.file_in_path, num_samples=0, print_to_screen=False, write_out=FLAGS.write_out, file_out=FLAGS.file_out_path, shuffle=False) logging.info( "F1 score: %f, EM score: %f, edit distance: %f, rough EM score: %f" % (f1, em, edit_dist, rem)) else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # if FLAGS.mode != 'loadProbs': # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir # with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: # json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=20, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == "official_eval_with_bidaf": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict, bidaf_dict, self_dict1, self_dict2, out_dict = generate_answers_with_bidaf( sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path print "Writing sims to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-bidaf', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(bidaf_dict, ensure_ascii=False))) print "Wrote sims to %s" % FLAGS.json_out_path print "Writing self sims1 to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-self1', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(self_dict1, ensure_ascii=False))) print "Wrote self sims1 to %s" % FLAGS.json_out_path print "Writing self sims2 to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-self2', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(self_dict2, ensure_ascii=False))) print "Wrote self sims2 to %s" % FLAGS.json_out_path print "Writing preds to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-preds', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(out_dict, ensure_ascii=False))) print "Wrote preds to %s" % FLAGS.json_out_path elif FLAGS.mode == 'saveProbs': if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = save_answer_probs(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'wb') as f: pickle.dump(answers_dict, f, protocol=2) # f.write(unicode(pickle.dumps(answers_dict, ensure_ascii=False))) # f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == 'loadProbs': if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) # word2id = pickle.load(open('word2id', 'rb')) # pickle.dump(word2id, open('word2id', 'wb')) print 'Loaded data' dictLists = [] for file in os.listdir('./pickles'): f = os.path.join('./pickles', file) print 'Loading predictions from ', f prob_dict = pickle.load(open(f, 'rb')) dictLists += [prob_dict] # mainDict = {} # stdiDict = {} # for probs in dictLists: # for k in dictLists[0].keys(): # stdi = 1.0 / (np.std(np.array(probs[k][0])) + np.std(np.array(probs[k][1])) + 1e-2) # stdiDict[k] = stdi # try: # mainDict[k] = (mainDict[k][0] + stdi * np.array(probs[k][0]), mainDict[k][1] + stdi* np.array(probs[k][1])) # except KeyError: # mainDict[k] = (stdi* np.array(probs[k][0]), stdi*np.array(probs[k][1])) uuid2ans = {} # maps uuid to string containing predicted answer detokenizer = MosesDetokenizer() # for k in mainDict.keys(): # start_dist = mainDict[k][0] / stdiDict[k] # end_dist = mainDict[k][1] / stdiDict[k] # # Take argmax to get start_pos and end_post, both shape (batch_size) # end_dp = np.zeros(end_dist.shape) # # start_pos = np.argmax(start_dist) # # end_pos = np.argmax(end_dist) # end_dp[-1]=end_dist[-1] # for i in range(len(end_dist)-2,-1,-1): # end_dp[i]=np.amax([end_dist[i],end_dp[i+1]]) # start_pos=np.argmax(start_dist*end_dp) # end_pos = start_pos + np.argmax(end_dist[start_pos:]) # uuid2ans[k] = (start_pos, end_pos) for k in dictLists[0].keys(): spanDict = {} for probs in dictLists: start_dist = np.array(probs[k][0]) end_dist = np.array(probs[k][1]) # Take argmax to get start_pos and end_post, both shape (batch_size) end_dp = np.zeros(end_dist.shape) end_dp[-1] = end_dist[-1] for i in range(len(end_dist) - 2, -1, -1): end_dp[i] = np.amax([end_dist[i], end_dp[i + 1]]) start_pos = np.argmax(start_dist * end_dp) end_pos = start_pos + np.argmax(end_dist[start_pos:]) try: spanDict[(start_pos, end_pos)] += [ start_dist[start_pos] * end_dist[end_pos] ] except KeyError: spanDict[(start_pos, end_pos)] = [ start_dist[start_pos] * end_dist[end_pos] ] best_span = (0, 0) best_span_votes = 0 best_span_prob = 0 for span in spanDict.keys(): if len(spanDict[span]) > best_span_votes: best_span = span best_span_votes = len(spanDict[span]) best_span_prob = max(spanDict[span]) elif len( spanDict[span] ) == best_span_votes and best_span_prob < max(spanDict[span]): best_span = span best_span_votes = len(spanDict[span]) best_span_prob = max(spanDict[span]) uuid2ans[k] = (best_span[0], best_span[1]) result = {} data_size = len(qn_uuid_data) num_batches = ((data_size - 1) / FLAGS.batch_size) + 1 batch_num = 0 print "Generating answers..." for batch in get_batch_generator(word2id, qn_uuid_data, context_token_data, qn_token_data, FLAGS.batch_size, FLAGS.context_len, FLAGS.question_len): # For each example in the batch: for ex_idx in range(FLAGS.batch_size): # Detokenize and add to dict try: uuid = batch.uuids[ex_idx] pred_start, pred_end = uuid2ans[uuid] # Original context tokens (no UNKs or padding) for this example context_tokens = batch.context_tokens[ ex_idx] # list of strings # Check the predicted span is in range assert pred_start in range(len(context_tokens)) assert pred_end in range(len(context_tokens)) # Predicted answer tokens pred_ans_tokens = context_tokens[pred_start:pred_end + 1] # list of strings result[uuid] = detokenizer.detokenize(pred_ans_tokens, return_str=True) except IndexError: pass batch_num += 1 if batch_num % 10 == 0: print "Generated answers for %i/%i batches = %.2f%%" % ( batch_num, num_batches, batch_num * 100.0 / num_batches) print "Finished generating answers for dataset." answers_dict = result # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)