def run_epoch(self, epoch, frameTrain, frameVal): """ Run 1 epoch. Train on training examples, evaluate on validation set. """ path = self.options['path'] train_losses = [] numTrain = frameTrain.shape[0] prog = Progbar(target=1 + int(numTrain / self.options["batch_size"])) for i, frameBatch in enumerate( get_minibatches(frameTrain, self.options["batch_size"])): batch = loadData(frameBatch, **(self.options)) loss, lr, gs = self.optimize(*batch) train_losses.append(loss) if (self.global_step % self.options["print_every"]) == 0: logging.info("Iteration {0}: with minibatch training l2_loss = {1:.3g} and mse of {2:.2g}"\ .format(self.global_step, loss, loss/options["batch_size"])) prog.update(i + 1, [("train loss", loss)], [("learning rate", lr), ("global step", gs)]) total_train_mse = np.sum(train_losses) / numTrain val_losses = [] if epoch >= 11: numVal = frameVal.shape[0] prog = Progbar(target=1 + int(numVal / self.options["batch_size"])) for i, frameBatch in enumerate( get_minibatches(frameVal, self.options["batch_size"])): batch = loadData(frameBatch, **(self.options)) loss = self.validate(*batch) val_losses.append(loss) prog.update(i + 1, [("validation loss", loss)]) total_val_mse = np.sum(val_losses) / numVal else: total_val_mse = -1 return total_train_mse, train_losses, total_val_mse, val_losses
def validate( self, sess, valid_dataset_address ): # only used for unseen examples, ie when you wanna check your model """ Iterate through the validation dataset and determine what the validation cost is. This method calls self.test() which explicitly calculates validation cost. How you implement this function is dependent on how you design your data iteration function :return: """ valid_cost = 0. if self.FLAGS.debug: dataset, _ = get_sample(valid_dataset_address, self.FLAGS.context_paragraph_max_length, self.FLAGS.batch_size) dataset = [ dataset ] # expecting a list of minibatches, but get sample returns a single minibatch else: dataset = get_minibatches(valid_dataset_address, self.FLAGS.context_paragraph_max_length, self.FLAGS.batch_size) for question_batch, context_batch, answer_start_batch, answer_end_batch in dataset: valid_cost += self.test(sess, question_batch, context_batch, answer_start_batch, answer_end_batch) return valid_cost
def run_epoch(self, sess, train_writer, inputs, labels, epochNum): """Runs an epoch of training. Args: sess: tf.Session() object train_writer: writer object to write to TensorBoard logs inputs: np.ndarray of shape (n_samples, n_features) labels: np.ndarray of shape (n_samples,) epochNum: the number (0-indexed) of the epoch we should run Returns: average_loss: scalar. Average minibatch loss of model on epoch. """ # Get all of the Tensorboard summary nodes merged = tf.summary.merge_all() n_batches = int(inputs.shape[0] / self.config.batch_size) + ( 1 if inputs.shape[0] % self.config.batch_size > 0 else 0) total_loss = 0 for i, (input_batch, labels_batch) in enumerate( get_minibatches([inputs, labels], self.config.batch_size, shuffle=False)): loss, output_summary = self.train_on_batch(sess, merged, input_batch, labels_batch) total_loss += loss # Add the summaries to our TensorBoard output train_writer.add_summary(output_summary, epochNum * n_batches + i) return 1.0 * total_loss / n_batches
def test(data_matrix, data_labels, saved_model_path, batch_size=1000): tf.reset_default_graph() pred, input_placeholder, labels_placeholder, _, loss_op = build_model( data_matrix, data_labels) saver = tf.train.Saver() loss_list = [] label_list = [] pred_list = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, saved_model_path) print("Model restored.") minibatches = util.get_minibatches(data_matrix, data_labels, batch_size) for tup in minibatches: pred_temp, loss, labels_temp = sess.run( [pred, loss_op, labels_placeholder], feed_dict={ input_placeholder: tup[0], labels_placeholder: tup[1] }) for i, row in enumerate(pred_temp): pred_list.append(np.where(row == max(row))[0][0]) for i, row in enumerate(labels_temp): label_list.append(np.where(row == max(row))[0][0]) loss_list.append(loss) print "Loss: " + str(np.mean(loss_list)) + "\n" util.outputConfusionMatrix(pred_list, label_list, "confusion_matrix_baseline_test") util.get_accuracy(pred_list, label_list)
def run_epoch(self, session, inputs): """Runs an epoch of training. Args: sess: tf.Session() object inputs: datasets represented as a dictionary labels: np.ndarray of shape (n_samples, n_classes) Returns: average_loss: scalar. Average minibatch loss of model on epoch. """ losses = [] n_minibatches, total_loss = 0, 0 for [question_batch, context_batch, labels_batch, q_mask_batch, p_mask_batch] in \ get_minibatches([inputs['Questions'], inputs['Paragraphs'], inputs['Labels'], inputs['Questions_masks'], inputs['Paragraphs_masks']], self.config.batch_size): start_index_loss, end_index_loss = self.optimize( session, question_batch, context_batch, labels_batch, q_mask_batch, p_mask_batch) n_minibatches += 1 losses.append([start_index_loss, start_index_loss]) mean = np.mean(losses, axis=0) logging.info( "Logged mean epoch losses: train : %f dev : %f ", mean[0], mean[1]) return losses
def evaluate(self, session, dataset): input_feed = {self.train_phase: False} output_feed = [self.loss, self.psnr, self.out] test, loader = dataset total_loss = 0. metrics = [] prog = Progbar(target=(len(test) - 1) / self.flags.batch_size + 1) for i, batch in enumerate( get_minibatches(test, self.flags.batch_size, shuffle=False)): input_feed[self.im_placeholder] = [loader(b[0]) for b in batch] input_feed[self.gt_placeholder] = [loader(b[1]) for b in batch] loss, psnr, out = session.run(output_feed, input_feed) total_loss += loss * len(batch) all_ssim = [ ssim(im - resid, gt, multichannel=True) for resid, im, gt in zip(out, input_feed[self.im_placeholder], input_feed[self.gt_placeholder]) ] metrics.extend(zip([b[0] for b in batch], psnr, all_ssim)) prog.update(i + 1, exact=[("total loss", total_loss)]) return total_loss, metrics
def get_mini(self, dataset_trainOrVal, batch_size, shuffle, span=True): feed = [dataset_trainOrVal['.ids.context'], dataset_trainOrVal['.ids.question'], dataset_trainOrVal['.ids.context.mask'], dataset_trainOrVal['.ids.question.mask'], dataset_trainOrVal['.ids.context.lens'], dataset_trainOrVal['.ids.question.lens']] if span: feed.append(dataset_trainOrVal['.span']) return util.get_minibatches(feed, batch_size, shuffle)
def run_epoch(self, sess, train, label): """ """ prog = Progbar(target=1 + int(len(train) / self.config.batch_size)) losses, grad_norms = [], [] for inputs_minibatch, labels_minibatch in get_minibatches( [train, label], self.config.batch_size): #for i, batch in enumerate(minibatches(train, label, self.config.batch_size)): loss, grad_norm = self.train_on_batch(sess, inputs_minibatch, labels_minibatch) losses.append(loss) grad_norms.append(grad_norm) prog.update(i + 1, [("train loss", loss)])
def evaluate_answer(self, session, dataset, sample=100, log=False, ds_type="dev"): """ Evaluate the model's performance using the harmonic mean of F1 and Exact Match (EM) with the set of true answer labels This step actually takes quite some time. So we can only sample 100 examples from either training or testing set. :param session: session should always be centrally managed in train.py :param dataset: a representation of our data, in some implementations, you can pass in multiple components (arguments) of one dataset to this function :param sample: how many examples in dataset we look at :param log: whether we print to std out stream """ f1 = 0. em = 0. sampled_dataset = next(get_minibatches(dataset, sample, shuffle=True)) questions, contexts, spans = sampled_dataset tic = time.time() preds = self.decode(session, [questions, contexts]) pred_spans = self._preds_to_spans(preds) toc = time.time() logger.info("evaluate_answer pred_spans: took %f secs" % (toc - tic)) logger.info("Predicted/actual spans: ") for p, a in zip(pred_spans, spans): logger.info(str(p) + " || " + str(a)) for context, pred_span, actual_span in zip(contexts, pred_spans, spans): em += self.exact_match_score(pred_span, actual_span) tokens_pred = context[pred_span[0]:pred_span[1] + 1] tokens_actual = context[actual_span[0]:actual_span[1] + 1] f1 += self.f1_score(tokens_pred, tokens_actual) if log: logger.info("{}: F1: {}, EM: {}, for {} samples".format( ds_type, f1, em, sample)) return f1, em
def fit(self, sess, saver, train_examples_raw, dev_set_raw): best_score = 0. train_examples = self.preprocess_sequence_data(train_examples_raw) dev_set = self.preprocess_sequence_data(dev_set_raw) for epoch in range(self.config.n_epochs): logger.info("Epoch %d out of %d", epoch + 1, self.config.n_epochs) # You may use the progress bar to monitor the training progress # Addition of progress bar will not be graded, but may help when debugging prog = Progbar(target=1 + int(len(train_examples) / self.config.batch_size)) # The general idea is to loop over minibatches from train_examples, and run train_on_batch inside the loop # Hint: train_examples could be a list containing the feature data and label data # Read the doc for utils.get_minibatches to find out how to use it. # Note that get_minibatches could either return a list, or a list of list # [features, labels]. This makes expanding tuples into arguments (* operator) handy ### YOUR CODE HERE (2-3 lines) for inputs_minibatch in get_minibatches( train_examples, minibatch_size=self.config.batch_size): # a,b,c=inputs_minibatch self.train_on_batch(sess, *zip(*inputs_minibatch)) # self.train_on_batch(sess, inputs, np.eye(self.config.n_classes)[labels]) ### END YOUR CODE logger.info("Evaluating on development data") token_cm, entity_scores = self.evaluate(sess, dev_set, dev_set_raw) logger.debug("Token-level confusion matrix:\n" + token_cm.as_table()) logger.debug("Token-level scores:\n" + token_cm.summary()) logger.info("Entity level P/R/F1: %.2f/%.2f/%.2f", *entity_scores) score = entity_scores[-1] if score > best_score: best_score = score if saver: logger.info("New best score! Saving model in %s", self.config.model_output) saver.save(sess, self.config.model_output) print("") if self.report: self.report.log_epoch() self.report.save() return best_score
def run_epoch(self, sess, inputs): """Runs an epoch of training. Args: sess: tf.Session() object inputs: datasets represented as a dictionary labels: np.ndarray of shape (n_samples, n_classes) Returns: average_loss: scalar. Average minibatch loss of model on epoch. """ n_minibatches, total_loss = 0, 0 for batch in get_minibatches([inputs['Questions'], inputs['Paragraphs'], inputs['Labels']], self.config.batch_size): n_minibatches += 1 total_loss += self.train_on_batch(sess, *batch) return total_loss / n_minibatches
def train(data_matrix, data_labels, save_path, title, hidden_size=256, lr=0.005, saved_model_path=None, RESUME=False, batch_size=256, n_epochs=30): tf.reset_default_graph() _, input_placeholder, labels_placeholder, train_op, loss_op = build_model( data_matrix, data_labels, hidden_size=hidden_size, lr=lr) saver = tf.train.Saver() avg_loss_list = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) if RESUME: sess.run(tf.global_variables_initializer()) saver.restore(sess, saved_model_path) print("Model restored.") minibatches = util.get_minibatches(data_matrix, data_labels, batch_size) for i in range(n_epochs): batch_loss_list = [] print "Epoch " + str(i + 1) + ": " for tup in minibatches: _, loss = sess.run([train_op, loss_op], feed_dict={ input_placeholder: tup[0], labels_placeholder: tup[1] }) batch_loss_list.append(loss) avg_loss_list.append(np.mean(batch_loss_list)) print "=====>loss: " + str(avg_loss_list[i]) + " " if (i > 0) and (avg_loss_list[i] < avg_loss_list[i - 1]): tmp_path = save_path + "--smallest loss" saver.save(sess, tmp_path) print "New min loss at epoch %s! Model saved in path %s" % ( str(i + 1), tmp_path) saver.save(sess, save_path) print("Final model saved in path: %s" % save_path) util.dumpVar("losses/ " + title + " " + today + ".pkl", avg_loss_list) generatePlots(range(len(avg_loss_list)), avg_loss_list, "Number of Epochs", "Cross-Entropy Loss", title)
def test(data_matrix, data_labels, saved_model_path, title, batch_size=256): tf.reset_default_graph() pred, input_placeholder, labels_placeholder, _, loss_op = build_model( data_matrix, data_labels) saver = tf.train.Saver() loss_list = [] label_list = [] pred_list = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, saved_model_path) print("Model restored.") minibatches = util.get_minibatches(data_matrix, data_labels, batch_size) for tup in minibatches: pred_temp, loss, labels_temp = sess.run( [pred, loss_op, labels_placeholder], feed_dict={ input_placeholder: tup[0], labels_placeholder: tup[1] }) for i, row in enumerate(pred_temp): pred_list.append(np.where(row == max(row))[0][0]) for i, row in enumerate(tup[1]): label_list.append(np.where(row == max(row))[0][0]) loss_list.append(loss) count = 0 for i in range(len(pred_list)): if pred_list[i] != label_list[i]: count += 1 print "sentence: ", reconstruct_sentence( (tup[0][i:i + 1, :]).tolist()) print "predicted label: ", pred_list[i] print "correct label: ", label_list[i] if count > 4: break print "Loss: " + str(np.mean(loss_list)) + "\n" util.outputConfusionMatrix( pred_list, label_list, "confusion matrices/confusion_matrix " + title + " " + today) util.get_accuracy(pred_list, label_list)
def generate_answers(sess, model, dataset, rev_vocab, batch_size): """ Loop over the dev or test dataset and generate answer. Note: output format must be answers[uuid] = "real answer" You must provide a string of words instead of just a list, or start and end index In main() function we are dumping onto a JSON file evaluate.py will take the output JSON along with the original JSON file and output a F1 and EM You must implement this function in order to submit to Leaderboard. :param sess: active TF session :param model: a built QASystem model :param rev_vocab: this is a list of vocabulary that maps index to actual words :return: """ answers = {} all_contexts, all_questions, all_question_uuids = dataset prog = Progbar(target=1 + int(len(dataset[0]) / batch_size)) for i, batch in enumerate( get_minibatches([all_contexts, all_questions, all_question_uuids], batch_size, shuffle=False)): contexts, questions, question_uuids = batch context_lengths = (contexts != 0).sum(1) pred_spans = model.answer(sess, [questions, contexts]) pred_spans[:, 0] = np.minimum(pred_spans[:, 0], pred_spans[:, 1]) pred_spans[:, 1] = np.minimum(pred_spans[:, 1], context_lengths) for (s, e), uuid, context in zip(pred_spans, question_uuids, contexts): answers[uuid] = " ".join( [rev_vocab[context[c]] for c in range(s, e + 1)]) prog.update(i + 1, []) return answers
def optimize(self, session, dataset, epoch): input_feed = {self.train_phase: True} output_feed = [self.train_op, self.loss, self.norm] train, loader = dataset total_loss = 0. prog = Progbar(target=(len(train) - 1) / self.flags.batch_size + 1) for i, batch in enumerate(get_minibatches(train, self.flags.batch_size)): input_feed[self.im_placeholder] = [loader(b[0]) for b in batch] input_feed[self.gt_placeholder] = [loader(b[1]) for b in batch] _, loss, norm = session.run(output_feed, input_feed) prog.update(i + 1, [("train loss", loss), ("norm", norm)]) total_loss += loss return total_loss
def optimize(self, session, dataset_address): """ Takes in actual data to optimize your model This method is equivalent to a step() function :return: """ counter = -1 #loss = 0.0 if self.FLAGS.debug: dataset, _ = get_sample(dataset_address, self.FLAGS.context_paragraph_max_length, self.FLAGS.batch_size) dataset = [ dataset ] # put in a list, becuase get_sample returns one minibatch and we want a list of minibatches else: dataset = get_minibatches(dataset_address, self.FLAGS.context_paragraph_max_length, self.FLAGS.batch_size) #with tf.variable_scope("qa"): #tf.get_variable_scope().reuse_variables() epoch_loss = 0.0 for question_batch, context_batch, answer_start_batch, answer_end_batch in dataset: answer_start_batch = unlistify( answer_start_batch ) # batch returns dim=[batch_size,1] need dim=[batch_size,] answer_end_batch = unlistify( answer_end_batch ) # batch returns dim=[batch_size,1] need dim=[batch_size,] input_feed = self.create_feed_dict(question_batch, context_batch, answer_start_batch, answer_end_batch, self.FLAGS.dropout) output_feed = [self.updates, self.loss, self.global_grad_norm] outputs = session.run(output_feed, feed_dict=input_feed) epoch_loss += np.sum(outputs[1]) global_grad_norm = outputs[2] counter = (counter + 1) % self.FLAGS.print_every if counter == 0: logging.info( "Global grad norm for update: {}".format(global_grad_norm)) return epoch_loss
def run_epoch(self, session, train_dataset, val_dataset): prog = Progbar(target=1 + int(len(train_dataset[0]) / self.config.batch_size), logger=logger) for i, train_batch in enumerate( get_minibatches(train_dataset, self.config.batch_size, shuffle=True)): _, loss, grad_norm = self.optimize(session, train_batch) prog.update(i + 1, [("train loss", loss), ("grad norm", grad_norm)]) self.evaluate_answer(session, train_dataset, log=True, ds_type="Train dataset") f1, em = self.evaluate_answer(session, val_dataset, log=True, ds_type="Dev dataset") return f1
def train(data_matrix, data_labels, save_path, batch_size=100, n_epochs=30): _, input_placeholder, labels_placeholder, train_op, loss_op = build_model( data_matrix, data_labels) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) minibatches = util.get_minibatches(data_matrix, data_labels, batch_size) for i in range(n_epochs): print "Epoch " + str(i + 1) + ": " loss_list = [] label_list = [] pred_list = [] for tup in minibatches: _, loss = sess.run([train_op, loss_op], feed_dict={ input_placeholder: tup[0], labels_placeholder: tup[1] }) loss_list.append(loss) print "=====>loss: " + str(np.mean(loss_list)) + " " save_path = saver.save(sess, save_path) print("Final baseline model saved in path: %s" % save_path)
def minibatches(self, data, shuffle=True): # batchs = [[sentences sets] [labels sets]] batches = [np.array(col) for col in zip(*data)] return get_minibatches(batches, self.config.batch_size, shuffle)
def email_minibatches(email_examples, batch_size): body_ids, labels = email_records_to_word_ids(email_examples, token_mapping, n_features) return get_minibatches([body_ids, labels], batch_size)