def eval_batches(self, loss_fn, sess, eval_set, num_batches): """Evaluate the loss on a number of given minibatches of a dataset. Args: loss_fn: loss function sess: tf.Session() eval_set: full dataset, as passed to run_epoch num_batches: number of batches to evaluate Returns: loss: loss over the batches (a scalar) """ losses = [] for i, (inputs_batch, outputs_batch) in enumerate( minibatches(eval_set, self.batch_size)): if i >= num_batches: break feed = { self.image_in: inputs_batch, self.emotion_label: outputs_batch, self.gaussian_in: np.random.normal(size=(len(inputs_batch), self.style_dim)) } loss = self.eval_on_batch(loss_fn, feed, sess) losses.append(loss) return np.mean(losses)
def predict(self, sess, dev, labels): predicted_sentences = [] dev_processed, labels_processed = self.preprocess_sequence_data( dev, labels) prog = Progbar(target=1 + int(len(dev_processed) / self.config.batch_size)) # predict in batches for i, (enc_batch, dec_batch) in enumerate( minibatches(dev_processed, labels_processed, self.config.batch_size, shuffle=False)): #print enc_batch.shape, dec_batch.shape feed = self.create_feed_dict(enc_batch, dec_batch) outputs = sess.run(self.test_op, feed_dict=feed) for j in range(outputs.shape[0]): sentence = [self.labels_vocab.id2tok[k] for k in outputs[j, :]] if "</s>" in sentence: sentence = sentence[:sentence.index("</s>")] predicted = " ".join(sentence) print predicted predicted_sentences.append(predicted) prog.update(i + 1) return predicted_sentences
def predict(self, sess, dev): predicted_sentences = [] dev_processed, labels_processed = self.preprocess_sequence_data( dev[0], dev[1]) prog = Progbar(target=int(len(dev_processed) / self.config.batch_size)) # predict in batches tot_loss = 0 for i, (enc_batch, dec_batch) in enumerate( minibatches(dev_processed, labels_processed, self.config.batch_size, False)): #print enc_batch.shape, dec_batch.shape feed = self.create_feed_dict(enc_batch, dec_batch) if self.beam_search != None: outputs = sess.run(self.test_op, feed_dict=feed) else: outputs, loss = sess.run([self.test_op, self.predict_loss], feed_dict=feed) #loss = 0 #outputs = sess.run(self.test_op, feed_dict = feed) tot_loss += loss for j in range(outputs.shape[0]): predicted = self.construct_sentence( outputs[j, :], dev[2][(i * self.config.batch_size) + j]) predicted_sentences.append(predicted) #print predicted if self.beam_search != None: prog.update(i + 1) else: prog.update(i + 1, [("test loss", tot_loss / (i + 1))]) print "" return predicted_sentences
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw, dev_set_raw, epoch, last_epoch): prog = Progbar(target=1 + int(len(train_examples) / self.config.batch_size)) curr_loss = 0. num_encountered = 0 for i, batch in enumerate(minibatches(train_examples, self.config.batch_size)): loss = self.train_on_batch(sess, *batch) curr_loss += loss prog.update(i + 1, [("train loss", loss)]) if self.report: self.report.log_train_loss(loss) num_encountered +=1 # print curr_loss/num_encountered losses.append(curr_loss/num_encountered) epochs.append(epoch+1) print("") logger.info("Evaluating on development data") token_cm, metrics = self.evaluate(sess, dev_set, dev_set_raw) if last_epoch: self.outputConfusionMatrix(token_cm.as_data()) logger.debug("Token-level confusion matrix:\n" + token_cm.as_table()) logger.debug("Token-level scores:\n" + token_cm.summary()) logger.info("Accuracy: %.2f", metrics[1]) logger.info("Error: %.2f", metrics[0]) return metrics[0], metrics[1]
def run_epoch(self, tf_ops, loss_fns, sess, train_examples, dev_set, batch_size, logfile=None): for i, (inputs_batch, outputs_batch) in enumerate( minibatches(train_examples, batch_size)): feed = { self.image_in: inputs_batch, self.emotion_label: outputs_batch, self.gaussian_in: np.random.normal(size=(len(inputs_batch), self.style_dim)) } self.train_on_batch(tf_ops, feed, sess) dev_loss_sum = 0 for (loss_fn, loss_name) in loss_fns: train_loss = self.eval_batches(loss_fn, sess, train_examples, self.n_eval_batches) print("Train {} loss: {:.6f}".format(loss_name, train_loss)) dev_loss = self.eval_batches(loss_fn, sess, dev_set, self.n_eval_batches) print("Dev {} loss: {:.6f}".format(loss_name, dev_loss)) dev_loss_sum += dev_loss print("") return dev_loss_sum
def run_epoch(self, sess, train, labels, dev=None, dev_labels=None): prog = Progbar(target=1 + int(len(train) / self.config.batch_size)) for i, (enc_batch, dec_batch) in enumerate( minibatches(train, labels, self.config.batch_size)): loss = self.train_on_batch(sess, enc_batch, dec_batch) prog.update(i + 1, [("train loss", loss)]) logger.info("Finished Epoch! Running Dev tests") if dev != None and dev_labels != None: return self.dev_loss(sess, dev, dev_labels) return loss
def get_gaussians(self, sess, dev_set, num_samples, output_path): for i, (inputs_batch, outputs_batch) in enumerate(minibatches(dev_set, num_samples)): feed = { self.image_in: inputs_batch, self.emotion_label: outputs_batch } style = np.array(sess.run(self.gen_styles, feed_dict=feed)) break save_path = os.path.join(output_path, "gaussians.pkl") pickle.dump(style, open(save_path, "wb"))
def output(self, sess, inputs_raw, inputs=None): if inputs is None: inputs = self.preprocess_sequence_data(self.helper.vectorize(inputs_raw)) preds = [] prog = Progbar(target=1 + int(len(inputs) / self.config.batch_size)) for i, batch in enumerate(minibatches(inputs, self.config.batch_size, shuffle=False)): batch = batch[:1] + batch[2:] preds_ = self.predict_on_batch(sess, *batch) preds += list(preds_) prog.update(i + 1, []) return self.consolidate_predictions(inputs_raw, inputs, preds)
def predict_on_batch(self, session, dataset): batch_num = int(np.ceil(len(dataset) * 1.0 / self.config.batch_size)) # prog = Progbar(target=batch_num) predicts = [] for i, batch in tqdm( enumerate( minibatches(dataset, self.config.batch_size, shuffle=False))): pred = self.answer(session, batch) # prog.update(i + 1) predicts.extend(pred) return predicts
def run_epoch(self, sess, train, labels): prog = Progbar(target=1 + int(len(train) / self.config.batch_size)) losses, grad_norms = [], [] for i, (enc_batch, dec_batch) in enumerate( minibatches(train, labels, self.config.batch_size)): #print batch loss = self.train_on_batch(sess, enc_batch, dec_batch) #losses.append(loss) #grad_norms.append(grad_norm) #loss = 1 prog.update(i + 1, [("train loss", loss)]) return loss
def dev_loss(self, sess, dev_processed, labels_processed): prog = Progbar(target=1 + int(len(dev_processed) / self.config.batch_size)) tot_loss = 0 for i, (enc_batch, dec_batch) in enumerate( minibatches(dev_processed, labels_processed, self.config.batch_size, shuffle=False)): feed = self.create_feed_dict(enc_batch, labels_batch=dec_batch) _, loss = sess.run([self.test_op, self.predict_loss], feed_dict=feed) tot_loss += loss prog.update(i + 1, [("dev loss", tot_loss)]) return tot_loss
def get_reconstructions(self, sess, dev_set, num_samples, output_path): for i, (inputs_batch, outputs_batch) in enumerate(minibatches(dev_set, num_samples)): feed = { self.image_in: inputs_batch, self.emotion_label: outputs_batch } outputs = np.array( sess.run(self.gen_images_autoencode, feed_dict=feed)) break for i in range(len(outputs)): imsave(os.path.join(output_path, "{}.png".format(i)), np.squeeze(inputs_batch[i])) imsave(os.path.join(output_path, "{}_recon.png".format(i)), np.squeeze(outputs[i]))
def get_reconstructions(self, sess, dev_set, num_samples, output_path): for i, (inputs_batch, outputs_batch) in enumerate( minibatches(dev_set, num_samples, shuffle=False)): feed = { self.image_in: inputs_batch, self.emotion_label: outputs_batch } outputs = np.array( sess.run(self.gen_images_autoencode, feed_dict=feed)) break if not os.path.exists(output_path): os.makedirs(output_path) for i in range(len(outputs)): im = toimage(np.squeeze(inputs_batch[i]), cmin=0, cmax=1) im.save(os.path.join(output_path, "{}.png".format(i))) im = toimage(np.squeeze(outputs[i]), cmin=0, cmax=1) im.save(os.path.join(output_path, "{}_recon.png".format(i)))
def output(self, sess, inputs_raw, inputs): """ Reports the output of the model on examples (uses helper to featurize each example). """ # always require valid inputs arg # if inputs is None: # inputs = self.preprocess_sequence_data(self.helper.vectorize(inputs_raw)) preds = [] prog = Progbar(target=1 + int(len(inputs) / self.config.batch_size)) for i, batch in enumerate( minibatches(inputs, self.config.batch_size, shuffle=False)): # Ignore predict batch = batch[:2] + batch[3:] preds_ = self.predict_on_batch(sess, *batch) preds += list(preds_) prog.update(i + 1, []) return self.consolidate_predictions(inputs_raw, inputs, preds)
def eval_batches(self, sess, eval_set, num_batches): """Evaluate the loss on a number of given minibatches of a dataset. Args: sess: tf.Session() eval_set: full dataset, as passed to run_epoch num_batches: number of batches to evaluate Returns: loss: loss over the batches (a scalar) """ losses = [] for i, (inputs_batch, outputs_batch) in enumerate( minibatches(eval_set, self.config.batch_size)): if i >= num_batches: break loss = self.eval_on_batch(sess, inputs_batch, outputs_batch) losses.append(loss) return np.mean(losses)
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw, dev_set_raw): prog = Progbar(target=1 + int(len(train_examples) / self.config.batch_size)) for i, batch in enumerate( minibatches(train_examples, self.config.batch_size)): loss = self.train_on_batch(sess, *batch) prog.update(i + 1, [("train loss", loss)]) if self.report: self.report.log_train_loss(loss) print("") logger.info("Evaluating on development data") token_cm, entity_scores = self.evaluate(sess, dev_set, dev_set_raw) logger.debug("Token-level confusion matrix:\n" + token_cm.as_table()) logger.debug("Token-level scores:\n" + token_cm.summary()) logger.info("Entity level P/R/F1: %.2f/%.2f/%.2f", *entity_scores) f1 = entity_scores[-1] return f1
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw, dev_set_raw, epoch): prog = Progbar(target=1 + int(len(train_examples) / self.config.batch_size)) curr_loss = 0. num_encountered = 0 for i, batch in enumerate(minibatches(train_examples, self.config.batch_size)): loss = self.train_on_batch(sess, *batch) prog.update(i + 1, [("train loss", loss)]) curr_loss += loss num_encountered += 1 if self.report: self.report.log_train_loss(loss) train_loss.append(curr_loss/num_encountered) print("") logger.info("Evaluating on development data") divergence = self.evaluate(sess, dev_set, dev_set_raw) logger.info("KL divergence: %.2f", divergence) dev_loss.append(divergence) return divergence
def evaluate(self, sess, examples, examples_raw): global pred_latest global gold_latest global example_latest avg_div = 0.0 seen = 0 for i, batch in enumerate(minibatches(examples, self.config.batch_size, shuffle=False)): pred_label = self.predict_on_batch(sess, batch[0]) gold_label = batch[1] divergence = np_kl_divergence(gold_label, pred_label) pred_latest = pred_label[:40] gold_latest = gold_label[:40] example_latest = [example[0] for example in examples_raw[:40]] avg_div += divergence seen += 1 return avg_div/seen
def run_epoch(self, sess, train_examples, dev_set, logfile=None): prog = Progbar(target=1 + train_examples[0].shape[0] / self.config.batch_size) for i, (inputs_batch, outputs_batch) in enumerate( minibatches(train_examples, self.config.batch_size)): loss = self.train_on_batch(sess, inputs_batch, outputs_batch, get_loss=True) prog.update(i + 1, [("train loss", loss)]) print("") print("Evaluating on train set...") train_loss = self.eval_batches(sess, train_examples, self.config.n_eval_batches) print("Train Loss: {0:.6f}".format(train_loss)) print("Evaluating on dev set...") dev_loss = self.eval_batches(sess, dev_set, self.config.n_eval_batches) print("Dev Loss: {0:.6f}".format(dev_loss)) logfile.write(",{0:.5f},{1:.5f}\n".format(float(train_loss), float(dev_loss))) return dev_loss
def run_epoch(self, session, epoch_num, training_set, vocab, validation_set, sample_size=400): set_num = len(training_set) batch_size = self.config.batch_size batch_num = int(np.ceil(set_num * 1.0 / batch_size)) sample_size = 400 prog = Progbar(target=batch_num) avg_loss = 0 for i, batch in enumerate( minibatches(training_set, self.config.batch_size, shuffle=False)): global_batch_num = batch_num * epoch_num + i _, summary, loss = self.optimize(session, batch) prog.update(i + 1, [("training loss", loss)]) if self.config.tensorboard and global_batch_num % self.config.log_batch_num == 0: self.train_writer.add_summary(summary, global_batch_num) if (i + 1) % self.config.log_batch_num == 0: logging.info('') self.evaluate_answer(session, training_set, vocab, sample=sample_size, log=True) self.evaluate_answer(session, validation_set, vocab, sample=sample_size, log=True) avg_loss += loss avg_loss /= batch_num logging.info("Average training loss: {}".format(avg_loss)) return avg_loss
def predict(self, sess, dev, labels): predicted_sentences = [] dev_processed, labels_processed = self.preprocess_sequence_data( dev, labels) print self.config.batch_size for i, (enc_batch, dec_batch) in enumerate( minibatches(dev_processed, labels_processed, self.config.batch_size, shuffle=False)): print enc_batch.shape, dec_batch.shape feed = self.create_feed_dict(enc_batch, dec_batch) outputs = sess.run(self.pred_proj, feed_dict=feed) for i in range(outputs.shape[0]): predicted = " ".join( [self.labels_vocab.id2tok[i] for i in outputs[i, :]]) print predicted predicted_sentences.append(predicted) return predicted_sentences
def predict(self, sess, dev, labels): predicted_sentences = [] dev_processed, labels_processed = self.preprocess_sequence_data( dev, labels) prog = Progbar(target=1 + int(len(dev_processed) / self.config.batch_size)) # predict in batches for i, (enc_batch, dec_batch) in enumerate( minibatches(dev_processed, labels_processed, self.config.batch_size, shuffle=False)): #print enc_batch.shape, dec_batch.shape feed = self.create_feed_dict(enc_batch, dec_batch) outputs = sess.run(self.test_op, feed_dict=feed) for j in range(outputs.shape[0]): predicted = self.construct_sentence(outputs[j, :]) #print predicted predicted_sentences.append(predicted) prog.update(i + 1) return predicted_sentences
def validate(self, sess, valid_dataset): """ Iterate through the validation dataset and determine what the validation cost is. This method calls self.test() which explicitly calculates validation cost. How you implement this function is dependent on how you design your data iteration function :return: """ batch_num = int( np.ceil(len(valid_dataset) * 1.0 / self.config.batch_size)) prog = Progbar(target=batch_num) avg_loss = 0 for i, batch in enumerate( minibatches(valid_dataset, self.config.batch_size)): loss = self.test(sess, batch)[0] prog.update(i + 1, [("validation loss", loss)]) avg_loss += loss avg_loss /= batch_num logging.info("Average validation loss: {}".format(avg_loss)) return avg_loss