def predict(self, session, X, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 dp = 1 losses = [] results = [] if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(X, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) for step, (x, y) in enumerate(data): feed = self.create_feed_dict(input_batch=x, dropout=dp) if np.any(y): feed[self.labels_placeholder] = y loss, preds = session.run([self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) return np.mean(losses), results
def run_epoch(self, sess, data, train_op=None, verbose=10): dropout = self.config.dropout if train_op == None: train_op = tf.no_op() dropout = 1 total_steps = sum(1 for x in data_iterator( data, self.config.batch_size, self.config.num_steps)) total_loss = [] state = self.initial_state.eval() for step, (x, y) in enumerate( data_iterator(data, self.config.batch_size, self.config.num_steps)): feed = { self.input_placeholder: x, self.label_placeholder: y, self.dropout_placeholder: dropout, self.initial_state: state } loss, state, _ = sess.run( [self.calculate_loss, self.final_state, train_op], feed_dict=feed) total_loss.append(loss) if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : Avg. Loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') return np.mean(total_loss)
def predict(self, session, X, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 dp = 1 losses = [] results = [] if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(X, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) for step, (x, y) in enumerate(data): feed = self.create_feed_dict(input_batch=x, dropout=dp) if np.any(y): feed[self.labels_placeholder] = y loss, preds = session.run( [self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) return np.mean(losses), results
def predict(self, session, X, y=None): dp = 1 losses = [] results = [] if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(X, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) for step, (x, y) in enumerate(data): feed = self.create_feed_dict(input_batch=x, dropout=dp) if np.any(y): feed[self.labels_placeholder] = y loss, preds = session.run([self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) return np.mean(losses), results
def predict(self, session, X, y=None, verbose=False): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 dp = 1 losses = [] results = [] if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, tagset_size=self.tagset_size) else: data = data_iterator(X, batch_size=self.config.batch_size, tagset_size=self.tagset_size) for step, (x, y) in enumerate(data): feed = self.create_feed_dict(input_batch=x, dropout=dp) if np.any(y): feed[self.labels_placeholder] = y loss, preds = session.run( [self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) if verbose: print 'SENTENCE %d:' % step sentences, _, labels = print_labels(x, preds, y, self.num_to_word, self.num_to_tag) print sentences print labels return np.mean(losses), results
def predict(self, session, X, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 dp = 1 losses = [] results = [] z = 1 if np.any(y): #It would evaluate to false if all y are zero!!!!! data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) #print 'np.any(y) seems to be TRUE' #print type(data) #print data.shape else: data = data_iterator(X, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) #print 'np.any(y) seems to be FALSE' #print type(data) #print data.shape for step, (x, y) in enumerate( data): #Howz it working when np.any(y) is False feed = self.create_feed_dict(input_batch=x, dropout=dp) #print 'x.shape in predict function, followed by feed.input_placeholder.shape. I guess both should be same' #print x.shape #print feed[self.input_placeholder].shape if np.any(y): #print 'y>>>: ' #print y feed[self.labels_placeholder] = y loss, preds = session.run([self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) # tf.nn.softmax(y) # y = self.add_model(window) # window = self.add_embedding() # predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) if len(losses) == 0: return None, results return np.mean(losses), results
def predict(self, session, input_data, disallow_other=False): dp = 1. l2 = 0. losses = [] results = [] data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length) for step, (lines, line_lengths, labels) in enumerate(data): try: self.index_to_weight except: if self.config.weight_loss: # f = np.log f = lambda x: np.sqrt(x) normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()]) self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()} else: self.index_to_weight = {k:1. for k in range(len(self.speakers))} feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels]) loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed) if disallow_other: preds[:,self.speakers.speaker_to_index["OTHER"]] = 0. predicted_indices = preds.argmax(axis=1) losses.append(loss) results.extend(predicted_indices) return np.mean(losses), results
def run_epoch(self, session, input_data, shuffle=False, verbose=True): dp = self.config.dropout l2 = self.config.l2 # We're interested in keeping track of the loss and accuracy during training total_loss = [] total_correct_examples = 0 total_processed_examples = 0 if self.config.batch_size == "chapter": total_steps = len(input_data) else: total_steps = len(input_data) / self.config.batch_size data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length, shuffle=shuffle) for step, (lines, line_lengths, labels) in enumerate(data): if self.config.weight_loss: # f = np.log f = lambda x: np.sqrt(x) normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()]) self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()} feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels]) else: self.index_to_weight = {k:1. for k in range(len(self.speakers))} feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2) loss, total_correct, _ = session.run([self.loss, self.correct_predictions, self.train_op], feed_dict=feed) total_processed_examples += len(labels) total_correct_examples += total_correct total_loss.append(loss) ## if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format(step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True): loss_history = [] var_objective_history = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = len(X) / self.batch_size for step, (X_batch, y_batch) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=shuffle)): #supervised feed = self.create_feed_dict(input_batch=X_batch, label_batch=y_batch) loss, total_correct, _ = session.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed) total_processed_examples += len(X_batch) total_correct_examples += total_correct loss_history.append(loss) #unsupervised var_objective, accuracy = self.unsup_update(trees[step]) var_objective_history.append(var_objective) if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}, var_objective = {}'.format( step, total_steps, np.mean(loss_history), np.mean(var_objective))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return loss_history, total_correct_examples / float(total_processed_examples)
def decoding(loaded_model, test_dataset, arg_parser): beam_size = arg_parser.beam_size max_len = arg_parser.max_decode_len decoding_method = loaded_model.beam_search if arg_parser.decoding == 'beam_search' else loaded_model.decode_greedy loaded_model.eval() model_outputs = [] model_outputs_kb = [] gold_queries_kb = [] gold_queries = [] with torch.no_grad(): for src_sent_batch, gold_target in tqdm(data_iterator(test_dataset, batch_size=1, shuffle=False), total=280): example_hyps = decoding_method(src_sent=src_sent_batch, max_len=max_len, beam_size=beam_size) strings_model = [ detokenize(example_hyp) for example_hyp in example_hyps ] string_gold = gold_target[0] model_outputs_kb.append(strings_model) gold_queries_kb.append(string_gold) strings_model, string_gold = format_lf(strings_model, string_gold) model_outputs.append(strings_model) gold_queries.append(string_gold) return model_outputs, gold_queries, model_outputs_kb, gold_queries_kb
def run_epoch(self, sess, input_data, input_labels): """Runs an epoch of training. Trains the model for one-epoch. Args: sess: tf.Session() object input_data: np.ndarray of shape (n_samples, n_features) input_labels: np.ndarray of shape (n_samples, n_classes) Returns: average_loss: scalar. Average minibatch loss of model on epoch. """ # And then after everything is built, start the training loop. average_loss = 0 for step, (input_batch, label_batch) in enumerate( data_iterator(input_data, input_labels, batch_size=self.config.batch_size, label_size=self.config.n_classes)): # Fill a feed dictionary with the actual set of images and labels # for this particular training step. feed_dict = self.create_feed_dict(input_batch, label_batch) # Run one step of the model. The return values are the activations # from the `self.train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value = sess.run([self.train_op, self.loss], feed_dict=feed_dict) average_loss += loss_value average_loss = average_loss / step return average_loss
def run_epoch(self, session, input_data, input_labels, shuffle=True, verbose=True): orig_X, orig_y = input_data, input_labels dp = self.config.dropout # We're interested in keeping track of the loss and accuracy during training total_loss = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = len(orig_X) / self.config.batch_size for step, (x, y) in enumerate( data_iterator(orig_X, orig_y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=shuffle)): feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y) loss, total_correct, _ = session.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed) total_processed_examples += len(x) total_correct_examples += total_correct total_loss.append(loss) ## if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def run_epoch(self, session, sent1_data, sent2_data, len1_data, len2_data, input_labels, verbose=100): orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y = ( sent1_data, sent2_data, len1_data, len2_data, input_labels ) kp = self.config.kp total_loss = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = int( orig_sent1.shape[0] / self.config.batch_size) for step, (sent1, sent2, len1, len2, y) in enumerate( data_iterator(orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y, batch_size=self.config.batch_size, label_size=self.config.label_size)): feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp) loss, total_correct, _ = session.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed) total_processed_examples += len(y) total_correct_examples += total_correct total_loss.append(loss) if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return np.mean(total_loss), total_correct_examples / float(total_processed_examples), total_loss
def run_epoch(self, session, input_data, input_labels, shuffle=True, verbose=True): orig_X, orig_y = input_data, input_labels # 定义数据 x,y,dropout dp = self.config.dropout # We're interested in keeping track of the loss and accuracy during training total_loss = [] total_correct_examples = 0 # 计算分类准确的比例:correct/total processed total_processed_examples = 0 total_steps = len(orig_X) / self.config.batch_size # X的总size/每批的size 得到批次数 for step, (x, y) in enumerate( data_iterator(orig_X, orig_y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=shuffle)): feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y) loss, total_correct, _ = session.run( # 前面都是定义,这里才是执行 [self.loss, self.correct_predictions, self.train_op], # 1.喂进去x y dropout,执行 loss,correct_predictions,train_op feed_dict=feed) # train后,参数W等就已经被更新了 total_processed_examples += len(x) total_correct_examples += total_correct total_loss.append(loss) ## if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return np.mean(total_loss), total_correct_examples / float(total_processed_examples) # 返回 mean(loss) 和 准确度
def run_epoch(self, sess, input_data, input_labels): average_loss = 0 for step, (input_batch, label_batch) in enumerate(data_iterator(input_data,input_labels,batch_size = self.config.batch_size, label_size=self.config.n_classes)): feed_dict = self.create_feed_dict(input_batch, label_batch) _, loss_value = sess.run([self.train_op, self.loss],feed_dict = feed_dict) average_loss += loss_value average_loss = average_loss / step return average_loss
def predict(self, session, X, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 dp = 1 losses = [] results = [] if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False)
def predict(self, session, sent1_data, sent2_data, len1_data, len2_data, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 kp = 1.0 losses = [] results = [] if np.any(y): data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) for step, (sent1, sent2, len1, len2, y) in enumerate(data): feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp) if np.any(y): loss, preds = session.run([self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) results.extend(preds) return np.mean(losses), np.array(results)
def predict(self, session, X, y=None): losses = [] results = [] for step, (x, y) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=False)): feed = self.create_feed_dict(input_batch=x, label_batch=y) if np.any(y): loss, preds = session.run( [self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) results.extend(preds) return results, np.mean(losses)
def run_epoch(self, sess, input_data, input_labels, shuffle=True, verbose=True): """Runs an epoch of training. Trains the model for one-epoch. Args: sess: tf.Session() object input_data: np.ndarray of shape (n_samples, n_features) input_labels: np.ndarray of shape (n_samples, n_classes) Returns: average_loss: scalar. Average minibatch loss of model on epoch. """ # And then after everything is built, start the training loop. orig_X, orig_y = input_data, input_labels total_loss = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = len(orig_X) / self.config.batch_size for step, (input_batch, label_batch) in enumerate( data_iterator(orig_X, orig_y, batch_size=self.config.batch_size, label_size=self.config.n_classes, shuffle=shuffle)): feed_dict = self.create_feed_dict(input_batch, label_batch) loss, total_correct, _ = sess.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed_dict) total_processed_examples += len(input_batch) total_correct_examples += total_correct total_loss.append(loss) if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return np.mean(total_loss), total_correct_examples / float( total_processed_examples)
def predict(self, session, sent1_data, sent2_data, len1_data, len2_data, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 kp = 1.0 losses = [] results = [] if np.any(y): data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) for step, (sent1, sent2, len1, len2, y) in enumerate(data): feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp) if np.any(y): loss, preds = session.run( [self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) results.extend(preds) return np.mean(losses), np.array(results)
def predict(self, session, input_data, disallow_other=False): dp = 1. l2 = 0. losses = [] results = [] data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length) for step, (lines, line_lengths, labels) in enumerate(data): feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels]) loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed) if disallow_other: preds[:,self.speakers.speaker_to_index["OTHER"]] = 0. predicted_indices = preds.argmax(axis=1) losses.append(loss) results.extend(predicted_indices) return np.mean(losses), results
def predict(self, session, X, y=None): # dropout dp = 1 # 损失 losses = [] # 输出结果 results = [] # 判定y是否为None if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(X, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) # 每步循环 for step, (x, y) in enumerate(data): feed = self.create_feed_dict(input_batch=x, dropout=dp) # 如果y非空,获取预测值和损失 if np.any(y): feed[self.labels_placeholder] = y loss, preds = session.run([self.loss, self.predictions], feed_dict=feed) losses.append(loss) # 如果y为空,只获取预测值 else: preds = session.run(self.predictions, feed_dict=feed) predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) # 返回平均损失和预测结果 return np.mean(losses), results
def predict(self, session, X, y=None): losses = [] results = [] for step, (x, y) in enumerate( data_iterator(X, y, batch_size=self.batch_size, shuffle=False)): feed = self.create_feed_dict(input_batch=x, label_batch=y) if np.any(y): loss, preds = session.run([self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) results.extend(preds) return results, np.mean(losses)
def run_epoch(self, sess, input_data, input_labels, shuffle=True, verbose=True): # 所有的数据 orig_X, orig_y = input_data, input_labels dp = self.config.dropout # 记录各个循环的误差 total_loss = [] # 记录所有训练数据中预测正确的数量 total_correct_examples = 0 # 记录所有已经处理的数据 total_processed_examples = 0 total_steps = len(orig_X) / self.config.batch_size # 对每份批量数据 for step, (x, y) in enumerate( data_iterator(orig_X, orig_y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=shuffle)): # 获取feed字典 feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y) # 运行计算图,获取损失和正确预测个数 loss, total_correct, _ = sess.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed) total_processed_examples += len(x) total_correct_examples += total_correct total_loss.append(loss) # 若可显示,则输出每个循环/每步迭代的结果 if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() # 返回平均误差和正确率 return np.mean(total_loss), total_correct_examples / float( total_processed_examples)
def run_epoch(self, session, input_data, input_labels, shuffle=True, verbose=True): orig_X, orig_y = input_data, input_labels dp = self.config.dropout # We're interested in keeping track of the loss and accuracy during training total_loss = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = len(orig_X) / self.config.batch_size for step, (x, y) in enumerate( data_iterator(orig_X, orig_y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=shuffle)): feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y) # Add tf.merge_all_summaries() which will provide summaries. #merged = tf.merge_all_summaries() #writer = tf.train.SummaryWriter("/tmp/q2_NER_logs", session.graph) # Run #loss, total_correct, _, summary_str = session.run( # [self.loss, self.correct_predictions, self.train_op, merged], # feed_dict=feed) loss, total_correct, _ = session.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed) #print "=========================" #print summary_str #writer.add_summary(summary_str) total_processed_examples += len(x) total_correct_examples += total_correct total_loss.append(loss) ## if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format( step, total_steps, np.mean(total_loss))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def decoding(loaded_model, test_dataset, arg_parser): beam_size = arg_parser.beam_size max_len = arg_parser.max_decode_len decoding_method = loaded_model.beam_search if arg_parser.decoding == 'beam_search' else loaded_model.decode_greedy loaded_model.eval() hypotheses = [] gold_queries = [] scores = 0 count = 0 with torch.no_grad(): for src_sent_batch, gold_target in tqdm(data_iterator(test_dataset, batch_size=1, shuffle=False), total=280): example_hyps = decoding_method(sources=src_sent_batch, max_len=max_len, beam_size=beam_size) hypotheses.append(example_hyps) gold_queries.append(gold_target[0]) return hypotheses, gold_queries
def predict(self, session, input_data, disallow_other=False): dp = 1. l2 = 0. losses = [] results = [] data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length) for step, (lines, line_lengths, labels) in enumerate(data): feed = self.create_feed_dict( lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels]) loss, preds, predicted_indices = session.run( [self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed) if disallow_other: preds[:, self.speakers.speaker_to_index["OTHER"]] = 0. predicted_indices = preds.argmax(axis=1) losses.append(loss) results.extend(predicted_indices) return np.mean(losses), results
def main(): vocab = Vocab('bert-base-uncased') test_inputs = get_dataset_finish_by('geoQueryData', 'train', '_recomb.tsv') with open('tokenization_tests.txt', 'w') as test_file: test_file.truncate() num_matches = 0 total_examples = 0 for batch_idx, batch_examples in enumerate( data_iterator(test_inputs, batch_size=1, shuffle=False)): tokens_list = vocab.to_input_tokens(batch_examples[1])[0] detokenized = detokenize(tokens_list) if detokenized == batch_examples[1][0]: num_matches += 1 else: test_file.write('wrong example:\n') test_file.write(batch_examples[1][0] + '\n') test_file.write(detokenized + '\n') test_file.write('\n' + '-' * 15 + '\n') total_examples += 1 print( f"we obtained the following result: {num_matches / total_examples:.2f} accuracy for detokenization method on given dataset" ) return
def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True): loss_history = [] var_objective_history = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = len(X) / self.batch_size for step, (X_batch, y_batch) in enumerate( data_iterator(X, y, batch_size=self.batch_size, shuffle=shuffle)): #supervised feed = self.create_feed_dict(input_batch=X_batch, label_batch=y_batch) loss, total_correct, _ = session.run( [self.loss, self.correct_predictions, self.train_op], feed_dict=feed) total_processed_examples += len(X_batch) total_correct_examples += total_correct loss_history.append(loss) #unsupervised var_objective, accuracy = self.unsup_update(trees[step]) var_objective_history.append(var_objective) if verbose and step % verbose == 0: sys.stdout.write( '\r{} / {} : loss = {}, var_objective = {}'.format( step, total_steps, np.mean(loss_history), np.mean(var_objective))) sys.stdout.flush() if verbose: sys.stdout.write('\r') sys.stdout.flush() return loss_history, total_correct_examples / float( total_processed_examples)
def train(arg_parser): logs_path = os.path.join(arg_parser.log_dir, arg_parser.subdir) if not os.path.isdir(logs_path): os.makedirs(logs_path) file_name_epoch_indep = get_model_name(arg_parser) recombination = arg_parser.recombination_method vocab = Vocab(f'bert-{arg_parser.BERT}-uncased') model_type = TSP if arg_parser.TSP_BSP else BSP model = model_type(input_vocab=vocab, target_vocab=vocab, d_model=arg_parser.d_model, d_int=arg_parser.d_int, d_k=arg_parser.d_k, h=arg_parser.h, n_layers=arg_parser.n_layers, dropout_rate=arg_parser.dropout, max_len_pe=arg_parser.max_len_pe, bert_name=arg_parser.BERT) file_path = os.path.join( arg_parser.models_path, f"{file_name_epoch_indep}_epoch_{arg_parser.epoch_to_load}.pt") if arg_parser.train_load: train_dataset = get_dataset_finish_by(arg_parser.data_folder, 'train', f"600_entity_recomb.tsv") test_dataset = get_dataset_finish_by(arg_parser.data_folder, 'dev', f"100_entity_recomb.tsv") load_model(file_path=file_path, model=model) #load_model(file_path=os.path.join('models_to_keep', 'BSP_d_model256_layers4_recombentity+nesting+concat2_extrastrain1800_extrasdev300_epoch_75.pt'), model=model) print('loaded model') else: train_dataset = get_dataset_finish_by( arg_parser.data_folder, 'train', f"{600 + arg_parser.extras_train}_{recombination}_recomb.tsv") test_dataset = get_dataset_finish_by( arg_parser.data_folder, 'dev', f"{100 + arg_parser.extras_dev}_{recombination}_recomb.tsv") model.train() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model = model.to(device) if arg_parser.optimizer: optimizer = NoamOpt(model_size=arg_parser.d_model, factor=1, warmup=arg_parser.warmups_steps, \ optimizer=torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) else: optimizer = torch.optim.Adam(model.parameters(), lr=arg_parser.lr) model.device = device summary_writer = SummaryWriter( log_dir=logs_path) if arg_parser.log else None n_train = len(train_dataset) n_test = len(test_dataset) for epoch in range(arg_parser.epochs): running_loss = 0.0 last_log_time = time.time() # Training train_loss = 0.0 for batch_idx, batch_examples in enumerate( data_iterator(train_dataset, batch_size=arg_parser.batch_size, shuffle=arg_parser.shuffle)): if ((batch_idx % 100) == 0) and batch_idx > 1: print( "epoch {} | batch {} | mean running loss {:.2f} | {:.2f} batch/s" .format(epoch, batch_idx, running_loss / 100, 100 / (time.time() - last_log_time))) last_log_time = time.time() running_loss = 0.0 sources, targets = batch_examples[0], batch_examples[1] example_losses = -model(sources, targets) # (batch_size,) batch_loss = example_losses.sum() loss = batch_loss / arg_parser.batch_size # clip gradient grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), arg_parser.clip_grad) if arg_parser.optimizer: loss.backward() optimizer.step() optimizer.optimizer.zero_grad() else: optimizer.zero_grad() loss.backward() optimizer.step() # add loss running_loss += loss.item() train_loss += loss.item() print("Epoch train loss : {}".format( math.sqrt(train_loss / math.ceil(n_train / arg_parser.batch_size)))) if summary_writer is not None: summary_writer.add_scalar( "train/loss", train_loss / math.ceil(n_train / arg_parser.batch_size), global_step=epoch) if (epoch % arg_parser.save_every == 0) and arg_parser.log and epoch > 0: if arg_parser.train_load: save_model( arg_parser.models_path, f"{file_name_epoch_indep}_epoch_{epoch + arg_parser.epoch_to_load}.pt", model, device) else: save_model( arg_parser.models_path, f"{file_name_epoch_indep}_epoch_{epoch + arg_parser.epoch_to_load}.pt", model, device) ## TEST test_loss = 0.0 for batch_idx, batch_examples in enumerate( data_iterator(test_dataset, batch_size=arg_parser.batch_size, shuffle=arg_parser.shuffle)): with torch.no_grad(): sources, targets = batch_examples[0], batch_examples[1] example_losses = -model(sources, targets) # (batch_size,) batch_loss = example_losses.sum() loss = batch_loss / arg_parser.batch_size test_loss += loss.item() if summary_writer is not None: summary_writer.add_scalar( "test/loss", test_loss / math.ceil(n_test / arg_parser.batch_size), global_step=epoch) print("TEST loss | epoch {} | {:.2f}".format( epoch, test_loss / math.ceil(n_test / arg_parser.batch_size))) return None
self.train_op = self.add_training_op(self.loss) self.merged_summaries = tf.merge_all_summaries() self.summary_writer = None def run_epoch(self, session, input_data, input_labels, shuffle=True, verbose=True, epoch=0): orig_X, orig_y = input_data, input_labels dp = self.config.dropout # We're interested in keeping track of the loss and accuracy during training total_loss = [] total_correct_examples = 0 total_processed_examples = 0 total_steps = len(orig_X) / self.config.batch_size for step, (x, y) in enumerate( data_iterator(orig_X, orig_y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=shuffle)): feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y) loss, total_correct, _, merged = session.run( [self.loss, self.correct_predictions, self.train_op, self.merged_summaries], feed_dict=feed) if step % 50 == 0: self.summary_writer.add_summary(merged, epoch * total_steps + step) total_processed_examples += len(x) total_correct_examples += total_correct total_loss.append(loss) ## if verbose and step % verbose == 0: sys.stdout.write('\r{} / {} : loss = {}'.format(
def next_batch(self, batch_size, shuffle=True): """Return the next `batch_size` examples from this data set. Takes Y in one-hot encodings only""" for x, y in data_iterator(self._x, self._y, batch_size, shuffle): yield x, y
if use_word2vec: encoder = EncodeSentence(len(vocab2idx), word_dim, wordEmbedding=word_emb, use_lstm=use_lstm).to(device) else: encoder = EncodeSentence(len(vocab2idx), word_dim, use_lstm=use_lstm).to(device) model = FeedForward(len(tag2idx), tag_dim, word_dim).to(device) optimizer = optim.Adam(list(model.parameters()) + list(encoder.parameters())) # Training for it in range(train_iter): print("Iteration: {}".format(it)) train_sents, train_tags = shuffle(train_sents, train_tags) for data, label in data_iterator(train_sents, train_tags, batch_size=100): optimizer.zero_grad() train_data = torch.tensor(pad_sents(data), device=device) train_data = encoder(train_data) train_label = torch.tensor(pad_sents(label, 1), device=device) maxlen = train_data.size(1) loss_sum = torch.tensor([0.], device=device) for i in range(1, maxlen): output = model(train_data[:, i], train_label[:, i - 1]) loss_sum += F.nll_loss(output, train_label[:, i]) loss_sum.backward() optimizer.step() print("Loss: {:.4f}".format(loss_sum.item() / maxlen)) # Testing test_data = torch.tensor(pad_sents(test_sents), device=device)