Beispiel #1
0
 def predict(self, session, X, y=None):
     """Make predictions from the provided model."""
     # If y is given, the loss is also calculated
     # We deactivate dropout by setting it to 1
     dp = 1
     losses = []
     results = []
     if np.any(y):
         data = data_iterator(X,
                              y,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
     else:
         data = data_iterator(X,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
     for step, (x, y) in enumerate(data):
         feed = self.create_feed_dict(input_batch=x, dropout=dp)
         if np.any(y):
             feed[self.labels_placeholder] = y
             loss, preds = session.run([self.loss, self.predictions],
                                       feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         predicted_indices = preds.argmax(axis=1)
         results.extend(predicted_indices)
     return np.mean(losses), results
Beispiel #2
0
 def run_epoch(self, sess, data, train_op=None, verbose=10):
     dropout = self.config.dropout
     if train_op == None:
         train_op = tf.no_op()
         dropout = 1
     total_steps = sum(1 for x in data_iterator(
         data, self.config.batch_size, self.config.num_steps))
     total_loss = []
     state = self.initial_state.eval()
     for step, (x, y) in enumerate(
             data_iterator(data, self.config.batch_size,
                           self.config.num_steps)):
         feed = {
             self.input_placeholder: x,
             self.label_placeholder: y,
             self.dropout_placeholder: dropout,
             self.initial_state: state
         }
         loss, state, _ = sess.run(
             [self.calculate_loss, self.final_state, train_op],
             feed_dict=feed)
         total_loss.append(loss)
         if verbose and step % verbose == 0:
             sys.stdout.write('\r{} / {} : Avg. Loss = {}'.format(
                 step, total_steps, np.mean(total_loss)))
             sys.stdout.flush()
     if verbose:
         sys.stdout.write('\r')
     return np.mean(total_loss)
Beispiel #3
0
 def predict(self, session, X, y=None):
   """Make predictions from the provided model."""
   # If y is given, the loss is also calculated
   # We deactivate dropout by setting it to 1
   dp = 1
   losses = []
   results = []
   if np.any(y):
       data = data_iterator(X, y, batch_size=self.config.batch_size,
                            label_size=self.config.label_size, shuffle=False)
   else:
       data = data_iterator(X, batch_size=self.config.batch_size,
                            label_size=self.config.label_size, shuffle=False)
   for step, (x, y) in enumerate(data):
     feed = self.create_feed_dict(input_batch=x, dropout=dp)
     if np.any(y):
       feed[self.labels_placeholder] = y
       loss, preds = session.run(
           [self.loss, self.predictions], feed_dict=feed)
       losses.append(loss)
     else:
       preds = session.run(self.predictions, feed_dict=feed)
     predicted_indices = preds.argmax(axis=1)
     results.extend(predicted_indices)
   return np.mean(losses), results
Beispiel #4
0
 def predict(self, session, X, y=None):
     dp = 1
     losses = []
     results = []
     if np.any(y):
         data = data_iterator(X,
                              y,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
     else:
         data = data_iterator(X,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
     for step, (x, y) in enumerate(data):
         feed = self.create_feed_dict(input_batch=x, dropout=dp)
         if np.any(y):
             feed[self.labels_placeholder] = y
             loss, preds = session.run([self.loss, self.predictions],
                                       feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         predicted_indices = preds.argmax(axis=1)
         results.extend(predicted_indices)
     return np.mean(losses), results
Beispiel #5
0
 def predict(self, session, X, y=None, verbose=False):
     """Make predictions from the provided model."""
     # If y is given, the loss is also calculated
     # We deactivate dropout by setting it to 1
     dp = 1
     losses = []
     results = []
     if np.any(y):
         data = data_iterator(X, y, batch_size=self.config.batch_size,
                              tagset_size=self.tagset_size)
     else:
         data = data_iterator(X, batch_size=self.config.batch_size,
                              tagset_size=self.tagset_size)
     for step, (x, y) in enumerate(data):
         feed = self.create_feed_dict(input_batch=x, dropout=dp)
         if np.any(y):
             feed[self.labels_placeholder] = y
             loss, preds = session.run(
                 [self.loss, self.predictions], feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         predicted_indices = preds.argmax(axis=1)
         results.extend(predicted_indices)
         if verbose:
             print 'SENTENCE %d:' % step
             sentences, _, labels = print_labels(x, preds, y,
                                                 self.num_to_word,
                                                 self.num_to_tag)
             print sentences
             print labels
     return np.mean(losses), results
Beispiel #6
0
 def predict(self, session, X, y=None):
     """Make predictions from the provided model."""
     # If y is given, the loss is also calculated
     # We deactivate dropout by setting it to 1
     dp = 1
     losses = []
     results = []
     z = 1
     if np.any(y):  #It would evaluate to false if all y are zero!!!!!
         data = data_iterator(X,
                              y,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
         #print 'np.any(y) seems to be TRUE'
         #print type(data)
         #print data.shape
     else:
         data = data_iterator(X,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
         #print 'np.any(y) seems to be FALSE'
         #print type(data)
         #print data.shape
     for step, (x, y) in enumerate(
             data):  #Howz it working when np.any(y) is False
         feed = self.create_feed_dict(input_batch=x, dropout=dp)
         #print 'x.shape in predict function, followed by feed.input_placeholder.shape. I guess both should be same'
         #print x.shape
         #print feed[self.input_placeholder].shape
         if np.any(y):
             #print 'y>>>: '
             #print y
             feed[self.labels_placeholder] = y
             loss, preds = session.run([self.loss, self.predictions],
                                       feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
             #                                   tf.nn.softmax(y)
             #                                                              y = self.add_model(window)
             #                                                                                                   window = self.add_embedding()
         #
         predicted_indices = preds.argmax(axis=1)
         results.extend(predicted_indices)
     if len(losses) == 0:
         return None, results
     return np.mean(losses), results
Beispiel #7
0
    def predict(self, session, input_data, disallow_other=False):
        dp = 1.
        l2 = 0.
        losses = []
        results = []
        data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length)

        for step, (lines, line_lengths, labels) in enumerate(data):
            try:
                self.index_to_weight
            except:
                if self.config.weight_loss:
                    # f = np.log
                    f = lambda x: np.sqrt(x)
                    normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
                    self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
                else:
                    self.index_to_weight = {k:1. for k in range(len(self.speakers))}

            feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
            loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed)
            if disallow_other:
                preds[:,self.speakers.speaker_to_index["OTHER"]] = 0.
                predicted_indices = preds.argmax(axis=1)
            losses.append(loss)
            results.extend(predicted_indices)
        return np.mean(losses), results
Beispiel #8
0
 def run_epoch(self, session, input_data, shuffle=False, verbose=True):
     dp = self.config.dropout
     l2 = self.config.l2
     # We're interested in keeping track of the loss and accuracy during training
     total_loss = []
     total_correct_examples = 0
     total_processed_examples = 0
     if self.config.batch_size == "chapter":
         total_steps = len(input_data)
     else:
         total_steps = len(input_data) / self.config.batch_size
     data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length, shuffle=shuffle)
     for step, (lines, line_lengths, labels) in enumerate(data):
         if self.config.weight_loss:
             # f = np.log
             f = lambda x: np.sqrt(x)
             normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
             self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
             feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
         else:
             self.index_to_weight = {k:1. for k in range(len(self.speakers))}
             feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2)
         loss, total_correct, _ = session.run([self.loss, self.correct_predictions, self.train_op], feed_dict=feed)
         total_processed_examples += len(labels)
         total_correct_examples += total_correct
         total_loss.append(loss)
         ##
         if verbose and step % verbose == 0:
             sys.stdout.write('\r{} / {} : loss = {}'.format(step, total_steps, np.mean(total_loss)))
             sys.stdout.flush()
     if verbose:
         sys.stdout.write('\r')
         sys.stdout.flush()
     return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
Beispiel #9
0
    def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True):
        loss_history = []
        var_objective_history = []
        total_correct_examples = 0
        total_processed_examples = 0
        total_steps = len(X) / self.batch_size
        for step, (X_batch, y_batch) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=shuffle)):
            #supervised
            feed = self.create_feed_dict(input_batch=X_batch, label_batch=y_batch)
            loss, total_correct, _ = session.run(
                    [self.loss, self.correct_predictions, self.train_op],
                    feed_dict=feed)
            total_processed_examples += len(X_batch)
            total_correct_examples += total_correct
            loss_history.append(loss)

            #unsupervised
            var_objective, accuracy = self.unsup_update(trees[step])
            var_objective_history.append(var_objective)

            if verbose and step % verbose == 0:
                sys.stdout.write('\r{} / {} : loss = {}, var_objective = {}'.format(
                        step, total_steps, np.mean(loss_history), np.mean(var_objective)))
                sys.stdout.flush()
        if verbose:
                sys.stdout.write('\r')
                sys.stdout.flush()
        return loss_history, total_correct_examples / float(total_processed_examples)
def decoding(loaded_model, test_dataset, arg_parser):
    beam_size = arg_parser.beam_size
    max_len = arg_parser.max_decode_len
    decoding_method = loaded_model.beam_search if arg_parser.decoding == 'beam_search' else loaded_model.decode_greedy
    loaded_model.eval()
    model_outputs = []
    model_outputs_kb = []
    gold_queries_kb = []
    gold_queries = []
    with torch.no_grad():
        for src_sent_batch, gold_target in tqdm(data_iterator(test_dataset,
                                                              batch_size=1,
                                                              shuffle=False),
                                                total=280):
            example_hyps = decoding_method(src_sent=src_sent_batch,
                                           max_len=max_len,
                                           beam_size=beam_size)
            strings_model = [
                detokenize(example_hyp) for example_hyp in example_hyps
            ]
            string_gold = gold_target[0]
            model_outputs_kb.append(strings_model)
            gold_queries_kb.append(string_gold)
            strings_model, string_gold = format_lf(strings_model, string_gold)
            model_outputs.append(strings_model)
            gold_queries.append(string_gold)
    return model_outputs, gold_queries, model_outputs_kb, gold_queries_kb
Beispiel #11
0
    def run_epoch(self, sess, input_data, input_labels):
        """Runs an epoch of training.

    Trains the model for one-epoch.
  
    Args:
      sess: tf.Session() object
      input_data: np.ndarray of shape (n_samples, n_features)
      input_labels: np.ndarray of shape (n_samples, n_classes)
    Returns:
      average_loss: scalar. Average minibatch loss of model on epoch.
    """
        # And then after everything is built, start the training loop.
        average_loss = 0
        for step, (input_batch, label_batch) in enumerate(
                data_iterator(input_data,
                              input_labels,
                              batch_size=self.config.batch_size,
                              label_size=self.config.n_classes)):

            # Fill a feed dictionary with the actual set of images and labels
            # for this particular training step.
            feed_dict = self.create_feed_dict(input_batch, label_batch)

            # Run one step of the model.  The return values are the activations
            # from the `self.train_op` (which is discarded) and the `loss` Op.  To
            # inspect the values of your Ops or variables, you may include them
            # in the list passed to sess.run() and the value tensors will be
            # returned in the tuple from the call.
            _, loss_value = sess.run([self.train_op, self.loss],
                                     feed_dict=feed_dict)
            average_loss += loss_value

        average_loss = average_loss / step
        return average_loss
Beispiel #12
0
 def run_epoch(self, session, input_data, input_labels,
               shuffle=True, verbose=True):
   orig_X, orig_y = input_data, input_labels
   dp = self.config.dropout
   # We're interested in keeping track of the loss and accuracy during training
   total_loss = []
   total_correct_examples = 0
   total_processed_examples = 0
   total_steps = len(orig_X) / self.config.batch_size
   for step, (x, y) in enumerate(
     data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                  label_size=self.config.label_size, shuffle=shuffle)):
     feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
     loss, total_correct, _ = session.run(
         [self.loss, self.correct_predictions, self.train_op],
         feed_dict=feed)
     total_processed_examples += len(x)
     total_correct_examples += total_correct
     total_loss.append(loss)
     ##
     if verbose and step % verbose == 0:
       sys.stdout.write('\r{} / {} : loss = {}'.format(
           step, total_steps, np.mean(total_loss)))
       sys.stdout.flush()
   if verbose:
       sys.stdout.write('\r')
       sys.stdout.flush()
   return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
 def run_epoch(self, session, sent1_data, sent2_data, len1_data, len2_data, input_labels,
         verbose=100):
     orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y = ( sent1_data,
             sent2_data, len1_data, len2_data, input_labels )
     kp = self.config.kp
     total_loss = []
     total_correct_examples = 0
     total_processed_examples = 0
     total_steps = int( orig_sent1.shape[0] / self.config.batch_size)
     for step, (sent1, sent2, len1, len2, y) in enumerate(
         data_iterator(orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y,
                 batch_size=self.config.batch_size, label_size=self.config.label_size)):
         feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
         loss, total_correct, _ = session.run(
             [self.loss, self.correct_predictions, self.train_op],
             feed_dict=feed)
         total_processed_examples += len(y)
         total_correct_examples += total_correct
         total_loss.append(loss)
         if verbose and step % verbose == 0:
             sys.stdout.write('\r{} / {} : loss = {}'.format(
                 step, total_steps, np.mean(total_loss)))
             sys.stdout.flush()
     if verbose:
         sys.stdout.write('\r')
         sys.stdout.flush()
     return np.mean(total_loss), total_correct_examples / float(total_processed_examples), total_loss
Beispiel #14
0
  def run_epoch(self, sess, input_data, input_labels):
    """Runs an epoch of training.

    Trains the model for one-epoch.
  
    Args:
      sess: tf.Session() object
      input_data: np.ndarray of shape (n_samples, n_features)
      input_labels: np.ndarray of shape (n_samples, n_classes)
    Returns:
      average_loss: scalar. Average minibatch loss of model on epoch.
    """
    # And then after everything is built, start the training loop.
    average_loss = 0
    for step, (input_batch, label_batch) in enumerate(
        data_iterator(input_data, input_labels,
                      batch_size=self.config.batch_size,
                      label_size=self.config.n_classes)):

      # Fill a feed dictionary with the actual set of images and labels
      # for this particular training step.
      feed_dict = self.create_feed_dict(input_batch, label_batch)

      # Run one step of the model.  The return values are the activations
      # from the `self.train_op` (which is discarded) and the `loss` Op.  To
      # inspect the values of your Ops or variables, you may include them
      # in the list passed to sess.run() and the value tensors will be
      # returned in the tuple from the call.
      _, loss_value = sess.run([self.train_op, self.loss], feed_dict=feed_dict)
      average_loss += loss_value

    average_loss = average_loss / step
    return average_loss 
 def run_epoch(self, session, input_data, input_labels,
               shuffle=True, verbose=True):
   orig_X, orig_y = input_data, input_labels
   dp = self.config.dropout
   # We're interested in keeping track of the loss and accuracy during training
   total_loss = []
   total_correct_examples = 0
   total_processed_examples = 0
   total_steps = len(orig_X) / self.config.batch_size
   for step, (x, y) in enumerate(
     data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                  label_size=self.config.label_size, shuffle=shuffle)):
     feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
     loss, total_correct, _ = session.run(
         [self.loss, self.correct_predictions, self.train_op],
         feed_dict=feed)
     total_processed_examples += len(x)
     total_correct_examples += total_correct
     total_loss.append(loss)
     ##
     if verbose and step % verbose == 0:
       sys.stdout.write('\r{} / {} : loss = {}'.format(
           step, total_steps, np.mean(total_loss)))
       sys.stdout.flush()
   if verbose:
       sys.stdout.write('\r')
       sys.stdout.flush()
   return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
 def run_epoch(self, session, input_data, shuffle=False, verbose=True):
     dp = self.config.dropout
     l2 = self.config.l2
     # We're interested in keeping track of the loss and accuracy during training
     total_loss = []
     total_correct_examples = 0
     total_processed_examples = 0
     if self.config.batch_size == "chapter":
         total_steps = len(input_data)
     else:
         total_steps = len(input_data) / self.config.batch_size
     data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length, shuffle=shuffle)
     for step, (lines, line_lengths, labels) in enumerate(data):
         if self.config.weight_loss:
             # f = np.log
             f = lambda x: np.sqrt(x)
             normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
             self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
             feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
         else:
             self.index_to_weight = {k:1. for k in range(len(self.speakers))}
             feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2)
         loss, total_correct, _ = session.run([self.loss, self.correct_predictions, self.train_op], feed_dict=feed)
         total_processed_examples += len(labels)
         total_correct_examples += total_correct
         total_loss.append(loss)
         ##
         if verbose and step % verbose == 0:
             sys.stdout.write('\r{} / {} : loss = {}'.format(step, total_steps, np.mean(total_loss)))
             sys.stdout.flush()
     if verbose:
         sys.stdout.write('\r')
         sys.stdout.flush()
     return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
Beispiel #17
0
 def run_epoch(self, session, input_data, input_labels,
               shuffle=True, verbose=True):
   orig_X, orig_y = input_data, input_labels			# 定义数据 x,y,dropout
   dp = self.config.dropout
   # We're interested in keeping track of the loss and accuracy during training
   total_loss = []
   total_correct_examples = 0							# 计算分类准确的比例:correct/total processed
   total_processed_examples = 0
   total_steps = len(orig_X) / self.config.batch_size	# X的总size/每批的size 得到批次数
   for step, (x, y) in enumerate(
     data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                  label_size=self.config.label_size, shuffle=shuffle)):
     feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
     loss, total_correct, _ = session.run(											# 前面都是定义,这里才是执行	
         [self.loss, self.correct_predictions, self.train_op],		# 1.喂进去x y dropout,执行 loss,correct_predictions,train_op
         feed_dict=feed)															# train后,参数W等就已经被更新了
     total_processed_examples += len(x)
     total_correct_examples += total_correct
     total_loss.append(loss)
     ##
     if verbose and step % verbose == 0:
       sys.stdout.write('\r{} / {} : loss = {}'.format(
           step, total_steps, np.mean(total_loss)))
       sys.stdout.flush()
   if verbose:
       sys.stdout.write('\r')
       sys.stdout.flush()
   return np.mean(total_loss), total_correct_examples / float(total_processed_examples)		# 返回 mean(loss) 和 准确度
Beispiel #18
0
    def run_epoch(self, sess, input_data, input_labels):
        average_loss = 0
        for step, (input_batch, label_batch) in enumerate(data_iterator(input_data,input_labels,batch_size = self.config.batch_size, label_size=self.config.n_classes)):
            feed_dict = self.create_feed_dict(input_batch, label_batch)
            _, loss_value = sess.run([self.train_op, self.loss],feed_dict = feed_dict)
            average_loss += loss_value

        average_loss = average_loss / step
        return average_loss
Beispiel #19
0
 def predict(self, session, X, y=None):
   """Make predictions from the provided model."""
   # If y is given, the loss is also calculated
   # We deactivate dropout by setting it to 1
   dp = 1
   losses = []
   results = []
   if np.any(y):
       data = data_iterator(X, y, batch_size=self.config.batch_size,
                            label_size=self.config.label_size, shuffle=False)
 def predict(self,
             session,
             sent1_data,
             sent2_data,
             len1_data,
             len2_data,
             y=None):
     """Make predictions from the provided model."""
     # If y is given, the loss is also calculated
     # We deactivate dropout by setting it to 1
     kp = 1.0
     losses = []
     results = []
     if np.any(y):
         data = data_iterator(sent1_data,
                              sent2_data,
                              len1_data,
                              len2_data,
                              y,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
     else:
         data = data_iterator(sent1_data,
                              sent2_data,
                              len1_data,
                              len2_data,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=False)
     for step, (sent1, sent2, len1, len2, y) in enumerate(data):
         feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
         if np.any(y):
             loss, preds = session.run([self.loss, self.predictions],
                                       feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         results.extend(preds)
     return np.mean(losses), np.array(results)
Beispiel #21
0
 def predict(self, session, X, y=None):
     losses = []
     results = []
     for step, (x, y) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=False)):
         feed = self.create_feed_dict(input_batch=x, label_batch=y)
         if np.any(y):
             loss, preds = session.run(
                     [self.loss, self.predictions], feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         results.extend(preds)
     return results, np.mean(losses)
Beispiel #22
0
    def run_epoch(self,
                  sess,
                  input_data,
                  input_labels,
                  shuffle=True,
                  verbose=True):
        """Runs an epoch of training.

            Trains the model for one-epoch.

            Args:
              sess: tf.Session() object
              input_data: np.ndarray of shape (n_samples, n_features)
              input_labels: np.ndarray of shape (n_samples, n_classes)
            Returns:
              average_loss: scalar. Average minibatch loss of model on epoch.
            """

        # And then after everything is built, start the training loop.

        orig_X, orig_y = input_data, input_labels
        total_loss = []
        total_correct_examples = 0
        total_processed_examples = 0
        total_steps = len(orig_X) / self.config.batch_size

        for step, (input_batch, label_batch) in enumerate(
                data_iterator(orig_X,
                              orig_y,
                              batch_size=self.config.batch_size,
                              label_size=self.config.n_classes,
                              shuffle=shuffle)):

            feed_dict = self.create_feed_dict(input_batch, label_batch)

            loss, total_correct, _ = sess.run(
                [self.loss, self.correct_predictions, self.train_op],
                feed_dict=feed_dict)
            total_processed_examples += len(input_batch)
            total_correct_examples += total_correct
            total_loss.append(loss)
            if verbose and step % verbose == 0:
                sys.stdout.write('\r{} / {} : loss = {}'.format(
                    step, total_steps, np.mean(total_loss)))
                sys.stdout.flush()
        if verbose:
            sys.stdout.write('\r')
            sys.stdout.flush()

        return np.mean(total_loss), total_correct_examples / float(
            total_processed_examples)
 def predict(self, session, sent1_data, sent2_data, len1_data, len2_data, y=None):
     """Make predictions from the provided model."""
     # If y is given, the loss is also calculated
     # We deactivate dropout by setting it to 1
     kp = 1.0
     losses = []
     results = []
     if np.any(y):
         data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, y, batch_size=self.config.batch_size,
                              label_size=self.config.label_size, shuffle=False)
     else:
         data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, batch_size=self.config.batch_size,
                              label_size=self.config.label_size, shuffle=False)
     for step, (sent1, sent2, len1, len2, y) in enumerate(data):
         feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
         if np.any(y):
             loss, preds = session.run(
                 [self.loss, self.predictions], feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         results.extend(preds)
     return np.mean(losses), np.array(results)
 def predict(self, session, input_data, disallow_other=False):
     dp = 1.
     l2 = 0.
     losses = []
     results = []
     data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length)
     for step, (lines, line_lengths, labels) in enumerate(data):
         feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
         loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed)
         if disallow_other:
             preds[:,self.speakers.speaker_to_index["OTHER"]] = 0.
             predicted_indices = preds.argmax(axis=1)
         losses.append(loss)
         results.extend(predicted_indices)
     return np.mean(losses), results
Beispiel #25
0
    def predict(self, session, X, y=None):
        # dropout
        dp = 1
        # 损失
        losses = []
        # 输出结果
        results = []
        # 判定y是否为None
        if np.any(y):
            data = data_iterator(X,
                                 y,
                                 batch_size=self.config.batch_size,
                                 label_size=self.config.label_size,
                                 shuffle=False)
        else:
            data = data_iterator(X,
                                 batch_size=self.config.batch_size,
                                 label_size=self.config.label_size,
                                 shuffle=False)

        # 每步循环
        for step, (x, y) in enumerate(data):
            feed = self.create_feed_dict(input_batch=x, dropout=dp)
            # 如果y非空,获取预测值和损失
            if np.any(y):
                feed[self.labels_placeholder] = y
                loss, preds = session.run([self.loss, self.predictions],
                                          feed_dict=feed)
                losses.append(loss)
            # 如果y为空,只获取预测值
            else:
                preds = session.run(self.predictions, feed_dict=feed)
            predicted_indices = preds.argmax(axis=1)
            results.extend(predicted_indices)
        # 返回平均损失和预测结果
        return np.mean(losses), results
Beispiel #26
0
 def predict(self, session, X, y=None):
     losses = []
     results = []
     for step, (x, y) in enumerate(
             data_iterator(X, y, batch_size=self.batch_size,
                           shuffle=False)):
         feed = self.create_feed_dict(input_batch=x, label_batch=y)
         if np.any(y):
             loss, preds = session.run([self.loss, self.predictions],
                                       feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         results.extend(preds)
     return results, np.mean(losses)
Beispiel #27
0
    def run_epoch(self,
                  sess,
                  input_data,
                  input_labels,
                  shuffle=True,
                  verbose=True):
        # 所有的数据
        orig_X, orig_y = input_data, input_labels
        dp = self.config.dropout
        # 记录各个循环的误差
        total_loss = []
        # 记录所有训练数据中预测正确的数量
        total_correct_examples = 0
        # 记录所有已经处理的数据
        total_processed_examples = 0
        total_steps = len(orig_X) / self.config.batch_size
        # 对每份批量数据
        for step, (x, y) in enumerate(
                data_iterator(orig_X,
                              orig_y,
                              batch_size=self.config.batch_size,
                              label_size=self.config.label_size,
                              shuffle=shuffle)):
            # 获取feed字典
            feed = self.create_feed_dict(input_batch=x,
                                         dropout=dp,
                                         label_batch=y)
            # 运行计算图,获取损失和正确预测个数
            loss, total_correct, _ = sess.run(
                [self.loss, self.correct_predictions, self.train_op],
                feed_dict=feed)
            total_processed_examples += len(x)
            total_correct_examples += total_correct
            total_loss.append(loss)

            # 若可显示,则输出每个循环/每步迭代的结果
            if verbose and step % verbose == 0:
                sys.stdout.write('\r{} / {} : loss = {}'.format(
                    step, total_steps, np.mean(total_loss)))
                sys.stdout.flush()

        if verbose:
            sys.stdout.write('\r')
            sys.stdout.flush()
        # 返回平均误差和正确率
        return np.mean(total_loss), total_correct_examples / float(
            total_processed_examples)
  def run_epoch(self, session, input_data, input_labels,
                shuffle=True, verbose=True):
    orig_X, orig_y = input_data, input_labels
    dp = self.config.dropout
    # We're interested in keeping track of the loss and accuracy during training
    total_loss = []
    total_correct_examples = 0
    total_processed_examples = 0
    total_steps = len(orig_X) / self.config.batch_size
    for step, (x, y) in enumerate(
      data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                   label_size=self.config.label_size, shuffle=shuffle)):
      feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)

      # Add tf.merge_all_summaries() which will provide summaries.
      #merged = tf.merge_all_summaries()
      #writer = tf.train.SummaryWriter("/tmp/q2_NER_logs", session.graph)

      # Run
      #loss, total_correct, _, summary_str = session.run(
    #      [self.loss, self.correct_predictions, self.train_op, merged],
    #      feed_dict=feed)

      loss, total_correct, _ = session.run(
          [self.loss, self.correct_predictions, self.train_op],
          feed_dict=feed)

      #print "========================="
      #print summary_str
      #writer.add_summary(summary_str)

      total_processed_examples += len(x)
      total_correct_examples += total_correct
      total_loss.append(loss)
      ##
      if verbose and step % verbose == 0:
        sys.stdout.write('\r{} / {} : loss = {}'.format(
            step, total_steps, np.mean(total_loss)))
        sys.stdout.flush()
    if verbose:
        sys.stdout.write('\r')
        sys.stdout.flush()
    return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def decoding(loaded_model, test_dataset, arg_parser):
    beam_size = arg_parser.beam_size
    max_len = arg_parser.max_decode_len
    decoding_method = loaded_model.beam_search if arg_parser.decoding == 'beam_search' else loaded_model.decode_greedy
    loaded_model.eval()
    hypotheses = []
    gold_queries = []
    scores = 0
    count = 0
    with torch.no_grad():
        for src_sent_batch, gold_target in tqdm(data_iterator(test_dataset,
                                                              batch_size=1,
                                                              shuffle=False),
                                                total=280):
            example_hyps = decoding_method(sources=src_sent_batch,
                                           max_len=max_len,
                                           beam_size=beam_size)
            hypotheses.append(example_hyps)
            gold_queries.append(gold_target[0])
    return hypotheses, gold_queries
 def predict(self, session, input_data, disallow_other=False):
     dp = 1.
     l2 = 0.
     losses = []
     results = []
     data = data_iterator(input_data,
                          batch_size=self.config.batch_size,
                          chop_limit=self.config.max_line_length)
     for step, (lines, line_lengths, labels) in enumerate(data):
         feed = self.create_feed_dict(
             lines, line_lengths, labels, dp, l2,
             [self.index_to_weight[l] for l in labels])
         loss, preds, predicted_indices = session.run(
             [self.loss, self.predictions, self.one_hot_predictions],
             feed_dict=feed)
         if disallow_other:
             preds[:, self.speakers.speaker_to_index["OTHER"]] = 0.
             predicted_indices = preds.argmax(axis=1)
         losses.append(loss)
         results.extend(predicted_indices)
     return np.mean(losses), results
Beispiel #31
0
def main():
    vocab = Vocab('bert-base-uncased')
    test_inputs = get_dataset_finish_by('geoQueryData', 'train', '_recomb.tsv')
    with open('tokenization_tests.txt', 'w') as test_file:
        test_file.truncate()
        num_matches = 0
        total_examples = 0
        for batch_idx, batch_examples in enumerate(
                data_iterator(test_inputs, batch_size=1, shuffle=False)):
            tokens_list = vocab.to_input_tokens(batch_examples[1])[0]
            detokenized = detokenize(tokens_list)
            if detokenized == batch_examples[1][0]:
                num_matches += 1
            else:
                test_file.write('wrong example:\n')
                test_file.write(batch_examples[1][0] + '\n')
                test_file.write(detokenized + '\n')
                test_file.write('\n' + '-' * 15 + '\n')
            total_examples += 1
        print(
            f"we obtained the following result: {num_matches / total_examples:.2f} accuracy for detokenization method on given dataset"
        )
    return
Beispiel #32
0
    def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True):
        loss_history = []
        var_objective_history = []
        total_correct_examples = 0
        total_processed_examples = 0
        total_steps = len(X) / self.batch_size
        for step, (X_batch, y_batch) in enumerate(
                data_iterator(X,
                              y,
                              batch_size=self.batch_size,
                              shuffle=shuffle)):
            #supervised
            feed = self.create_feed_dict(input_batch=X_batch,
                                         label_batch=y_batch)
            loss, total_correct, _ = session.run(
                [self.loss, self.correct_predictions, self.train_op],
                feed_dict=feed)
            total_processed_examples += len(X_batch)
            total_correct_examples += total_correct
            loss_history.append(loss)

            #unsupervised
            var_objective, accuracy = self.unsup_update(trees[step])
            var_objective_history.append(var_objective)

            if verbose and step % verbose == 0:
                sys.stdout.write(
                    '\r{} / {} : loss = {}, var_objective = {}'.format(
                        step, total_steps, np.mean(loss_history),
                        np.mean(var_objective)))
                sys.stdout.flush()
        if verbose:
            sys.stdout.write('\r')
            sys.stdout.flush()
        return loss_history, total_correct_examples / float(
            total_processed_examples)
def train(arg_parser):
    logs_path = os.path.join(arg_parser.log_dir, arg_parser.subdir)
    if not os.path.isdir(logs_path):
        os.makedirs(logs_path)
    file_name_epoch_indep = get_model_name(arg_parser)
    recombination = arg_parser.recombination_method

    vocab = Vocab(f'bert-{arg_parser.BERT}-uncased')
    model_type = TSP if arg_parser.TSP_BSP else BSP
    model = model_type(input_vocab=vocab,
                       target_vocab=vocab,
                       d_model=arg_parser.d_model,
                       d_int=arg_parser.d_int,
                       d_k=arg_parser.d_k,
                       h=arg_parser.h,
                       n_layers=arg_parser.n_layers,
                       dropout_rate=arg_parser.dropout,
                       max_len_pe=arg_parser.max_len_pe,
                       bert_name=arg_parser.BERT)

    file_path = os.path.join(
        arg_parser.models_path,
        f"{file_name_epoch_indep}_epoch_{arg_parser.epoch_to_load}.pt")
    if arg_parser.train_load:
        train_dataset = get_dataset_finish_by(arg_parser.data_folder, 'train',
                                              f"600_entity_recomb.tsv")
        test_dataset = get_dataset_finish_by(arg_parser.data_folder, 'dev',
                                             f"100_entity_recomb.tsv")
        load_model(file_path=file_path, model=model)
        #load_model(file_path=os.path.join('models_to_keep', 'BSP_d_model256_layers4_recombentity+nesting+concat2_extrastrain1800_extrasdev300_epoch_75.pt'), model=model)
        print('loaded model')
    else:
        train_dataset = get_dataset_finish_by(
            arg_parser.data_folder, 'train',
            f"{600 + arg_parser.extras_train}_{recombination}_recomb.tsv")
        test_dataset = get_dataset_finish_by(
            arg_parser.data_folder, 'dev',
            f"{100 + arg_parser.extras_dev}_{recombination}_recomb.tsv")
    model.train()

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    model = model.to(device)
    if arg_parser.optimizer:
        optimizer = NoamOpt(model_size=arg_parser.d_model, factor=1, warmup=arg_parser.warmups_steps, \
                            optimizer=torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
    else:
        optimizer = torch.optim.Adam(model.parameters(), lr=arg_parser.lr)
    model.device = device
    summary_writer = SummaryWriter(
        log_dir=logs_path) if arg_parser.log else None

    n_train = len(train_dataset)
    n_test = len(test_dataset)

    for epoch in range(arg_parser.epochs):
        running_loss = 0.0
        last_log_time = time.time()

        # Training
        train_loss = 0.0
        for batch_idx, batch_examples in enumerate(
                data_iterator(train_dataset,
                              batch_size=arg_parser.batch_size,
                              shuffle=arg_parser.shuffle)):
            if ((batch_idx % 100) == 0) and batch_idx > 1:
                print(
                    "epoch {} | batch {} | mean running loss {:.2f} | {:.2f} batch/s"
                    .format(epoch, batch_idx, running_loss / 100,
                            100 / (time.time() - last_log_time)))
                last_log_time = time.time()
                running_loss = 0.0

            sources, targets = batch_examples[0], batch_examples[1]
            example_losses = -model(sources, targets)  # (batch_size,)
            batch_loss = example_losses.sum()
            loss = batch_loss / arg_parser.batch_size

            # clip gradient
            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),
                                                       arg_parser.clip_grad)

            if arg_parser.optimizer:
                loss.backward()
                optimizer.step()
                optimizer.optimizer.zero_grad()
            else:
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

            # add loss
            running_loss += loss.item()
            train_loss += loss.item()

        print("Epoch train loss : {}".format(
            math.sqrt(train_loss /
                      math.ceil(n_train / arg_parser.batch_size))))

        if summary_writer is not None:
            summary_writer.add_scalar(
                "train/loss",
                train_loss / math.ceil(n_train / arg_parser.batch_size),
                global_step=epoch)
        if (epoch % arg_parser.save_every
                == 0) and arg_parser.log and epoch > 0:
            if arg_parser.train_load:
                save_model(
                    arg_parser.models_path,
                    f"{file_name_epoch_indep}_epoch_{epoch + arg_parser.epoch_to_load}.pt",
                    model, device)
            else:
                save_model(
                    arg_parser.models_path,
                    f"{file_name_epoch_indep}_epoch_{epoch + arg_parser.epoch_to_load}.pt",
                    model, device)
        ## TEST
        test_loss = 0.0

        for batch_idx, batch_examples in enumerate(
                data_iterator(test_dataset,
                              batch_size=arg_parser.batch_size,
                              shuffle=arg_parser.shuffle)):
            with torch.no_grad():
                sources, targets = batch_examples[0], batch_examples[1]
                example_losses = -model(sources, targets)  # (batch_size,)
                batch_loss = example_losses.sum()
                loss = batch_loss / arg_parser.batch_size

                test_loss += loss.item()

        if summary_writer is not None:
            summary_writer.add_scalar(
                "test/loss",
                test_loss / math.ceil(n_test / arg_parser.batch_size),
                global_step=epoch)
        print("TEST loss | epoch {} | {:.2f}".format(
            epoch, test_loss / math.ceil(n_test / arg_parser.batch_size)))

    return None
Beispiel #34
0
    self.train_op = self.add_training_op(self.loss)
    self.merged_summaries = tf.merge_all_summaries()
    self.summary_writer = None

  def run_epoch(self, session, input_data, input_labels,
                shuffle=True, verbose=True, epoch=0):
    orig_X, orig_y = input_data, input_labels
    dp = self.config.dropout
    # We're interested in keeping track of the loss and accuracy during training
    total_loss = []
    total_correct_examples = 0
    total_processed_examples = 0
    total_steps = len(orig_X) / self.config.batch_size

    for step, (x, y) in enumerate(
      data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                   label_size=self.config.label_size, shuffle=shuffle)):
      feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)

      loss, total_correct, _, merged = session.run(
          [self.loss, self.correct_predictions, self.train_op, self.merged_summaries],
          feed_dict=feed)

      if step % 50 == 0:
        self.summary_writer.add_summary(merged, epoch * total_steps + step)

      total_processed_examples += len(x)
      total_correct_examples += total_correct
      total_loss.append(loss)
      ##
      if verbose and step % verbose == 0:
        sys.stdout.write('\r{} / {} : loss = {}'.format(
Beispiel #35
0
 def next_batch(self, batch_size, shuffle=True):
     """Return the next `batch_size` examples from this data set.
        Takes Y in one-hot encodings only"""
     for x, y in data_iterator(self._x, self._y, batch_size, shuffle):
         yield x, y
Beispiel #36
0
if use_word2vec:
    encoder = EncodeSentence(len(vocab2idx),
                             word_dim,
                             wordEmbedding=word_emb,
                             use_lstm=use_lstm).to(device)
else:
    encoder = EncodeSentence(len(vocab2idx), word_dim,
                             use_lstm=use_lstm).to(device)
model = FeedForward(len(tag2idx), tag_dim, word_dim).to(device)
optimizer = optim.Adam(list(model.parameters()) + list(encoder.parameters()))

# Training
for it in range(train_iter):
    print("Iteration: {}".format(it))
    train_sents, train_tags = shuffle(train_sents, train_tags)
    for data, label in data_iterator(train_sents, train_tags, batch_size=100):
        optimizer.zero_grad()
        train_data = torch.tensor(pad_sents(data), device=device)
        train_data = encoder(train_data)
        train_label = torch.tensor(pad_sents(label, 1), device=device)
        maxlen = train_data.size(1)
        loss_sum = torch.tensor([0.], device=device)
        for i in range(1, maxlen):
            output = model(train_data[:, i], train_label[:, i - 1])
            loss_sum += F.nll_loss(output, train_label[:, i])
        loss_sum.backward()
        optimizer.step()
    print("Loss: {:.4f}".format(loss_sum.item() / maxlen))

# Testing
test_data = torch.tensor(pad_sents(test_sents), device=device)