def predict(self, session, X, y=None):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
dp = 1
losses = []
results = []
if np.any(y):
data = data_iterator(X,
y,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
else:
data = data_iterator(X,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
for step, (x, y) in enumerate(data):
feed = self.create_feed_dict(input_batch=x, dropout=dp)
if np.any(y):
feed[self.labels_placeholder] = y
loss, preds = session.run([self.loss, self.predictions],
feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
predicted_indices = preds.argmax(axis=1)
results.extend(predicted_indices)
return np.mean(losses), results
def run_epoch(self, sess, data, train_op=None, verbose=10):
dropout = self.config.dropout
if train_op == None:
train_op = tf.no_op()
dropout = 1
total_steps = sum(1 for x in data_iterator(
data, self.config.batch_size, self.config.num_steps))
total_loss = []
state = self.initial_state.eval()
for step, (x, y) in enumerate(
data_iterator(data, self.config.batch_size,
self.config.num_steps)):
feed = {
self.input_placeholder: x,
self.label_placeholder: y,
self.dropout_placeholder: dropout,
self.initial_state: state
}
loss, state, _ = sess.run(
[self.calculate_loss, self.final_state, train_op],
feed_dict=feed)
total_loss.append(loss)
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : Avg. Loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
return np.mean(total_loss)
def predict(self, session, X, y=None):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
dp = 1
losses = []
results = []
if np.any(y):
data = data_iterator(X, y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=False)
else:
data = data_iterator(X, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=False)
for step, (x, y) in enumerate(data):
feed = self.create_feed_dict(input_batch=x, dropout=dp)
if np.any(y):
feed[self.labels_placeholder] = y
loss, preds = session.run(
[self.loss, self.predictions], feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
predicted_indices = preds.argmax(axis=1)
results.extend(predicted_indices)
return np.mean(losses), results
def predict(self, session, X, y=None):
dp = 1
losses = []
results = []
if np.any(y):
data = data_iterator(X,
y,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
else:
data = data_iterator(X,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
for step, (x, y) in enumerate(data):
feed = self.create_feed_dict(input_batch=x, dropout=dp)
if np.any(y):
feed[self.labels_placeholder] = y
loss, preds = session.run([self.loss, self.predictions],
feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
predicted_indices = preds.argmax(axis=1)
results.extend(predicted_indices)
return np.mean(losses), results
def predict(self, session, X, y=None, verbose=False):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
dp = 1
losses = []
results = []
if np.any(y):
data = data_iterator(X, y, batch_size=self.config.batch_size,
tagset_size=self.tagset_size)
else:
data = data_iterator(X, batch_size=self.config.batch_size,
tagset_size=self.tagset_size)
for step, (x, y) in enumerate(data):
feed = self.create_feed_dict(input_batch=x, dropout=dp)
if np.any(y):
feed[self.labels_placeholder] = y
loss, preds = session.run(
[self.loss, self.predictions], feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
predicted_indices = preds.argmax(axis=1)
results.extend(predicted_indices)
if verbose:
print 'SENTENCE %d:' % step
sentences, _, labels = print_labels(x, preds, y,
self.num_to_word,
self.num_to_tag)
print sentences
print labels
return np.mean(losses), results
def predict(self, session, X, y=None):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
dp = 1
losses = []
results = []
z = 1
if np.any(y): #It would evaluate to false if all y are zero!!!!!
data = data_iterator(X,
y,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
#print 'np.any(y) seems to be TRUE'
#print type(data)
#print data.shape
else:
data = data_iterator(X,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
#print 'np.any(y) seems to be FALSE'
#print type(data)
#print data.shape
for step, (x, y) in enumerate(
data): #Howz it working when np.any(y) is False
feed = self.create_feed_dict(input_batch=x, dropout=dp)
#print 'x.shape in predict function, followed by feed.input_placeholder.shape. I guess both should be same'
#print x.shape
#print feed[self.input_placeholder].shape
if np.any(y):
#print 'y>>>: '
#print y
feed[self.labels_placeholder] = y
loss, preds = session.run([self.loss, self.predictions],
feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
# tf.nn.softmax(y)
# y = self.add_model(window)
# window = self.add_embedding()
#
predicted_indices = preds.argmax(axis=1)
results.extend(predicted_indices)
if len(losses) == 0:
return None, results
return np.mean(losses), results
def predict(self, session, input_data, disallow_other=False):
dp = 1.
l2 = 0.
losses = []
results = []
data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length)
for step, (lines, line_lengths, labels) in enumerate(data):
try:
self.index_to_weight
except:
if self.config.weight_loss:
# f = np.log
f = lambda x: np.sqrt(x)
normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
else:
self.index_to_weight = {k:1. for k in range(len(self.speakers))}
feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed)
if disallow_other:
preds[:,self.speakers.speaker_to_index["OTHER"]] = 0.
predicted_indices = preds.argmax(axis=1)
losses.append(loss)
results.extend(predicted_indices)
return np.mean(losses), results
def run_epoch(self, session, input_data, shuffle=False, verbose=True):
dp = self.config.dropout
l2 = self.config.l2
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
if self.config.batch_size == "chapter":
total_steps = len(input_data)
else:
total_steps = len(input_data) / self.config.batch_size
data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length, shuffle=shuffle)
for step, (lines, line_lengths, labels) in enumerate(data):
if self.config.weight_loss:
# f = np.log
f = lambda x: np.sqrt(x)
normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
else:
self.index_to_weight = {k:1. for k in range(len(self.speakers))}
feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2)
loss, total_correct, _ = session.run([self.loss, self.correct_predictions, self.train_op], feed_dict=feed)
total_processed_examples += len(labels)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True):
loss_history = []
var_objective_history = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(X) / self.batch_size
for step, (X_batch, y_batch) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=shuffle)):
#supervised
feed = self.create_feed_dict(input_batch=X_batch, label_batch=y_batch)
loss, total_correct, _ = session.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
total_processed_examples += len(X_batch)
total_correct_examples += total_correct
loss_history.append(loss)
#unsupervised
var_objective, accuracy = self.unsup_update(trees[step])
var_objective_history.append(var_objective)
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}, var_objective = {}'.format(
step, total_steps, np.mean(loss_history), np.mean(var_objective)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return loss_history, total_correct_examples / float(total_processed_examples)
def decoding(loaded_model, test_dataset, arg_parser):
beam_size = arg_parser.beam_size
max_len = arg_parser.max_decode_len
decoding_method = loaded_model.beam_search if arg_parser.decoding == 'beam_search' else loaded_model.decode_greedy
loaded_model.eval()
model_outputs = []
model_outputs_kb = []
gold_queries_kb = []
gold_queries = []
with torch.no_grad():
for src_sent_batch, gold_target in tqdm(data_iterator(test_dataset,
batch_size=1,
shuffle=False),
total=280):
example_hyps = decoding_method(src_sent=src_sent_batch,
max_len=max_len,
beam_size=beam_size)
strings_model = [
detokenize(example_hyp) for example_hyp in example_hyps
]
string_gold = gold_target[0]
model_outputs_kb.append(strings_model)
gold_queries_kb.append(string_gold)
strings_model, string_gold = format_lf(strings_model, string_gold)
model_outputs.append(strings_model)
gold_queries.append(string_gold)
return model_outputs, gold_queries, model_outputs_kb, gold_queries_kb
def run_epoch(self, sess, input_data, input_labels):
"""Runs an epoch of training.
Trains the model for one-epoch.
Args:
sess: tf.Session() object
input_data: np.ndarray of shape (n_samples, n_features)
input_labels: np.ndarray of shape (n_samples, n_classes)
Returns:
average_loss: scalar. Average minibatch loss of model on epoch.
"""
# And then after everything is built, start the training loop.
average_loss = 0
for step, (input_batch, label_batch) in enumerate(
data_iterator(input_data,
input_labels,
batch_size=self.config.batch_size,
label_size=self.config.n_classes)):
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = self.create_feed_dict(input_batch, label_batch)
# Run one step of the model. The return values are the activations
# from the `self.train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([self.train_op, self.loss],
feed_dict=feed_dict)
average_loss += loss_value
average_loss = average_loss / step
return average_loss
def run_epoch(self, session, input_data, input_labels,
shuffle=True, verbose=True):
orig_X, orig_y = input_data, input_labels
dp = self.config.dropout
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size
for step, (x, y) in enumerate(
data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=shuffle)):
feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
loss, total_correct, _ = session.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
total_processed_examples += len(x)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def run_epoch(self, session, sent1_data, sent2_data, len1_data, len2_data, input_labels,
verbose=100):
orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y = ( sent1_data,
sent2_data, len1_data, len2_data, input_labels )
kp = self.config.kp
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = int( orig_sent1.shape[0] / self.config.batch_size)
for step, (sent1, sent2, len1, len2, y) in enumerate(
data_iterator(orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y,
batch_size=self.config.batch_size, label_size=self.config.label_size)):
feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
loss, total_correct, _ = session.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
total_processed_examples += len(y)
total_correct_examples += total_correct
total_loss.append(loss)
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples), total_loss
def run_epoch(self, sess, input_data, input_labels):
"""Runs an epoch of training.
Trains the model for one-epoch.
Args:
sess: tf.Session() object
input_data: np.ndarray of shape (n_samples, n_features)
input_labels: np.ndarray of shape (n_samples, n_classes)
Returns:
average_loss: scalar. Average minibatch loss of model on epoch.
"""
# And then after everything is built, start the training loop.
average_loss = 0
for step, (input_batch, label_batch) in enumerate(
data_iterator(input_data, input_labels,
batch_size=self.config.batch_size,
label_size=self.config.n_classes)):
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = self.create_feed_dict(input_batch, label_batch)
# Run one step of the model. The return values are the activations
# from the `self.train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([self.train_op, self.loss], feed_dict=feed_dict)
average_loss += loss_value
average_loss = average_loss / step
return average_loss
def run_epoch(self, session, input_data, input_labels,
shuffle=True, verbose=True):
orig_X, orig_y = input_data, input_labels
dp = self.config.dropout
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size
for step, (x, y) in enumerate(
data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=shuffle)):
feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
loss, total_correct, _ = session.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
total_processed_examples += len(x)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def run_epoch(self, session, input_data, shuffle=False, verbose=True):
dp = self.config.dropout
l2 = self.config.l2
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
if self.config.batch_size == "chapter":
total_steps = len(input_data)
else:
total_steps = len(input_data) / self.config.batch_size
data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length, shuffle=shuffle)
for step, (lines, line_lengths, labels) in enumerate(data):
if self.config.weight_loss:
# f = np.log
f = lambda x: np.sqrt(x)
normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
else:
self.index_to_weight = {k:1. for k in range(len(self.speakers))}
feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2)
loss, total_correct, _ = session.run([self.loss, self.correct_predictions, self.train_op], feed_dict=feed)
total_processed_examples += len(labels)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def run_epoch(self, session, input_data, input_labels,
shuffle=True, verbose=True):
orig_X, orig_y = input_data, input_labels # 定义数据 x，y，dropout
dp = self.config.dropout
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0 # 计算分类准确的比例：correct／total processed
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size # X的总size／每批的size 得到批次数
for step, (x, y) in enumerate(
data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=shuffle)):
feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
loss, total_correct, _ = session.run( # 前面都是定义，这里才是执行
[self.loss, self.correct_predictions, self.train_op], # 1.喂进去x y dropout，执行 loss，correct_predictions，train_op
feed_dict=feed) # train后，参数W等就已经被更新了
total_processed_examples += len(x)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples) # 返回 mean(loss) 和 准确度
def run_epoch(self, sess, input_data, input_labels):
average_loss = 0
for step, (input_batch, label_batch) in enumerate(data_iterator(input_data,input_labels,batch_size = self.config.batch_size, label_size=self.config.n_classes)):
feed_dict = self.create_feed_dict(input_batch, label_batch)
_, loss_value = sess.run([self.train_op, self.loss],feed_dict = feed_dict)
average_loss += loss_value
average_loss = average_loss / step
return average_loss
def predict(self, session, X, y=None):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
dp = 1
losses = []
results = []
if np.any(y):
data = data_iterator(X, y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=False)
def predict(self,
session,
sent1_data,
sent2_data,
len1_data,
len2_data,
y=None):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
kp = 1.0
losses = []
results = []
if np.any(y):
data = data_iterator(sent1_data,
sent2_data,
len1_data,
len2_data,
y,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
else:
data = data_iterator(sent1_data,
sent2_data,
len1_data,
len2_data,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
for step, (sent1, sent2, len1, len2, y) in enumerate(data):
feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
if np.any(y):
loss, preds = session.run([self.loss, self.predictions],
feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
results.extend(preds)
return np.mean(losses), np.array(results)
def predict(self, session, X, y=None):
losses = []
results = []
for step, (x, y) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=False)):
feed = self.create_feed_dict(input_batch=x, label_batch=y)
if np.any(y):
loss, preds = session.run(
[self.loss, self.predictions], feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
results.extend(preds)
return results, np.mean(losses)
def run_epoch(self,
sess,
input_data,
input_labels,
shuffle=True,
verbose=True):
"""Runs an epoch of training.
Trains the model for one-epoch.
Args:
sess: tf.Session() object
input_data: np.ndarray of shape (n_samples, n_features)
input_labels: np.ndarray of shape (n_samples, n_classes)
Returns:
average_loss: scalar. Average minibatch loss of model on epoch.
"""
# And then after everything is built, start the training loop.
orig_X, orig_y = input_data, input_labels
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size
for step, (input_batch, label_batch) in enumerate(
data_iterator(orig_X,
orig_y,
batch_size=self.config.batch_size,
label_size=self.config.n_classes,
shuffle=shuffle)):
feed_dict = self.create_feed_dict(input_batch, label_batch)
loss, total_correct, _ = sess.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed_dict)
total_processed_examples += len(input_batch)
total_correct_examples += total_correct
total_loss.append(loss)
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(
total_processed_examples)
def predict(self, session, sent1_data, sent2_data, len1_data, len2_data, y=None):
"""Make predictions from the provided model."""
# If y is given, the loss is also calculated
# We deactivate dropout by setting it to 1
kp = 1.0
losses = []
results = []
if np.any(y):
data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=False)
else:
data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=False)
for step, (sent1, sent2, len1, len2, y) in enumerate(data):
feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
if np.any(y):
loss, preds = session.run(
[self.loss, self.predictions], feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
results.extend(preds)
return np.mean(losses), np.array(results)
def predict(self, session, input_data, disallow_other=False):
dp = 1.
l2 = 0.
losses = []
results = []
data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length)
for step, (lines, line_lengths, labels) in enumerate(data):
feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed)
if disallow_other:
preds[:,self.speakers.speaker_to_index["OTHER"]] = 0.
predicted_indices = preds.argmax(axis=1)
losses.append(loss)
results.extend(predicted_indices)
return np.mean(losses), results
def predict(self, session, X, y=None):
# dropout
dp = 1
# 损失
losses = []
# 输出结果
results = []
# 判定y是否为None
if np.any(y):
data = data_iterator(X,
y,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
else:
data = data_iterator(X,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=False)
# 每步循环
for step, (x, y) in enumerate(data):
feed = self.create_feed_dict(input_batch=x, dropout=dp)
# 如果y非空,获取预测值和损失
if np.any(y):
feed[self.labels_placeholder] = y
loss, preds = session.run([self.loss, self.predictions],
feed_dict=feed)
losses.append(loss)
# 如果y为空,只获取预测值
else:
preds = session.run(self.predictions, feed_dict=feed)
predicted_indices = preds.argmax(axis=1)
results.extend(predicted_indices)
# 返回平均损失和预测结果
return np.mean(losses), results
def predict(self, session, X, y=None):
losses = []
results = []
for step, (x, y) in enumerate(
data_iterator(X, y, batch_size=self.batch_size,
shuffle=False)):
feed = self.create_feed_dict(input_batch=x, label_batch=y)
if np.any(y):
loss, preds = session.run([self.loss, self.predictions],
feed_dict=feed)
losses.append(loss)
else:
preds = session.run(self.predictions, feed_dict=feed)
results.extend(preds)
return results, np.mean(losses)
def run_epoch(self,
sess,
input_data,
input_labels,
shuffle=True,
verbose=True):
# 所有的数据
orig_X, orig_y = input_data, input_labels
dp = self.config.dropout
# 记录各个循环的误差
total_loss = []
# 记录所有训练数据中预测正确的数量
total_correct_examples = 0
# 记录所有已经处理的数据
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size
# 对每份批量数据
for step, (x, y) in enumerate(
data_iterator(orig_X,
orig_y,
batch_size=self.config.batch_size,
label_size=self.config.label_size,
shuffle=shuffle)):
# 获取feed字典
feed = self.create_feed_dict(input_batch=x,
dropout=dp,
label_batch=y)
# 运行计算图,获取损失和正确预测个数
loss, total_correct, _ = sess.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
total_processed_examples += len(x)
total_correct_examples += total_correct
total_loss.append(loss)
# 若可显示,则输出每个循环/每步迭代的结果
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
# 返回平均误差和正确率
return np.mean(total_loss), total_correct_examples / float(
total_processed_examples)
def run_epoch(self, session, input_data, input_labels,
shuffle=True, verbose=True):
orig_X, orig_y = input_data, input_labels
dp = self.config.dropout
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size
for step, (x, y) in enumerate(
data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=shuffle)):
feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
# Add tf.merge_all_summaries() which will provide summaries.
#merged = tf.merge_all_summaries()
#writer = tf.train.SummaryWriter("/tmp/q2_NER_logs", session.graph)
# Run
#loss, total_correct, _, summary_str = session.run(
# [self.loss, self.correct_predictions, self.train_op, merged],
# feed_dict=feed)
loss, total_correct, _ = session.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
#print "========================="
#print summary_str
#writer.add_summary(summary_str)
total_processed_examples += len(x)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
step, total_steps, np.mean(total_loss)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return np.mean(total_loss), total_correct_examples / float(total_processed_examples)
def decoding(loaded_model, test_dataset, arg_parser):
beam_size = arg_parser.beam_size
max_len = arg_parser.max_decode_len
decoding_method = loaded_model.beam_search if arg_parser.decoding == 'beam_search' else loaded_model.decode_greedy
loaded_model.eval()
hypotheses = []
gold_queries = []
scores = 0
count = 0
with torch.no_grad():
for src_sent_batch, gold_target in tqdm(data_iterator(test_dataset,
batch_size=1,
shuffle=False),
total=280):
example_hyps = decoding_method(sources=src_sent_batch,
max_len=max_len,
beam_size=beam_size)
hypotheses.append(example_hyps)
gold_queries.append(gold_target[0])
return hypotheses, gold_queries
def predict(self, session, input_data, disallow_other=False):
dp = 1.
l2 = 0.
losses = []
results = []
data = data_iterator(input_data,
batch_size=self.config.batch_size,
chop_limit=self.config.max_line_length)
for step, (lines, line_lengths, labels) in enumerate(data):
feed = self.create_feed_dict(
lines, line_lengths, labels, dp, l2,
[self.index_to_weight[l] for l in labels])
loss, preds, predicted_indices = session.run(
[self.loss, self.predictions, self.one_hot_predictions],
feed_dict=feed)
if disallow_other:
preds[:, self.speakers.speaker_to_index["OTHER"]] = 0.
predicted_indices = preds.argmax(axis=1)
losses.append(loss)
results.extend(predicted_indices)
return np.mean(losses), results
def main():
vocab = Vocab('bert-base-uncased')
test_inputs = get_dataset_finish_by('geoQueryData', 'train', '_recomb.tsv')
with open('tokenization_tests.txt', 'w') as test_file:
test_file.truncate()
num_matches = 0
total_examples = 0
for batch_idx, batch_examples in enumerate(
data_iterator(test_inputs, batch_size=1, shuffle=False)):
tokens_list = vocab.to_input_tokens(batch_examples[1])[0]
detokenized = detokenize(tokens_list)
if detokenized == batch_examples[1][0]:
num_matches += 1
else:
test_file.write('wrong example:\n')
test_file.write(batch_examples[1][0] + '\n')
test_file.write(detokenized + '\n')
test_file.write('\n' + '-' * 15 + '\n')
total_examples += 1
print(
f"we obtained the following result: {num_matches / total_examples:.2f} accuracy for detokenization method on given dataset"
)
return
def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True):
loss_history = []
var_objective_history = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(X) / self.batch_size
for step, (X_batch, y_batch) in enumerate(
data_iterator(X,
y,
batch_size=self.batch_size,
shuffle=shuffle)):
#supervised
feed = self.create_feed_dict(input_batch=X_batch,
label_batch=y_batch)
loss, total_correct, _ = session.run(
[self.loss, self.correct_predictions, self.train_op],
feed_dict=feed)
total_processed_examples += len(X_batch)
total_correct_examples += total_correct
loss_history.append(loss)
#unsupervised
var_objective, accuracy = self.unsup_update(trees[step])
var_objective_history.append(var_objective)
if verbose and step % verbose == 0:
sys.stdout.write(
'\r{} / {} : loss = {}, var_objective = {}'.format(
step, total_steps, np.mean(loss_history),
np.mean(var_objective)))
sys.stdout.flush()
if verbose:
sys.stdout.write('\r')
sys.stdout.flush()
return loss_history, total_correct_examples / float(
total_processed_examples)
def train(arg_parser):
logs_path = os.path.join(arg_parser.log_dir, arg_parser.subdir)
if not os.path.isdir(logs_path):
os.makedirs(logs_path)
file_name_epoch_indep = get_model_name(arg_parser)
recombination = arg_parser.recombination_method
vocab = Vocab(f'bert-{arg_parser.BERT}-uncased')
model_type = TSP if arg_parser.TSP_BSP else BSP
model = model_type(input_vocab=vocab,
target_vocab=vocab,
d_model=arg_parser.d_model,
d_int=arg_parser.d_int,
d_k=arg_parser.d_k,
h=arg_parser.h,
n_layers=arg_parser.n_layers,
dropout_rate=arg_parser.dropout,
max_len_pe=arg_parser.max_len_pe,
bert_name=arg_parser.BERT)
file_path = os.path.join(
arg_parser.models_path,
f"{file_name_epoch_indep}_epoch_{arg_parser.epoch_to_load}.pt")
if arg_parser.train_load:
train_dataset = get_dataset_finish_by(arg_parser.data_folder, 'train',
f"600_entity_recomb.tsv")
test_dataset = get_dataset_finish_by(arg_parser.data_folder, 'dev',
f"100_entity_recomb.tsv")
load_model(file_path=file_path, model=model)
#load_model(file_path=os.path.join('models_to_keep', 'BSP_d_model256_layers4_recombentity+nesting+concat2_extrastrain1800_extrasdev300_epoch_75.pt'), model=model)
print('loaded model')
else:
train_dataset = get_dataset_finish_by(
arg_parser.data_folder, 'train',
f"{600 + arg_parser.extras_train}_{recombination}_recomb.tsv")
test_dataset = get_dataset_finish_by(
arg_parser.data_folder, 'dev',
f"{100 + arg_parser.extras_dev}_{recombination}_recomb.tsv")
model.train()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = model.to(device)
if arg_parser.optimizer:
optimizer = NoamOpt(model_size=arg_parser.d_model, factor=1, warmup=arg_parser.warmups_steps, \
optimizer=torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
else:
optimizer = torch.optim.Adam(model.parameters(), lr=arg_parser.lr)
model.device = device
summary_writer = SummaryWriter(
log_dir=logs_path) if arg_parser.log else None
n_train = len(train_dataset)
n_test = len(test_dataset)
for epoch in range(arg_parser.epochs):
running_loss = 0.0
last_log_time = time.time()
# Training
train_loss = 0.0
for batch_idx, batch_examples in enumerate(
data_iterator(train_dataset,
batch_size=arg_parser.batch_size,
shuffle=arg_parser.shuffle)):
if ((batch_idx % 100) == 0) and batch_idx > 1:
print(
"epoch {} | batch {} | mean running loss {:.2f} | {:.2f} batch/s"
.format(epoch, batch_idx, running_loss / 100,
100 / (time.time() - last_log_time)))
last_log_time = time.time()
running_loss = 0.0
sources, targets = batch_examples[0], batch_examples[1]
example_losses = -model(sources, targets) # (batch_size,)
batch_loss = example_losses.sum()
loss = batch_loss / arg_parser.batch_size
# clip gradient
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),
arg_parser.clip_grad)
if arg_parser.optimizer:
loss.backward()
optimizer.step()
optimizer.optimizer.zero_grad()
else:
optimizer.zero_grad()
loss.backward()
optimizer.step()
# add loss
running_loss += loss.item()
train_loss += loss.item()
print("Epoch train loss : {}".format(
math.sqrt(train_loss /
math.ceil(n_train / arg_parser.batch_size))))
if summary_writer is not None:
summary_writer.add_scalar(
"train/loss",
train_loss / math.ceil(n_train / arg_parser.batch_size),
global_step=epoch)
if (epoch % arg_parser.save_every
== 0) and arg_parser.log and epoch > 0:
if arg_parser.train_load:
save_model(
arg_parser.models_path,
f"{file_name_epoch_indep}_epoch_{epoch + arg_parser.epoch_to_load}.pt",
model, device)
else:
save_model(
arg_parser.models_path,
f"{file_name_epoch_indep}_epoch_{epoch + arg_parser.epoch_to_load}.pt",
model, device)
## TEST
test_loss = 0.0
for batch_idx, batch_examples in enumerate(
data_iterator(test_dataset,
batch_size=arg_parser.batch_size,
shuffle=arg_parser.shuffle)):
with torch.no_grad():
sources, targets = batch_examples[0], batch_examples[1]
example_losses = -model(sources, targets) # (batch_size,)
batch_loss = example_losses.sum()
loss = batch_loss / arg_parser.batch_size
test_loss += loss.item()
if summary_writer is not None:
summary_writer.add_scalar(
"test/loss",
test_loss / math.ceil(n_test / arg_parser.batch_size),
global_step=epoch)
print("TEST loss | epoch {} | {:.2f}".format(
epoch, test_loss / math.ceil(n_test / arg_parser.batch_size)))
return None
self.train_op = self.add_training_op(self.loss)
self.merged_summaries = tf.merge_all_summaries()
self.summary_writer = None
def run_epoch(self, session, input_data, input_labels,
shuffle=True, verbose=True, epoch=0):
orig_X, orig_y = input_data, input_labels
dp = self.config.dropout
# We're interested in keeping track of the loss and accuracy during training
total_loss = []
total_correct_examples = 0
total_processed_examples = 0
total_steps = len(orig_X) / self.config.batch_size
for step, (x, y) in enumerate(
data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
label_size=self.config.label_size, shuffle=shuffle)):
feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
loss, total_correct, _, merged = session.run(
[self.loss, self.correct_predictions, self.train_op, self.merged_summaries],
feed_dict=feed)
if step % 50 == 0:
self.summary_writer.add_summary(merged, epoch * total_steps + step)
total_processed_examples += len(x)
total_correct_examples += total_correct
total_loss.append(loss)
##
if verbose and step % verbose == 0:
sys.stdout.write('\r{} / {} : loss = {}'.format(
def next_batch(self, batch_size, shuffle=True):
"""Return the next `batch_size` examples from this data set.
Takes Y in one-hot encodings only"""
for x, y in data_iterator(self._x, self._y, batch_size, shuffle):
yield x, y
if use_word2vec:
encoder = EncodeSentence(len(vocab2idx),
word_dim,
wordEmbedding=word_emb,
use_lstm=use_lstm).to(device)
else:
encoder = EncodeSentence(len(vocab2idx), word_dim,
use_lstm=use_lstm).to(device)
model = FeedForward(len(tag2idx), tag_dim, word_dim).to(device)
optimizer = optim.Adam(list(model.parameters()) + list(encoder.parameters()))
# Training
for it in range(train_iter):
print("Iteration: {}".format(it))
train_sents, train_tags = shuffle(train_sents, train_tags)
for data, label in data_iterator(train_sents, train_tags, batch_size=100):
optimizer.zero_grad()
train_data = torch.tensor(pad_sents(data), device=device)
train_data = encoder(train_data)
train_label = torch.tensor(pad_sents(label, 1), device=device)
maxlen = train_data.size(1)
loss_sum = torch.tensor([0.], device=device)
for i in range(1, maxlen):
output = model(train_data[:, i], train_label[:, i - 1])
loss_sum += F.nll_loss(output, train_label[:, i])
loss_sum.backward()
optimizer.step()
print("Loss: {:.4f}".format(loss_sum.item() / maxlen))
# Testing
test_data = torch.tensor(pad_sents(test_sents), device=device)