def __init__(self, opts, test_opts=None):
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = 100
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.suffix_dim + self.opts.cap + self.opts.num + self.opts.jk_dim
self.outputs_dim = (1 + self.opts.bi) * self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.suffix_dim > 0:
inputs_list.append(self.add_suffix_embedding())
if self.opts.cap:
inputs_list.append(self.add_cap())
if self.opts.num:
inputs_list.append(self.add_num())
if self.opts.jk_dim > 0:
inputs_list.append(self.add_jackknife_embedding())
inputs_tensor = tf.concat(inputs_list,
2) ## [seq_len, batch_size, inputs_dim]
forward_inputs_tensor = self.add_dropout(inputs_tensor,
self.input_keep_prob)
for i in xrange(self.opts.num_layers):
c, h = self.add_lstm(forward_inputs_tensor, i,
'Forward') ## [seq_len, batch_size, units]
forward_inputs_tensor = self.add_dropout(
h, self.keep_prob) ## [seq_len, batch_size, units]
lstm_outputs = forward_inputs_tensor
cells = self.add_dropout(c, self.keep_prob)
if self.opts.bi:
backward_inputs_tensor = self.add_dropout(
tf.reverse(inputs_tensor, [0]), self.input_keep_prob)
for i in xrange(self.opts.num_layers):
c, h = self.add_lstm(
backward_inputs_tensor, i,
'Backward') ## [seq_len, batch_size, units]
backward_inputs_tensor = self.add_dropout(
h, self.keep_prob) ## [seq_len, batch_size, units]
backward_inputs_tensor = tf.reverse(backward_inputs_tensor, [0])
cells_backward = self.add_dropout(c, self.keep_prob)
lstm_outputs = tf.concat([lstm_outputs, backward_inputs_tensor],
2) ## [seq_len, batch_size, outputs_dim]
cells = tf.concat([cells, cells_backward],
2) ## [seq_len, batch_size, outputs_dim]
lstm_outputs = self.add_attention(lstm_outputs, cells)
#lstm_outputs = self.add_dropout(lstm_outputs, self.keep_prob)
projected_outputs = tf.map_fn(
lambda x: self.add_projection(x),
lstm_outputs) #[seq_len, batch_size, nb_tags]
projected_outputs = tf.transpose(
projected_outputs, perm=[1, 0,
2]) # [batch_size, seq_len, nb_tags]
self.weight = tf.cast(
tf.not_equal(
self.inputs_placeholder_list[0],
tf.zeros(tf.shape(self.inputs_placeholder_list[0]), tf.int32)),
tf.float32) ## [batch_size, seq_len]
self.loss = self.add_loss_op(projected_outputs)
self.train_op = self.add_train_op(self.loss)
self.add_accuracy(projected_outputs)
def __init__(self, opts, test_opts=None, beam_size=0):
## Notation:
## b: batch_size
## d: # units
## n: # tokens in the sentence
## B: beam_size
self.opts = opts
self.opts.bi = 0 ## no bidirecion
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = 100
#self.batch_size = 32
self.beam_size = beam_size
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.suffix_dim + self.opts.cap + self.opts.num + self.opts.jk_dim
self.outputs_dim = (1 + self.opts.bi) * self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.suffix_dim > 0:
inputs_list.append(self.add_suffix_embedding())
if self.opts.cap:
inputs_list.append(self.add_cap())
if self.opts.num:
inputs_list.append(self.add_num())
if self.opts.jk_dim > 0:
inputs_list.append(self.add_jackknife_embedding())
inputs_tensor = tf.concat(inputs_list,
2) ## [seq_len, batch_size, inputs_dim]
#forward_inputs_tensor = inputs_tensor
forward_inputs_tensor = tf.reverse(inputs_tensor, [0])
forward_inputs_tensor = self.add_dropout(forward_inputs_tensor,
self.input_keep_prob)
forward_cells = []
forward_hs = []
for i in xrange(self.opts.num_layers):
c, h = self.add_lstm(forward_inputs_tensor, i,
'Forward') ## [seq_len, batch_size, units]
forward_hs.append(h)
h = self.add_dropout(
h, self.keep_prob) ## [seq_len, batch_size, units]
forward_cells.append(c)
forward_inputs_tensor = h
# if beam_size > 0:
# self.predictions, self.scores, self.back_pointers = self.add_forward_beam_path_mt(forward_inputs_tensor, beam_size) ## [seq_len, batch_size, nb_tags]
# self.weight = tf.not_equal(self.inputs_placeholder_list[0], tf.zeros(tf.shape(self.inputs_placeholder_list[0]), tf.int32)) # [self.batch_size, seq_len]
# self.weight_beam = tf.reshape(tf.tile(self.weight, [1, beam_size]), [-1, tf.shape(self.weight)[1]]) # [batch_size, seq_len]
# else:
self.predictions, projected_outputs = self.add_forward_path_mt(
forward_cells, forward_hs,
forward_inputs_tensor) ## [seq_len, batch_size, nb_tags]
self.weight = tf.cast(
tf.not_equal(
self.inputs_placeholder_list[0],
tf.zeros(tf.shape(self.inputs_placeholder_list[0]), tf.int32)),
tf.float32) ## [batch_size, seq_len]
self.add_lm_accuracy()
self.loss = self.add_loss_op(projected_outputs)
self.train_op = self.add_train_op(self.loss)
def __init__(self, opts, test_opts=None):
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = 100
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.jk_dim + self.opts.stag_dim
self.outputs_dim = (1 + self.opts.bi) * self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.jk_dim:
inputs_list.append(self.add_jackknife_embedding())
if self.opts.stag_dim > 0:
inputs_list.append(self.add_stag_embedding())
inputs_tensor = tf.concat(inputs_list,
2) ## [seq_len, batch_size, inputs_dim]
forward_inputs_tensor = self.add_dropout(inputs_tensor,
self.input_keep_prob)
for i in xrange(self.opts.num_layers):
forward_inputs_tensor = self.add_dropout(
self.add_lstm(forward_inputs_tensor, i, 'Forward'),
self.keep_prob) ## [seq_len, batch_size, units]
lstm_outputs = forward_inputs_tensor
if self.opts.bi:
backward_inputs_tensor = self.add_dropout(
tf.reverse(inputs_tensor, [0]), self.input_keep_prob)
for i in xrange(self.opts.num_layers):
backward_inputs_tensor = self.add_dropout(
self.add_lstm(backward_inputs_tensor, i, 'Backward'),
self.keep_prob) ## [seq_len, batch_size, units]
backward_inputs_tensor = tf.reverse(backward_inputs_tensor, [0])
lstm_outputs = tf.concat([lstm_outputs, backward_inputs_tensor],
2) ## [seq_len, batch_size, outputs_dim]
self.arc_outputs, rel_outputs, self.rel_scores = self.add_biaffine(
lstm_outputs)
# projected_outputs = tf.map_fn(lambda x: self.add_projection(x), lstm_outputs) #[seq_len, batch_size, nb_tags]
# projected_outputs = tf.transpose(projected_outputs, perm=[1, 0, 2]) # [batch_size, seq_len, nb_tags]
inputs_shape = tf.shape(self.inputs_placeholder_dict['words'])
self.weight = tf.cast(
tf.not_equal(self.inputs_placeholder_dict['words'],
tf.zeros(inputs_shape, tf.int32)),
tf.float32) * tf.cast(
tf.not_equal(
self.inputs_placeholder_dict['words'],
tf.ones(inputs_shape, tf.int32) *
self.loader.word_index['<-root->']),
tf.float32) ## [batch_size, seq_len]
## no need to worry about the heads of <-root-> and zero-pads
self.loss = self.add_loss_op(
self.arc_outputs,
self.inputs_placeholder_dict['arcs']) + self.add_loss_op(
rel_outputs, self.inputs_placeholder_dict['rels'])
self.predicted_arcs, self.UAS = self.add_accuracy(
self.arc_outputs, self.inputs_placeholder_dict['arcs'])
self.predicted_rels, self.rel_acc = self.add_accuracy(
rel_outputs, self.inputs_placeholder_dict['rels'])
self.train_op = self.add_train_op(self.loss)
def __init__(self, opts, test_opts=None, beam_size=0):
## Notation:
## b: batch_size
## d: # units
## n: # tokens in the sentence
## B: beam_size
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = opts.batch_size
self.beam_size = beam_size
print('beam')
print(beam_size)
self.get_features()
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.suffix_dim + self.opts.cap + self.opts.num + self.opts.jk_dim + self.opts.nb_filters
self.outputs_dim = (1+self.opts.bi)*self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.suffix_dim > 0:
inputs_list.append(self.add_suffix_embedding())
if self.opts.cap:
inputs_list.append(self.add_cap())
if self.opts.num:
inputs_list.append(self.add_num())
if self.opts.jk_dim > 0:
inputs_list.append(self.add_jackknife_embedding())
if self.opts.chars_dim > 0:
inputs_list.append(self.add_char_embedding())
inputs_tensor = tf.concat(inputs_list, 2) ## [seq_len, batch_size, inputs_dim]
forward_inputs_tensor = self.add_dropout(inputs_tensor, self.input_keep_prob)
for i in xrange(self.opts.num_layers):
forward_inputs_tensor = self.add_dropout(self.add_lstm(forward_inputs_tensor, i, 'Forward'), self.keep_prob) ## [seq_len, batch_size, units]
lstm_outputs = forward_inputs_tensor
if self.opts.bi:
backward_inputs_tensor = self.add_dropout(tf.reverse(inputs_tensor, [0]), self.input_keep_prob)
for i in xrange(self.opts.num_layers):
backward_inputs_tensor = self.add_dropout(self.add_lstm(backward_inputs_tensor, i, 'Backward'), self.keep_prob) ## [seq_len, batch_size, units]
backward_inputs_tensor = tf.reverse(backward_inputs_tensor, [0])
lstm_outputs = tf.concat([lstm_outputs, backward_inputs_tensor], 2) ## [seq_len, batch_size, outputs_dim]
crf_inputs = tf.map_fn(lambda x: self.feed_to_crf(x), lstm_outputs) ## [seq_len, batch_size, outputs_dim] => [seq_len, batch_size, lm]
crf_inputs = self.add_dropout(crf_inputs, self.keep_prob)
# if beam_size > 0:
# self.predictions, self.scores, self.back_pointers = self.add_forward_beam_path(forward_inputs_tensor, backward_inputs_tensor, beam_size) ## [seq_len, batch_size, nb_tags]
# self.weight = tf.not_equal(self.inputs_placeholder_dict['words'], tf.zeros(tf.shape(self.inputs_placeholder_dict['words']), tf.int32)) # [self.batch_size, seq_len]
# self.weight_beam = tf.reshape(tf.tile(self.weight, [1, beam_size]), [-1, tf.shape(self.weight)[1]]) # [batch_size, seq_len]
# else:
self.predictions, projected_outputs = self.add_crf_path(crf_inputs) ## [seq_len, batch_size, nb_tags]
self.weight = tf.cast(tf.not_equal(self.inputs_placeholder_dict['words'], tf.zeros(tf.shape(self.inputs_placeholder_dict['words']), tf.int32)), tf.float32) ## [batch_size, seq_len]
self.add_lm_accuracy()
self.loss = self.add_loss_op(projected_outputs)
self.train_op = self.add_train_op(self.loss)
def __init__(self, opts, test_opts=None):
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = 100
self.get_features()
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.jk_dim + self.opts.stag_dim + self.opts.nb_filters
self.outputs_dim = (1+self.opts.bi)*self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.jk_dim:
inputs_list.append(self.add_jackknife_embedding())
if self.opts.stag_dim > 0:
inputs_list.append(self.add_stag_embedding())
if self.opts.chars_dim > 0:
inputs_list.append(self.add_char_embedding())
inputs_tensor = tf.concat(inputs_list, 2) ## [seq_len, batch_size, inputs_dim]
inputs_tensor = self.add_dropout(inputs_tensor, self.input_keep_prob)
inputs_shape = tf.shape(self.inputs_placeholder_dict['words'])
## no need to worry about the heads of <-root-> and zero-pads
## Let's get those non-padding places so we can reinitialize hidden states after each padding in the backward path
### because the backward path starts with zero pads.
self.weight = tf.cast(tf.not_equal(self.inputs_placeholder_dict['words'], tf.zeros(inputs_shape, tf.int32)), tf.float32) ## [batch_size, seq_len]
for i in xrange(self.opts.num_layers):
forward_outputs_tensor = self.add_dropout(self.add_lstm(inputs_tensor, i, 'Forward'), self.keep_prob) ## [seq_len, batch_size, units]
if self.opts.bi:
backward_outputs_tensor = self.add_dropout(self.add_lstm(tf.reverse(inputs_tensor, [0]), i, 'Backward', True), self.keep_prob) ## [seq_len, batch_size, units]
inputs_tensor = tf.concat([forward_outputs_tensor, tf.reverse(backward_outputs_tensor, [0])], 2)
else:
inputs_tensor = forward_outputs_tensor
self.weight = self.weight*tf.cast(tf.not_equal(self.inputs_placeholder_dict['words'], tf.ones(inputs_shape, tf.int32)*self.loader.word_index['<-root->']), tf.float32) ## [batch_size, seq_len]
lstm_outputs = inputs_tensor ## [seq_len, batch_size, outputs_dim]
self.arc_outputs, rel_outputs, self.rel_scores, joint_output, joint_output_jk = self.add_biaffine(lstm_outputs)
# projected_outputs = tf.map_fn(lambda x: self.add_projection(x), lstm_outputs) #[seq_len, batch_size, nb_tags]
# projected_outputs = tf.transpose(projected_outputs, perm=[1, 0, 2]) # [batch_size, seq_len, nb_tags]
self.loss = self.add_loss_op(self.arc_outputs, self.inputs_placeholder_dict['arcs']) + self.add_loss_op(rel_outputs, self.inputs_placeholder_dict['rels']) + self.add_loss_op(joint_output, self.inputs_placeholder_dict['stags']) + self.add_loss_op(joint_output_jk, self.inputs_placeholder_dict['jk'])
self.add_probs(joint_output)
self.predicted_arcs, self.UAS = self.add_accuracy(self.arc_outputs, self.inputs_placeholder_dict['arcs'])
self.predicted_rels, self.rel_acc = self.add_accuracy(rel_outputs, self.inputs_placeholder_dict['rels'])
self.predicted_stags, self.stag_acc = self.add_accuracy(joint_output, self.inputs_placeholder_dict['stags'])
self.predicted_jk, self.jk_acc = self.add_accuracy(joint_output_jk, self.inputs_placeholder_dict['jk'])
self.train_op = self.add_train_op(self.loss)
def __init__(self, opts, test_opts=None):
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = opts.batch_size
self.get_features()
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.suffix_dim + self.opts.cap + self.opts.num + self.opts.jk_dim + self.opts.nb_filters
self.outputs_dim = (1+self.opts.bi)*self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.suffix_dim > 0:
inputs_list.append(self.add_suffix_embedding())
if self.opts.cap:
inputs_list.append(self.add_cap())
if self.opts.num:
inputs_list.append(self.add_num())
if self.opts.jk_dim > 0:
inputs_list.append(self.add_jackknife_embedding())
if self.opts.chars_dim > 0:
inputs_list.append(self.add_char_embedding())
inputs_tensor = tf.concat(inputs_list, 2) ## [seq_len, batch_size, inputs_dim]
inputs_tensor = self.add_dropout(inputs_tensor, self.input_keep_prob)
self.weight = tf.cast(tf.not_equal(self.inputs_placeholder_dict['words'], tf.zeros(tf.shape(self.inputs_placeholder_dict['words']), tf.int32)), tf.float32) ## [batch_size, seq_len]
for i in xrange(self.opts.num_layers):
forward_outputs_tensor = self.add_dropout(self.add_lstm(inputs_tensor, i, 'Forward'), self.keep_prob) ## [seq_len, batch_size, units]
if self.opts.bi:
backward_outputs_tensor = self.add_dropout(self.add_lstm(tf.reverse(inputs_tensor, [0]), i, 'Backward', True), self.keep_prob) ## [seq_len, batch_size, units]
inputs_tensor = tf.concat([forward_outputs_tensor, tf.reverse(backward_outputs_tensor, [0])], 2)
else:
inputs_tensor = forward_outputs_tensor
lstm_outputs = inputs_tensor ## [seq_len, batch_size, outputs_dim]
self.predictions, projected_outputs = self.add_forward_path(lstm_outputs) ## [seq_len, batch_size, nb_tags]
self.add_lm_accuracy()
self.loss = self.add_loss_op(projected_outputs)
self.train_op = self.add_train_op(self.loss)
# if self.opts.bi:
self.add_accuracy(projected_outputs)
def __init__(self, opts, test_opts=None):
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = opts.batch_size
self.get_features()
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.suffix_dim + self.opts.cap + self.opts.num + self.opts.jk_dim + self.opts.nb_filters
self.outputs_dim = (1 + self.opts.bi) * self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.suffix_dim > 0:
inputs_list.append(self.add_suffix_embedding())
if self.opts.cap:
inputs_list.append(self.add_cap())
if self.opts.num:
inputs_list.append(self.add_num())
if self.opts.jk_dim > 0:
inputs_list.append(self.add_jackknife_embedding())
self.adv_embedding = inputs_list[:]
if self.opts.chars_dim > 0:
inputs_list.append(self.add_char_embedding())
inputs_tensor = tf.concat(inputs_list,
2) ## [seq_len, batch_size, inputs_dim]
self.weight = tf.cast(
tf.not_equal(
self.inputs_placeholder_dict['words'],
tf.zeros(tf.shape(self.inputs_placeholder_dict['words']),
tf.int32)), tf.float32) ## [batch_size, seq_len]
clean_loss, projected_outputs = self.feed_network_inputs(inputs_tensor)
self.add_accuracy(projected_outputs)
adv_loss, _ = self.add_adversarial_loss(
clean_loss) ## do not care about adversarial prediction accuracy
alpha = 0.5
#alpha = 1.0
self.loss = alpha * clean_loss + (1.0 - alpha) * adv_loss
self.train_op = self.add_train_op(self.loss)
def __init__(self, opts, test_opts=None, beam_size=16):
## Notation:
## b: batch_size
## d: # units
## n: # tokens in the sentence
## B: beam_size
self.opts = opts
self.test_opts = test_opts
self.loader = Dataset(opts, test_opts)
self.batch_size = 32
self.beam_size = beam_size
self.add_placeholders()
self.inputs_dim = self.opts.embedding_dim + self.opts.suffix_dim + self.opts.cap + self.opts.num + self.opts.jk_dim
self.outputs_dim = (1 + self.opts.bi) * self.opts.units
inputs_list = [self.add_word_embedding()]
if self.opts.suffix_dim > 0:
inputs_list.append(self.add_suffix_embedding())
if self.opts.cap:
inputs_list.append(self.add_cap())
if self.opts.num:
inputs_list.append(self.add_num())
if self.opts.jk_dim > 0:
inputs_list.append(self.add_jackknife_embedding())
inputs_tensor = tf.concat(inputs_list,
2) ## [seq_len, batch_size, inputs_dim]
forward_inputs_tensor = inputs_tensor
## Backward path is deterministic, just run it first and make it embeddings
if self.opts.bi:
backward_inputs_tensor = self.add_dropout(
tf.reverse(inputs_tensor, [0]), self.input_keep_prob)
for i in xrange(self.opts.num_layers):
backward_inputs_tensor = self.add_dropout(
self.add_lstm(backward_inputs_tensor, i, 'Backward'),
self.keep_prob) ## [seq_len, batch_size, units]
backward_inputs_tensor = tf.reverse(
backward_inputs_tensor, [0]) ## [seq_len, batch_size, units]
## backward path is done
forward_inputs_tensor = self.add_dropout(forward_inputs_tensor,
self.input_keep_prob)
if beam_size > 0:
self.weight = tf.cast(
tf.not_equal(
self.inputs_placeholder_list[0],
tf.zeros(tf.shape(self.inputs_placeholder_list[0]),
tf.int32)),
tf.float32) # [self.batch_size, seq_len]
self.predictions, self.scores, self.back_pointers = self.add_forward_beam_path(
forward_inputs_tensor, backward_inputs_tensor,
beam_size) ## [seq_len, batch_size, nb_tags]
self.weight_beam = tf.reshape(
tf.tile(self.weight, [1, beam_size]),
[-1, tf.shape(self.weight)[1]]) # [batch_size, seq_len]
self.accuracy = self.loss ## for dummy
#_, projected_outputs = self.add_forward_path(forward_inputs_tensor, backward_inputs_tensor, True) ## [seq_len, batch_size, nb_tags]
#self.loss += self.add_loss_op(projected_outputs)
self.train_op = self.add_train_op(self.loss)
else:
self.predictions, projected_outputs = self.add_forward_path(
forward_inputs_tensor,
backward_inputs_tensor) ## [seq_len, batch_size, nb_tags]
self.weight = tf.cast(
tf.not_equal(
self.inputs_placeholder_list[0],
tf.zeros(tf.shape(self.inputs_placeholder_list[0]),
tf.int32)), tf.float32) ## [batch_size, seq_len]
self.add_lm_accuracy()
self.loss = self.add_loss_op(projected_outputs)
self.train_op = self.add_train_op(self.loss)