def __init__(self, is_training=True, is_predict=False):
super(RnnEncoder, self).__init__()
self.is_training = is_training
self.is_predict = is_predict
vocabulary.init()
if FLAGS.encoder_end_mark == '</S>':
self.end_id = vocabulary.end_id()
else:
self.end_id = vocabulary.go_id(
) #NOTICE NUM_RESERVED_IDS must >= 3 TODO
assert self.end_id != vocabulary.vocab.unk_id(
), 'input vocab generated without end id'
create_rnn_cell = functools.partial(melt.create_rnn_cell,
num_units=FLAGS.rnn_hidden_size,
is_training=is_training,
keep_prob=FLAGS.keep_prob,
num_layers=FLAGS.num_layers,
cell_type=FLAGS.cell)
#follow models/textsum
self.cell = create_rnn_cell(
initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=123))
if FLAGS.rnn_method == melt.rnn.EncodeMethod.bidirectional:
self.bwcell = create_rnn_cell(
initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=113))
else:
self.bwcell = None
def __init__(self, is_training=True, is_predict=False):
self.is_training = is_training
self.is_predict = is_predict
vocabulary.init()
vocab_size = vocabulary.get_vocab_size()
self.vocab_size = vocab_size
self.end_id = vocabulary.end_id()
self.get_start_id()
assert self.end_id != vocabulary.vocab.unk_id(), 'input vocab generated without end id'
self.emb_dim = emb_dim = FLAGS.emb_dim
#--- for perf problem here exchange w_t and w https://github.com/tensorflow/tensorflow/issues/4138
self.num_units = num_units = FLAGS.rnn_hidden_size
with tf.variable_scope('output_projection'):
self.w_t = melt.variable.get_weights_truncated('w',
[vocab_size, num_units],
stddev=FLAGS.weight_stddev)
self.w = tf.transpose(self.w_t)
self.v = melt.variable.get_weights_truncated('v',
[vocab_size],
stddev=FLAGS.weight_stddev)
self.cell = melt.create_rnn_cell(
num_units=num_units,
is_training=is_training,
keep_prob=FLAGS.keep_prob,
num_layers=FLAGS.num_layers,
cell_type=FLAGS.cell)
num_sampled = FLAGS.num_sampled if not (is_predict and FLAGS.predict_no_sample) else 0
self.softmax_loss_function = self.gen_sampled_softmax_loss_function(num_sampled)
def __init__(self, is_training=True, is_predict=False):
self.scope = 'rnn'
self.is_training = is_training
self.is_predict = is_predict
assert not (FLAGS.decode_copy and FLAGS.decode_use_alignment)
vocabulary.init()
vocab_size = vocabulary.get_vocab_size()
self.vocab_size = vocab_size
self.end_id = vocabulary.end_id()
self.get_start_id()
assert self.end_id != vocabulary.vocab.unk_id(
), 'input vocab generated without end id'
self.emb_dim = emb_dim = FLAGS.emb_dim
#--- for perf problem here exchange w_t and w https://github.com/tensorflow/tensorflow/issues/4138
self.num_units = num_units = FLAGS.rnn_hidden_size
with tf.variable_scope('output_projection'):
self.w_t = melt.variable.get_weights_truncated(
'w_t', [vocab_size, num_units], stddev=FLAGS.weight_stddev)
#weights
self.w = tf.transpose(self.w_t)
#biases
self.v = melt.variable.get_weights_truncated(
'v', [vocab_size], stddev=FLAGS.weight_stddev)
#TODO https://github.com/tensorflow/tensorflow/issues/6761 tf 1.0 will fail if not scope='rnn' the same as when using self.cell...
self.cell = melt.create_rnn_cell(num_units=num_units,
is_training=is_training,
keep_prob=FLAGS.keep_prob,
num_layers=FLAGS.num_layers,
cell_type=FLAGS.cell)
num_sampled = FLAGS.num_sampled if not (
is_predict and FLAGS.predict_no_sample) else 0
self.softmax_loss_function = melt.seq2seq.gen_sampled_softmax_loss_function(
num_sampled,
self.vocab_size,
self.w_t,
self.v,
FLAGS.log_uniform_sample,
is_predict=self.is_predict,
sample_seed=FLAGS.predict_sample_seed,
vocabulary=vocabulary)
def get_decodes(shuffle_then_decode, dynamic_batch_length):
vocabulary.init()
global encoder_end_id
if FLAGS.encoder_end_mark == '</S>':
encoder_end_id = vocabulary.end_id()
else:
encoder_end_id = vocabulary.go_id(
) #NOTICE NUM_RESERVED_IDS must >= 3 TODO
assert encoder_end_id != vocabulary.vocab.unk_id(
), 'input vocab generated without end id'
if shuffle_then_decode:
inputs = melt.shuffle_then_decode.inputs
decode = lambda x: decode_examples(x, dynamic_batch_length)
else:
inputs = melt.decode_then_shuffle.inputs
decode = lambda x: decode_example(x, dynamic_batch_length)
return inputs, decode
def get_decoder_end_id():
if (FLAGS.input_with_end_mark):
return None
else:
return vocabulary.end_id()
def __init__(self, is_training=True, is_predict=False):
self.scope = 'rnn'
self.is_training = is_training
self.is_predict = is_predict
assert not (FLAGS.decode_copy and FLAGS.decode_use_alignment)
vocabulary.init()
vocab_size = vocabulary.get_vocab_size()
self.vocab_size = vocab_size
self.end_id = vocabulary.end_id()
self.start_id = None
self.get_start_id()
assert self.end_id != vocabulary.vocab.unk_id(
), 'input vocab generated without end id'
self.emb_dim = emb_dim = FLAGS.emb_dim
#--- for perf problem here exchange w_t and w https://github.com/tensorflow/tensorflow/issues/4138
self.num_units = num_units = FLAGS.rnn_hidden_size
with tf.variable_scope('output_projection'):
self.w_t = melt.variable.get_weights_truncated(
'w_t', [vocab_size, num_units], stddev=FLAGS.weight_stddev)
#weights
self.w = tf.transpose(self.w_t)
#biases
self.v = melt.variable.get_weights_truncated(
'v', [vocab_size], stddev=FLAGS.weight_stddev)
#TODO https://github.com/tensorflow/tensorflow/issues/6761 tf 1.0 will fail if not scope='rnn' the same as when using self.cell...
self.cell = melt.create_rnn_cell(num_units=num_units,
is_training=is_training,
keep_prob=FLAGS.keep_prob,
num_layers=FLAGS.num_layers,
cell_type=FLAGS.cell)
self.num_sampled = num_sampled = FLAGS.num_sampled if not (
is_predict and FLAGS.predict_no_sample) else 0
#self.softmax_loss_function is None means not need sample
self.softmax_loss_function = None
if FLAGS.gen_only:
self.softmax_loss_function = melt.seq2seq.gen_sampled_softmax_loss_function(
num_sampled,
self.vocab_size,
weights=self.w_t,
biases=self.v,
log_uniform_sample=FLAGS.log_uniform_sample,
is_predict=self.is_predict,
sample_seed=FLAGS.predict_sample_seed,
vocabulary=vocabulary)
if FLAGS.use_attention:
print('----attention_option:', FLAGS.attention_option)
if FLAGS.gen_copy_switch or FLAGS.gen_copy or FLAGS.copy_only:
assert FLAGS.use_attention is True, 'must use attention if not gen_only mode seq2seq'
FLAGS.gen_only = False
if FLAGS.gen_copy_switch:
print('-------gen copy switch mode!')
FLAGS.gen_copy = False
FLAGS.copy_only = False
elif FLAGS.gen_copy:
print('-------gen copy mode !')
FLAGS.copy_only = False
else:
print('-------copy only mode !')
else:
print('--------gen only mode')
#if use copy mode use score as alignment(no softmax)
self.score_as_alignment = False if FLAGS.gen_only else True
#gen only output_fn
self.output_fn = lambda cell_output: melt.dense(
cell_output, self.w, self.v)
def copy_output(indices, batch_size, cell_output, cell_state):
alignments = cell_state.alignments
updates = alignments
return tf.scatter_nd(indices,
updates,
shape=[batch_size, self.vocab_size])
self.copy_output_fn = copy_output
#one problem is big memory for large vocabulary
def gen_copy_output(indices, batch_size, cell_output, cell_state):
gen_logits = self.output_fn(cell_output)
copy_logits = copy_output(indices, batch_size, cell_output,
cell_state)
if FLAGS.gen_copy_switch:
gen_probability = cell_state.gen_probability
#[batch_size, 1] * [batch_size, vocab_size]
return gen_probability * tf.nn.softmax(gen_logits) + (
1 - gen_probability) * tf.nn.softmax(copy_logits)
else:
return gen_logits + copy_logits
self.gen_copy_output_fn = gen_copy_output
def gen_copy_output_train(time, indices, targets, sampled_values,
batch_size, cell_output, cell_state):
if self.softmax_loss_function is not None:
labels = tf.slice(targets, [0, time], [-1, 1])
sampled, true_expected_count, sampled_expected_count = sampled_values
sampled_values = \
sampled, tf.slice(tf.reshape(true_expected_count, [batch_size, -1]), [0, time], [-1, 1]), sampled_expected_count
sampled_ids, sampled_logits = melt.nn.compute_sampled_ids_and_logits(
weights=self.w_t,
biases=self.v,
labels=labels,
inputs=cell_output,
num_sampled=self.num_sampled,
num_classes=self.vocab_size,
sampled_values=sampled_values,
remove_accidental_hits=False)
gen_indices = melt.batch_values_to_indices(
tf.to_int32(sampled_ids))
gen_logits = tf.scatter_nd(gen_indices,
sampled_logits,
shape=[batch_size, self.vocab_size])
else:
gen_logits = self.output_fn(cell_output)
copy_logits = copy_output(indices, batch_size, cell_output,
cell_state)
if FLAGS.gen_copy_switch:
#gen_copy_switch == True.
gen_probability = cell_state.gen_probability
return gen_probability * tf.nn.softmax(gen_logits) + (
1 - gen_probability) * tf.nn.softmax(copy_logits)
else:
return gen_logits + copy_logits
self.gen_copy_output_train_fn = gen_copy_output_train