def inference(params):
embedding_size = params['embedding_size']
vocab_size = params['vocab_size']
sentence_len = params['num_words_before'] + params['num_words_after']
embedding_wd = utils.get_dict_value(params, 'embedding_wd')
embedding_device = utils.get_dict_value(params, 'embedding_device')
embedding_initializer = utils.get_dict_value(params, 'embedding_initializer')
embedding_keep_prob = utils.get_dict_value(params, 'embedding_keep_prob')
word_embedding_size = utils.get_dict_value(params, 'word_embedding_size', embedding_size)
word_len = params['word_len']
print("USING EMBEDDING DEVICE %s" %embedding_device)
if embedding_device is not None:
with tf.device(embedding_device):
embedding_matrix = nlp.variable_with_weight_decay('embedding_matrix',
[vocab_size, embedding_size],
initializer=embedding_initializer, wd=embedding_wd)
word_embedding_matrix = nlp.variable_with_weight_decay('word_embedding_matrix', [128, word_embedding_size],
initializer=embedding_initializer, wd=embedding_wd)
else:
embedding_matrix = nlp.variable_with_weight_decay('embedding_matrix', [vocab_size, embedding_size],
initializer=embedding_initializer, wd=embedding_wd)
word_embedding_matrix = nlp.variable_with_weight_decay('word_embedding_matrix', [128, word_embedding_size],
initializer=embedding_initializer, wd=embedding_wd)
input_sentence = tf.placeholder(tf.int32, [None, sentence_len], 'sentence')
input_word = tf.placeholder(tf.int32, [None, word_len], 'word')
emb_sentence = tf.nn.embedding_lookup(embedding_matrix, input_sentence, 'emb_sentence')
emb_word = tf.nn.embedding_lookup(word_embedding_matrix, input_word, 'emb_word')
if embedding_keep_prob is not None and embedding_keep_prob < 1.0:
[emb_sentence],_ = core.dropout([emb_sentence], [embedding_keep_prob])
[emb_word], _ = core.dropout([emb_word], [embedding_keep_prob])
enc_sentence, _ = sentence_encoder(emb_sentence, emb_word, params)
return enc_sentence, None
def conv2d_array(input,
sizes,
widths,
heights,
keep_probs=None,
w_wds=0.005,
b_wds=0.000,
w_initializers=tf.truncated_normal_initializer(stddev=0.05),
b_initializers=tf.truncated_normal_initializer(stddev=0.05),
name="conv2d_array",
vlistin=None,
vlistout=None):
w_wd_list = expand_list(w_wds, sizes)
b_wd_list = expand_list(b_wds, sizes)
keep_prob_list = expand_list(keep_probs, sizes)
w_initializer_list = expand_list(w_initializers, sizes)
b_initializer_list = expand_list(b_initializers, sizes)
input_size = input.get_shape().as_list()[-1]
print('embedding_dimension = ' + str(input_size))
with tf.variable_scope(name):
for i, (width, height, size, w_wd, b_wd, keep_prob, w_initializer,
b_initializer) in enumerate(
zip(widths, heights, sizes, w_wd_list, b_wd_list,
keep_prob_list, w_initializer_list,
b_initializer_list)):
with tf.variable_scope('conv%s' % str(i)) as scope:
w = variable_with_weight_decay(
'w', [width, height, input_size, size],
initializer=w_initializer,
wd=w_wd,
vlist=vlistin)
if vlistout is not None:
vlistout.append(w)
b = variable_with_weight_decay('b', [size],
initializer=b_initializer,
wd=b_wd,
vlist=vlistin)
if vlistout is not None:
vlistout.append(b)
w_out = tf.nn.conv2d(input,
filter=w,
strides=[1, 1, 1, 1],
padding='SAME')
b_out = tf.nn.bias_add(w_out, b)
out = tf.nn.relu(b_out)
if keep_prob is not None and keep_prob < 1.0:
[out], _ = core.dropout([out], keep_prob)
input_size = size
input = out
return out
def inference(params):
embedding_size = params['embedding_size']
vocab_size = params['vocab_size']
sentence_len = params['num_words_before'] + params['num_words_after']
embedding_wd = utils.get_dict_value(params, 'embedding_wd', 0.0)
embedding_device = utils.get_dict_value(params, 'embedding_device', None)
embedding_initializer = utils.get_dict_value(params,
'embedding_initializer', None)
embedding_keep_prob = utils.get_dict_value(params, 'embedding_keep_prob',
1.0)
print("USING EMBEDDING DEVICE %s" % embedding_device)
if embedding_device is not None:
with tf.device(embedding_device):
embedding_matrix = nlp.variable_with_weight_decay(
'embedding_matrix', [vocab_size, embedding_size],
initializer=embedding_initializer,
wd=embedding_wd)
else:
embedding_matrix = nlp.variable_with_weight_decay(
'embedding_matrix', [vocab_size, embedding_size],
initializer=embedding_initializer,
wd=embedding_wd)
timing_info = tf.placeholder(tf.float32, [None, sentence_len],
'timing_info')
input_sentence = tf.placeholder(tf.int32, [None, sentence_len],
'tcids_before')
emb_sentence = tf.nn.embedding_lookup(embedding_matrix, input_sentence,
'emb_sentence')
if embedding_keep_prob is not None and embedding_keep_prob < 1.0:
[emb_sentence], _ = core.dropout([emb_sentence], [embedding_keep_prob])
timing = tf.reshape(timing_info, np.asarray([-1, sentence_len, 1]))
emb_sentence = tf.concat([emb_sentence, timing], axis=2)
enc_sentence, _ = sentence_encoder(emb_sentence, params)
return enc_sentence, None
def inference(params):
# batch_size = params['batch_size']
num_steps = params['num_steps']
cell_size = params['cell_size']
vocab_size = params['vocab_size']
num_layers = params['num_layers']
cell_type = params['cell_type']
is_training = tf.get_default_graph().get_tensor_by_name('is_training:0')
embedding_wd = utils.get_dict_value(params, 'embedding_wd')
# embedding_device = utils.get_dict_value(params, 'embedding_device')
embedding_initializer = utils.get_dict_value(params,
'embedding_initializer')
embedding_keep_prob = utils.get_dict_value(params, 'embedding_keep_prob')
rnn_dropout_keep_prob = utils.get_dict_value(params,
'rnn_dropout_keep_prob', 1.0)
cell_activation = utils.get_dict_value(params, 'cell_activation', None)
embedding_matrix = nlp.variable_with_weight_decay(
'embedding_matrix', [vocab_size, cell_size],
initializer=embedding_initializer,
wd=embedding_wd)
words = tf.placeholder(tf.int32, [None, None], name='x')
emb_words = tf.nn.embedding_lookup(embedding_matrix, words, 'emb_words')
# add dropout if needed
if embedding_keep_prob is not None and embedding_keep_prob < 1.0:
[emb_words], _ = core.dropout([emb_words], [embedding_keep_prob])
if num_layers > 1:
cell_list = []
for _ in range(num_layers):
if cell_type == 'GRU':
cell = tf.contrib.rnn.GRUCell(cell_size)
elif cell_type == 'BlockLSTM':
cell = tf.contrib.rnn.LSTMBlockCell(cell_size)
else:
cell = tf.contrib.rnn.BasicLSTMCell(cell_size,
activation=cell_activation)
if rnn_dropout_keep_prob < 1.00:
[cell], _ = core.rnn_dropout([cell], [rnn_dropout_keep_prob])
cell_list.append(cell)
cell = tf.contrib.rnn.MultiRNNCell(cell_list, state_is_tuple=True)
else:
if cell_type == 'GRU':
cell = tf.contrib.rnn.GRUCell(cell_size)
elif cell_type == 'BlockLSTM':
cell = tf.contrib.rnn.LSTMBlockCell(cell_size)
else:
cell = tf.contrib.rnn.BasicLSTMCell(cell_size,
activation=cell_activation)
if rnn_dropout_keep_prob < 1.00:
[cell], _ = core.rnn_dropout([cell], [rnn_dropout_keep_prob])
"""
# change for GRU. not yet working!
+ if cell_type == 'GRU':
+ state_placeholder = tf.placeholder(tf.float32, [num_layers, None, cell_size], name='state')
+ l = tf.unstack(state_placeholder, axis=0)
+ state = tf.reshape(state_placeholder, [None, num_layers, cell_size])
+ else:
+ state_placeholder = tf.placeholder(tf.float32, [num_layers, 2, None, cell_size], name='state')
+ l = tf.unstack(state_placeholder, axis=0)
+ state = tuple(
+ [tf.nn.rnn_cell.LSTMStateTuple(l[idx][0], l[idx][1])
+ for idx in range(num_layers)]
+ )
"""
state_placeholder = tf.placeholder(tf.float32,
[num_layers, 2, None, cell_size],
name='state')
l = tf.unstack(state_placeholder, axis=0)
state = tuple([
tf.nn.rnn_cell.LSTMStateTuple(l[idx][0], l[idx][1])
for idx in range(num_layers)
])
outputs, final_state = tf.nn.dynamic_rnn(cell,
emb_words,
initial_state=state)
final = tf.identity(final_state, name='final_state')
output = tf.reshape(tf.concat(outputs, 1), [-1, cell_size])
softmax_w = nlp.variable_with_weight_decay('softmax_w',
[cell_size, vocab_size])
softmax_b = nlp.variable_with_weight_decay('softmax_b', [vocab_size])
logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)
logits = tf.reshape(logits, [-1, num_steps, vocab_size],
name='output_logits')
if utils.get_dict_value(params, 'use_single_sm', False):
smei = tf.placeholder(tf.int32, [None, None],
name='smei') # softmax evaluation index
exp_logits = tf.exp(logits)
numerator = tf.gather_nd(
exp_logits,
tf.stack([
tf.tile(tf.expand_dims(tf.range(tf.shape(smei)[0]), 1),
[1, tf.shape(smei)[1]]),
tf.transpose(
tf.tile(tf.expand_dims(tf.range(tf.shape(smei)[1]), 1),
[1, tf.shape(smei)[0]])), smei
], 2))
logits_smei = tf.divide(numerator, tf.reduce_sum(exp_logits, axis=-1),
'output_single_sm')
logits_sm = tf.nn.softmax(logits, name='output_logits_sm')
return [logits]