def inference(params): embedding_size = params['embedding_size'] vocab_size = params['vocab_size'] sentence_len = params['num_words_before'] + params['num_words_after'] embedding_wd = utils.get_dict_value(params, 'embedding_wd') embedding_device = utils.get_dict_value(params, 'embedding_device') embedding_initializer = utils.get_dict_value(params, 'embedding_initializer') embedding_keep_prob = utils.get_dict_value(params, 'embedding_keep_prob') word_embedding_size = utils.get_dict_value(params, 'word_embedding_size', embedding_size) word_len = params['word_len'] print("USING EMBEDDING DEVICE %s" %embedding_device) if embedding_device is not None: with tf.device(embedding_device): embedding_matrix = nlp.variable_with_weight_decay('embedding_matrix', [vocab_size, embedding_size], initializer=embedding_initializer, wd=embedding_wd) word_embedding_matrix = nlp.variable_with_weight_decay('word_embedding_matrix', [128, word_embedding_size], initializer=embedding_initializer, wd=embedding_wd) else: embedding_matrix = nlp.variable_with_weight_decay('embedding_matrix', [vocab_size, embedding_size], initializer=embedding_initializer, wd=embedding_wd) word_embedding_matrix = nlp.variable_with_weight_decay('word_embedding_matrix', [128, word_embedding_size], initializer=embedding_initializer, wd=embedding_wd) input_sentence = tf.placeholder(tf.int32, [None, sentence_len], 'sentence') input_word = tf.placeholder(tf.int32, [None, word_len], 'word') emb_sentence = tf.nn.embedding_lookup(embedding_matrix, input_sentence, 'emb_sentence') emb_word = tf.nn.embedding_lookup(word_embedding_matrix, input_word, 'emb_word') if embedding_keep_prob is not None and embedding_keep_prob < 1.0: [emb_sentence],_ = core.dropout([emb_sentence], [embedding_keep_prob]) [emb_word], _ = core.dropout([emb_word], [embedding_keep_prob]) enc_sentence, _ = sentence_encoder(emb_sentence, emb_word, params) return enc_sentence, None
def conv2d_array(input, sizes, widths, heights, keep_probs=None, w_wds=0.005, b_wds=0.000, w_initializers=tf.truncated_normal_initializer(stddev=0.05), b_initializers=tf.truncated_normal_initializer(stddev=0.05), name="conv2d_array", vlistin=None, vlistout=None): w_wd_list = expand_list(w_wds, sizes) b_wd_list = expand_list(b_wds, sizes) keep_prob_list = expand_list(keep_probs, sizes) w_initializer_list = expand_list(w_initializers, sizes) b_initializer_list = expand_list(b_initializers, sizes) input_size = input.get_shape().as_list()[-1] print('embedding_dimension = ' + str(input_size)) with tf.variable_scope(name): for i, (width, height, size, w_wd, b_wd, keep_prob, w_initializer, b_initializer) in enumerate( zip(widths, heights, sizes, w_wd_list, b_wd_list, keep_prob_list, w_initializer_list, b_initializer_list)): with tf.variable_scope('conv%s' % str(i)) as scope: w = variable_with_weight_decay( 'w', [width, height, input_size, size], initializer=w_initializer, wd=w_wd, vlist=vlistin) if vlistout is not None: vlistout.append(w) b = variable_with_weight_decay('b', [size], initializer=b_initializer, wd=b_wd, vlist=vlistin) if vlistout is not None: vlistout.append(b) w_out = tf.nn.conv2d(input, filter=w, strides=[1, 1, 1, 1], padding='SAME') b_out = tf.nn.bias_add(w_out, b) out = tf.nn.relu(b_out) if keep_prob is not None and keep_prob < 1.0: [out], _ = core.dropout([out], keep_prob) input_size = size input = out return out
def inference(params): embedding_size = params['embedding_size'] vocab_size = params['vocab_size'] sentence_len = params['num_words_before'] + params['num_words_after'] embedding_wd = utils.get_dict_value(params, 'embedding_wd', 0.0) embedding_device = utils.get_dict_value(params, 'embedding_device', None) embedding_initializer = utils.get_dict_value(params, 'embedding_initializer', None) embedding_keep_prob = utils.get_dict_value(params, 'embedding_keep_prob', 1.0) print("USING EMBEDDING DEVICE %s" % embedding_device) if embedding_device is not None: with tf.device(embedding_device): embedding_matrix = nlp.variable_with_weight_decay( 'embedding_matrix', [vocab_size, embedding_size], initializer=embedding_initializer, wd=embedding_wd) else: embedding_matrix = nlp.variable_with_weight_decay( 'embedding_matrix', [vocab_size, embedding_size], initializer=embedding_initializer, wd=embedding_wd) timing_info = tf.placeholder(tf.float32, [None, sentence_len], 'timing_info') input_sentence = tf.placeholder(tf.int32, [None, sentence_len], 'tcids_before') emb_sentence = tf.nn.embedding_lookup(embedding_matrix, input_sentence, 'emb_sentence') if embedding_keep_prob is not None and embedding_keep_prob < 1.0: [emb_sentence], _ = core.dropout([emb_sentence], [embedding_keep_prob]) timing = tf.reshape(timing_info, np.asarray([-1, sentence_len, 1])) emb_sentence = tf.concat([emb_sentence, timing], axis=2) enc_sentence, _ = sentence_encoder(emb_sentence, params) return enc_sentence, None
def inference(params): # batch_size = params['batch_size'] num_steps = params['num_steps'] cell_size = params['cell_size'] vocab_size = params['vocab_size'] num_layers = params['num_layers'] cell_type = params['cell_type'] is_training = tf.get_default_graph().get_tensor_by_name('is_training:0') embedding_wd = utils.get_dict_value(params, 'embedding_wd') # embedding_device = utils.get_dict_value(params, 'embedding_device') embedding_initializer = utils.get_dict_value(params, 'embedding_initializer') embedding_keep_prob = utils.get_dict_value(params, 'embedding_keep_prob') rnn_dropout_keep_prob = utils.get_dict_value(params, 'rnn_dropout_keep_prob', 1.0) cell_activation = utils.get_dict_value(params, 'cell_activation', None) embedding_matrix = nlp.variable_with_weight_decay( 'embedding_matrix', [vocab_size, cell_size], initializer=embedding_initializer, wd=embedding_wd) words = tf.placeholder(tf.int32, [None, None], name='x') emb_words = tf.nn.embedding_lookup(embedding_matrix, words, 'emb_words') # add dropout if needed if embedding_keep_prob is not None and embedding_keep_prob < 1.0: [emb_words], _ = core.dropout([emb_words], [embedding_keep_prob]) if num_layers > 1: cell_list = [] for _ in range(num_layers): if cell_type == 'GRU': cell = tf.contrib.rnn.GRUCell(cell_size) elif cell_type == 'BlockLSTM': cell = tf.contrib.rnn.LSTMBlockCell(cell_size) else: cell = tf.contrib.rnn.BasicLSTMCell(cell_size, activation=cell_activation) if rnn_dropout_keep_prob < 1.00: [cell], _ = core.rnn_dropout([cell], [rnn_dropout_keep_prob]) cell_list.append(cell) cell = tf.contrib.rnn.MultiRNNCell(cell_list, state_is_tuple=True) else: if cell_type == 'GRU': cell = tf.contrib.rnn.GRUCell(cell_size) elif cell_type == 'BlockLSTM': cell = tf.contrib.rnn.LSTMBlockCell(cell_size) else: cell = tf.contrib.rnn.BasicLSTMCell(cell_size, activation=cell_activation) if rnn_dropout_keep_prob < 1.00: [cell], _ = core.rnn_dropout([cell], [rnn_dropout_keep_prob]) """ # change for GRU. not yet working! + if cell_type == 'GRU': + state_placeholder = tf.placeholder(tf.float32, [num_layers, None, cell_size], name='state') + l = tf.unstack(state_placeholder, axis=0) + state = tf.reshape(state_placeholder, [None, num_layers, cell_size]) + else: + state_placeholder = tf.placeholder(tf.float32, [num_layers, 2, None, cell_size], name='state') + l = tf.unstack(state_placeholder, axis=0) + state = tuple( + [tf.nn.rnn_cell.LSTMStateTuple(l[idx][0], l[idx][1]) + for idx in range(num_layers)] + ) """ state_placeholder = tf.placeholder(tf.float32, [num_layers, 2, None, cell_size], name='state') l = tf.unstack(state_placeholder, axis=0) state = tuple([ tf.nn.rnn_cell.LSTMStateTuple(l[idx][0], l[idx][1]) for idx in range(num_layers) ]) outputs, final_state = tf.nn.dynamic_rnn(cell, emb_words, initial_state=state) final = tf.identity(final_state, name='final_state') output = tf.reshape(tf.concat(outputs, 1), [-1, cell_size]) softmax_w = nlp.variable_with_weight_decay('softmax_w', [cell_size, vocab_size]) softmax_b = nlp.variable_with_weight_decay('softmax_b', [vocab_size]) logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b) logits = tf.reshape(logits, [-1, num_steps, vocab_size], name='output_logits') if utils.get_dict_value(params, 'use_single_sm', False): smei = tf.placeholder(tf.int32, [None, None], name='smei') # softmax evaluation index exp_logits = tf.exp(logits) numerator = tf.gather_nd( exp_logits, tf.stack([ tf.tile(tf.expand_dims(tf.range(tf.shape(smei)[0]), 1), [1, tf.shape(smei)[1]]), tf.transpose( tf.tile(tf.expand_dims(tf.range(tf.shape(smei)[1]), 1), [1, tf.shape(smei)[0]])), smei ], 2)) logits_smei = tf.divide(numerator, tf.reduce_sum(exp_logits, axis=-1), 'output_single_sm') logits_sm = tf.nn.softmax(logits, name='output_logits_sm') return [logits]