def do_projection(): # Project raw outputs down with tf.name_scope("projection"): projection_width = int(total_output_width/(2*len(hidden_outputs))) w_p = tf_utils.initialize_weights([total_output_width, projection_width], "w_p", init_type="xavier") b_p = tf.get_variable("b_p", initializer=tf.constant(0.01, shape=[projection_width])) projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected") projected_nonlinearity = tf_utils.apply_nonlinearity(projected, self.nonlinearity) return projected_nonlinearity, projection_width
def forward(self, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, self.input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, self.input_x2) input_list.append(shape_embeddings) input_size += self.shape_size input_feats = tf.concat(axis=2, values=input_list) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = 2*self.hidden_dim with tf.name_scope("bilstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_dim, state_is_tuple=True, reuse=reuse) bwd_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_dim, state_is_tuple=True, reuse=reuse) lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) hidden_outputs = tf.concat(axis=2, values=lstm_outputs) h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss_A += tf.nn.l2_loss(w_tanh) self.l2_loss_A += tf.nn.l2_loss(b_tanh) self.l2_loss_B += tf.nn.l2_loss(w_tanh) self.l2_loss_B += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) return h2_drop
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): block_unflat_scores = [] with tf.variable_scope("forward", reuse=reuse): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: char_embeddings_masked = tf.multiply(self.char_embeddings, tf.expand_dims(self.input_mask, 2)) input_list.append(char_embeddings_masked) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size-1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size initial_filter_width = self.layers_map[0][1]['width'] initial_num_filters = self.layers_map[0][1]['filters'] filter_shape = [1, initial_filter_width, input_size, initial_num_filters] initial_layer_name = "conv0" if not reuse: print(input_list) print("Adding initial layer %s: width: %d; filters: %d" % ( initial_layer_name, initial_filter_width, initial_num_filters)) input_feats = tf.concat(axis=2, values=input_list) input_feats_expanded = tf.expand_dims(input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, input_dropout_keep_prob) print("input feats expanded drop", input_feats_expanded_drop.get_shape()) # first projection of embeddings w = tf_utils.initialize_weights(filter_shape, initial_layer_name + "_w", init_type='xavier', gain='relu') b = tf.get_variable(initial_layer_name + "_b", initializer=tf.constant(0.01, shape=[initial_num_filters])) conv0 = tf.nn.conv2d(input_feats_expanded_drop, w, strides=[1, 1, 1, 1], padding="SAME", name=initial_layer_name) h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv0, b), 'relu') initial_inputs = [h0] last_dims = initial_num_filters # Stacked atrous convolutions last_output = tf.concat(axis=3, values=initial_inputs) for block in range(self.repeats): print("last out shape", last_output.get_shape()) print("last dims", last_dims) hidden_outputs = [] total_output_width = 0 reuse_block = (block != 0 and self.share_repeats) or reuse block_name_suff = "" if self.share_repeats else str(block) inner_last_dims = last_dims inner_last_output = last_output with tf.variable_scope("block" + block_name_suff, reuse=reuse_block): for layer_name, layer in self.layers_map: dilation = layer['dilation'] filter_width = layer['width'] num_filters = layer['filters'] initialization = layer['initialization'] take_layer = layer['take'] if not reuse: print("Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % ( layer_name, dilation, filter_width, num_filters, take_layer)) with tf.name_scope("atrous-conv-%s" % layer_name): # [filter_height, filter_width, in_channels, out_channels] filter_shape = [1, filter_width, inner_last_dims, num_filters] w = tf_utils.initialize_weights(filter_shape, layer_name + "_w", init_type=initialization, gain=self.nonlinearity, divisor=self.num_classes) b = tf.get_variable(layer_name + "_b", initializer=tf.constant(0.0 if initialization == "identity" or initialization == "varscale" else 0.001, shape=[num_filters])) # h = tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, self.batch_norm, layer_name + "_r", # self.batch_size, max_seq_len, self.res_activation, self.training) \ # if last_output != input_feats_expanded_drop \ # else tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, False, layer_name + "_r", # self.batch_size, max_seq_len, 0, self.training) conv = tf.nn.atrous_conv2d(inner_last_output, w, rate=dilation, padding="SAME", name=layer_name) conv_b = tf.nn.bias_add(conv, b) h = tf_utils.apply_nonlinearity(conv_b, self.nonlinearity) # so, only apply "take" to last block (may want to change this later) if take_layer: hidden_outputs.append(h) total_output_width += num_filters inner_last_dims = num_filters inner_last_output = h h_concat = tf.concat(axis=3, values=hidden_outputs) last_output = tf.nn.dropout(h_concat, middle_dropout_keep_prob) last_dims = total_output_width h_concat_squeeze = tf.squeeze(h_concat, [1]) h_concat_flat = tf.reshape(h_concat_squeeze, [-1, total_output_width]) # Add dropout with tf.name_scope("hidden_dropout"): h_drop = tf.nn.dropout(h_concat_flat, hidden_dropout_keep_prob) def do_projection(): # Project raw outputs down with tf.name_scope("projection"): projection_width = int(total_output_width/(2*len(hidden_outputs))) w_p = tf_utils.initialize_weights([total_output_width, projection_width], "w_p", init_type="xavier") b_p = tf.get_variable("b_p", initializer=tf.constant(0.01, shape=[projection_width])) projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected") projected_nonlinearity = tf_utils.apply_nonlinearity(projected, self.nonlinearity) return projected_nonlinearity, projection_width # only use projection if we wanted to, and only apply middle dropout here if projection input_to_pred, proj_width = do_projection() if self.projection else (h_drop, total_output_width) input_to_pred_drop = tf.nn.dropout(input_to_pred, middle_dropout_keep_prob) if self.projection else input_to_pred # Final (unnormalized) scores and predictions with tf.name_scope("output"+block_name_suff): w_o = tf_utils.initialize_weights([proj_width, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable("b_o", initializer=tf.constant(0.01, shape=[self.num_classes])) self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(input_to_pred_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape(scores, tf.stack([self.batch_size, max_seq_len, self.num_classes])) block_unflat_scores.append(unflat_scores) return block_unflat_scores, h_concat_squeeze
def feature_layers(embeddings, reuse=True): """ 基于空洞卷积的特征提取 :param embeddings: :return: """ block_unflat_scores = [] l2_loss = tf.constant(0.0) # emeddings dim is [batch_size, max_seq_len, emb_dim] with tf.variable_scope("feature_layers", reuse=reuse): input_size = params["char_filters"] + params[ "dim"] # word-embeddings + char-embeddings-conv initial_filter_width = params["layers_map"]["conv0"]['width'] initial_num_filters = params["layers_map"]["conv0"]['filters'] filter_shape = [ 1, initial_filter_width, input_size, initial_num_filters ] # [h,w,in_channels, out_channels] initial_layer_name = "conv0" if not reuse: print("Adding initial layer %s: width: %d; filters: %d" % (initial_layer_name, initial_filter_width, initial_num_filters)) # [batch_size, max_seq_len, input_size] -> [batch_size, 1, max_seq_len, input_size] = [N, H, W, in_channels] input_feats_expanded = tf.expand_dims(embeddings, 1) # input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, params["input_dropout_keep_prob"]) print("input feats expanded ", input_feats_expanded.get_shape()) # first projection of embeddings w = tf_utils.initialize_weights(filter_shape, initial_layer_name + "_w", init_type='xavier', gain='relu') b = tf.get_variable(initial_layer_name + "_b", initializer=tf.constant( 0.01, shape=[initial_num_filters])) # conv0 shape = [batch_size, 1, max_seq_len, dim] conv0 = tf.nn.conv2d(input_feats_expanded, w, strides=[1, 1, 1, 1], padding="SAME", name=initial_layer_name) h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add( conv0, b), 'relu') # shape [batch_size, 1, max_seq_len, dim] initial_inputs = [h0] last_dims = initial_num_filters # Stacked atrous convolutions last_output = tf.concat( axis=3, values=initial_inputs) # [batch_size, 1, max_seq_len, last_dims] layer_name_sorted = sorted(params["layers_map"].keys()) for block in range(params["block_repeats"]): print("last out shape", last_output.get_shape()) print("last dims", last_dims) hidden_outputs = [] total_output_width = 0 reuse_block = (block != 0 and params["share_repeats"]) or reuse block_name_suff = "" if params["share_repeats"] else str(block) inner_last_dims = last_dims inner_last_output = last_output with tf.variable_scope("block" + block_name_suff, reuse=reuse_block): for layer_name in layer_name_sorted: # for layer_name, layer in params["layers_map"].items(): if layer_name == "conv0": continue layer = layers_map[layer_name] dilation = layer['dilation'] filter_width = layer['width'] num_filters = layer['filters'] initialization = layer['initialization'] take_layer = layer['take'] if not reuse: print( "Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % (layer_name, dilation, filter_width, num_filters, take_layer)) with tf.name_scope("atrous-conv-%s" % layer_name): # [filter_height, filter_width, in_channels, out_channels] filter_shape = [ 1, filter_width, inner_last_dims, num_filters ] w = tf_utils.initialize_weights( filter_shape, layer_name + "_w", init_type=initialization, gain=params["nonlinearity"], divisor=params["num_classes"]) b = tf.get_variable( layer_name + "_b", initializer=tf.constant( 0.0 if initialization == "identity" or initialization == "varscale" else 0.001, shape=[num_filters])) # h = tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, self.batch_norm, layer_name + "_r", # self.batch_size, max_seq_len, self.res_activation, self.training) \ # if last_output != input_feats_expanded_drop \ # else tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, False, layer_name + "_r", # self.batch_size, max_seq_len, 0, self.training) # conv shape: [batch, height, width, out_channels] -> [batch_size, 1, max_seq_len, last_dims] conv = tf.nn.atrous_conv2d(inner_last_output, w, rate=dilation, padding="SAME", name=layer_name) conv_b = tf.nn.bias_add(conv, b) # h shape: [batch_size, 1, max_seq_len, last_dims] h = tf_utils.apply_nonlinearity( conv_b, params["nonlinearity"]) # so, only apply "take" to last block (may want to change this later) if take_layer: hidden_outputs.append(h) total_output_width += num_filters inner_last_dims = num_filters inner_last_output = h h_concat = tf.concat(axis=3, values=hidden_outputs) last_output = tf.nn.dropout(h_concat, params["middle_dropout_keep_prob"]) last_dims = total_output_width h_concat_squeeze = tf.squeeze(h_concat, [1]) h_concat_flat = tf.reshape(h_concat_squeeze, [-1, total_output_width]) # Add dropout with tf.name_scope("hidden_dropout"): h_drop = tf.nn.dropout(h_concat_flat, params["hidden_dropout_keep_prob"]) def do_projection(): # Project raw outputs down with tf.name_scope("projection"): projection_width = int(total_output_width / (2 * len(hidden_outputs))) w_p = tf_utils.initialize_weights( [total_output_width, projection_width], "w_p", init_type="xavier") b_p = tf.get_variable("b_p", initializer=tf.constant( 0.01, shape=[projection_width])) projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected") projected_nonlinearity = tf_utils.apply_nonlinearity( projected, params["nonlinearity"]) return projected_nonlinearity, projection_width # only use projection if we wanted to, and only apply middle dropout here if projection input_to_pred, proj_width = do_projection( ) if params["projection"] else (h_drop, total_output_width) input_to_pred_drop = tf.nn.dropout(input_to_pred, params["middle_dropout_keep_prob"]) \ if params["projection"] else input_to_pred # Final (unnormalized) scores and predictions with tf.name_scope("output" + block_name_suff): w_o = tf_utils.initialize_weights( [proj_width, params["num_classes"]], "w_o", init_type="xavier") b_o = tf.get_variable("b_o", initializer=tf.constant( 0.01, shape=[params["num_classes"]])) l2_loss += tf.nn.l2_loss(w_o) l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(input_to_pred_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape( scores, tf.stack( [params["batch_size"], -1, params["num_classes"]])) block_unflat_scores.append(unflat_scores) return block_unflat_scores, h_concat_squeeze, l2_loss
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size # todo add embeddings for all discrete features if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size if self.use_geometric_feats: # it's giving some issue with the typing, so I'm just casting everythint ot be the same input_list.append(tf.cast(self.widths, tf.float32)) input_list.append(tf.cast(self.heights, tf.float32)) input_list.append(tf.cast(self.wh_ratios, tf.float32)) input_list.append(tf.cast(self.x_coords, tf.float32)) input_list.append(tf.cast(self.y_coords, tf.float32)) # input_list.append(tf.cast(self.pages, tf.float32)) input_list.append(tf.cast(self.lines, tf.float32)) input_list.append(tf.cast(self.zones, tf.float32)) input_size += 7 if self.use_lexicons: lex_embeddings_shape = (1, self.lex_size) # params for lexicon embeddings w_place = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_place") w_dept = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_dept") w_uni = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_uni") w_person = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_person") # embedding lookup tables place_embeddings = tf.nn.embedding_lookup( w_place, self.place_scores) dept_embeddings = tf.nn.embedding_lookup( w_dept, self.department_scores) uni_embeddings = tf.nn.embedding_lookup( w_uni, self.university_scores) person_embeddings = tf.nn.embedding_lookup( w_person, self.person_scores) # add lex embeddings to input list input_list.append(place_embeddings) input_list.append(dept_embeddings) input_list.append(uni_embeddings) input_list.append(person_embeddings) input_size += self.shape_size # input_list.append(tf.cast(self.place_scores, tf.float32)) # input_list.append(tf.cast(self.department_scores, tf.float32)) # input_list.append(tf.cast(self.university_scores, tf.float32)) # input_list.append(tf.cast(self.person_scores, tf.float32)) input_size += 4 * self.lex_size # print(input.get_shape()) # (w, h) = self.widths.get_shape() # print(tf.reshape(self.widths, (w, h, 1)).get_shape()) input_feats = tf.concat(2, input_list) print(input_feats.get_shape()) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = 2 * self.hidden_dim with tf.name_scope("bilstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) bwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn( cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, # inputs = input_feats, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) hidden_outputs = tf.concat(2, lstm_outputs) # concatenate the results of the forward and backward cells h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights( [total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant( 0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss += tf.nn.l2_loss(w_tanh) self.l2_loss += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): w_o = tf_utils.initialize_weights( [self.hidden_dim, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable(initializer=tf.constant( 0.01, shape=[self.num_classes]), name="b_o") self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape( scores, tf.pack([self.batch_size, max_seq_len, self.num_classes])) return unflat_scores
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size if self.use_geometric_feats: # TODO: add other features to input list, concat them to end of input_feats # todo this is the wrong shape to be concatenated # it's giving some issue with the typing, so I'm just casting everythint ot be the same input_list.append(tf.cast(self.widths, tf.float32)) input_list.append(tf.cast(self.heights, tf.float32)) input_list.append(tf.cast(self.wh_ratios, tf.float32)) input_list.append(tf.cast(self.x_coords, tf.float32)) input_list.append(tf.cast(self.y_coords, tf.float32)) input_list.append(tf.cast(self.pages, tf.float32)) input_list.append(tf.cast(self.lines, tf.float32)) input_list.append(tf.cast(self.zones, tf.float32)) input_size += 8 # print(input.get_shape()) # (w, h) = self.widths.get_shape() # print(tf.reshape(self.widths, (w, h, 1)).get_shape()) input_feats = tf.concat(2, input_list) print(input_feats.get_shape()) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = self.hidden_dim # 2*self.hidden_dim with tf.name_scope("lstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) lstm_outputs, _ = tf.nn.dynamic_rnn(cell=fwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, # inputs = input_feats, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) # hidden_outputs = tf.concat(2, lstm_outputs) # flatten the outputs of the lstm? is this necessary? h_concat_flat = tf.reshape(lstm_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss += tf.nn.l2_loss(w_tanh) self.l2_loss += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): w_o = tf_utils.initialize_weights([self.hidden_dim, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes]), name="b_o") self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape(scores, tf.pack([self.batch_size, max_seq_len, self.num_classes])) return unflat_scores
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size input_feats = tf.concat(axis=2, values=input_list) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = 2*self.hidden_dim with tf.name_scope("bilstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) bwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) hidden_outputs = tf.concat(axis=2, values=lstm_outputs) h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss += tf.nn.l2_loss(w_tanh) self.l2_loss += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): w_o = tf_utils.initialize_weights([self.hidden_dim, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes]), name="b_o") self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape(scores, tf.stack([self.batch_size, max_seq_len, self.num_classes])) return unflat_scores, hidden_outputs