def forward(self, input_x1, input_dropout_keep_prob, reuse=True): with tf.variable_scope("char-forward", reuse=reuse): char_embeddings_lookup = tf.nn.embedding_lookup(self.char_embeddings, input_x1) print(char_embeddings_lookup.get_shape()) char_embeddings_flat = tf.reshape(char_embeddings_lookup, tf.stack([self.batch_size*self.max_seq_len, self.max_tok_len, self.embedding_size])) print(char_embeddings_flat.get_shape()) tok_lens_flat = tf.reshape(self.token_lengths, [self.batch_size*self.max_seq_len]) print(tok_lens_flat.get_shape()) input_feats_expanded = tf.expand_dims(char_embeddings_flat, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, input_dropout_keep_prob) with tf.name_scope("char-cnn"): filter_shape = [1, self.filter_width, self.embedding_size, self.hidden_dim] w = tf_utils.initialize_weights(filter_shape, "conv0_w", init_type='xavier', gain='relu') b = tf.get_variable("conv0_b", initializer=tf.constant(0.01, shape=[self.hidden_dim])) conv0 = tf.nn.conv2d(input_feats_expanded_drop, w, strides=[1, 1, 1, 1], padding="SAME", name="conv0") print("conv0", conv0.get_shape()) h_squeeze = tf.squeeze(conv0, [1]) print("squeeze", h_squeeze.get_shape()) hidden_outputs = tf.reduce_max(h_squeeze, 1) print("max", hidden_outputs.get_shape()) hidden_outputs_unflat = tf.reshape(hidden_outputs, tf.stack([self.batch_size, self.max_seq_len, self.hidden_dim])) return hidden_outputs_unflat
def do_projection(): # Project raw outputs down with tf.name_scope("projection"): projection_width = int(total_output_width/(2*len(hidden_outputs))) w_p = tf_utils.initialize_weights([total_output_width, projection_width], "w_p", init_type="xavier") b_p = tf.get_variable("b_p", initializer=tf.constant(0.01, shape=[projection_width])) projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected") projected_nonlinearity = tf_utils.apply_nonlinearity(projected, self.nonlinearity) return projected_nonlinearity, projection_width
def forward(self, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, self.input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, self.input_x2) input_list.append(shape_embeddings) input_size += self.shape_size input_feats = tf.concat(axis=2, values=input_list) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = 2*self.hidden_dim with tf.name_scope("bilstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_dim, state_is_tuple=True, reuse=reuse) bwd_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_dim, state_is_tuple=True, reuse=reuse) lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) hidden_outputs = tf.concat(axis=2, values=lstm_outputs) h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss_A += tf.nn.l2_loss(w_tanh) self.l2_loss_A += tf.nn.l2_loss(b_tanh) self.l2_loss_B += tf.nn.l2_loss(w_tanh) self.l2_loss_B += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) return h2_drop
def calc_unflat_scores_B(self, lstm_output, reuse=True): with tf.variable_scope("forward_B", reuse=reuse): # Final (unnormalized) scores and predictions with tf.name_scope("output_B"): w_o_B = tf_utils.initialize_weights([self.hidden_dim, self.num_classes_B], "w_o_B", init_type="xavier") b_o_B = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes_B]), name="b_o_B") self.l2_loss_B += tf.nn.l2_loss(w_o_B) self.l2_loss_B += tf.nn.l2_loss(b_o_B) scores_B = tf.nn.xw_plus_b(lstm_output, w_o_B, b_o_B, name="scores") unflat_scores_B = tf.reshape(scores_B, tf.stack([self.batch_size, self.max_seq_len, self.num_classes_B])) return unflat_scores_B
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): block_unflat_scores = [] with tf.variable_scope("forward", reuse=reuse): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: char_embeddings_masked = tf.multiply(self.char_embeddings, tf.expand_dims(self.input_mask, 2)) input_list.append(char_embeddings_masked) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size-1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size initial_filter_width = self.layers_map[0][1]['width'] initial_num_filters = self.layers_map[0][1]['filters'] filter_shape = [1, initial_filter_width, input_size, initial_num_filters] initial_layer_name = "conv0" if not reuse: print(input_list) print("Adding initial layer %s: width: %d; filters: %d" % ( initial_layer_name, initial_filter_width, initial_num_filters)) input_feats = tf.concat(axis=2, values=input_list) input_feats_expanded = tf.expand_dims(input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, input_dropout_keep_prob) print("input feats expanded drop", input_feats_expanded_drop.get_shape()) # first projection of embeddings w = tf_utils.initialize_weights(filter_shape, initial_layer_name + "_w", init_type='xavier', gain='relu') b = tf.get_variable(initial_layer_name + "_b", initializer=tf.constant(0.01, shape=[initial_num_filters])) conv0 = tf.nn.conv2d(input_feats_expanded_drop, w, strides=[1, 1, 1, 1], padding="SAME", name=initial_layer_name) h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv0, b), 'relu') initial_inputs = [h0] last_dims = initial_num_filters # Stacked atrous convolutions last_output = tf.concat(axis=3, values=initial_inputs) for block in range(self.repeats): print("last out shape", last_output.get_shape()) print("last dims", last_dims) hidden_outputs = [] total_output_width = 0 reuse_block = (block != 0 and self.share_repeats) or reuse block_name_suff = "" if self.share_repeats else str(block) inner_last_dims = last_dims inner_last_output = last_output with tf.variable_scope("block" + block_name_suff, reuse=reuse_block): for layer_name, layer in self.layers_map: dilation = layer['dilation'] filter_width = layer['width'] num_filters = layer['filters'] initialization = layer['initialization'] take_layer = layer['take'] if not reuse: print("Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % ( layer_name, dilation, filter_width, num_filters, take_layer)) with tf.name_scope("atrous-conv-%s" % layer_name): # [filter_height, filter_width, in_channels, out_channels] filter_shape = [1, filter_width, inner_last_dims, num_filters] w = tf_utils.initialize_weights(filter_shape, layer_name + "_w", init_type=initialization, gain=self.nonlinearity, divisor=self.num_classes) b = tf.get_variable(layer_name + "_b", initializer=tf.constant(0.0 if initialization == "identity" or initialization == "varscale" else 0.001, shape=[num_filters])) # h = tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, self.batch_norm, layer_name + "_r", # self.batch_size, max_seq_len, self.res_activation, self.training) \ # if last_output != input_feats_expanded_drop \ # else tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, False, layer_name + "_r", # self.batch_size, max_seq_len, 0, self.training) conv = tf.nn.atrous_conv2d(inner_last_output, w, rate=dilation, padding="SAME", name=layer_name) conv_b = tf.nn.bias_add(conv, b) h = tf_utils.apply_nonlinearity(conv_b, self.nonlinearity) # so, only apply "take" to last block (may want to change this later) if take_layer: hidden_outputs.append(h) total_output_width += num_filters inner_last_dims = num_filters inner_last_output = h h_concat = tf.concat(axis=3, values=hidden_outputs) last_output = tf.nn.dropout(h_concat, middle_dropout_keep_prob) last_dims = total_output_width h_concat_squeeze = tf.squeeze(h_concat, [1]) h_concat_flat = tf.reshape(h_concat_squeeze, [-1, total_output_width]) # Add dropout with tf.name_scope("hidden_dropout"): h_drop = tf.nn.dropout(h_concat_flat, hidden_dropout_keep_prob) def do_projection(): # Project raw outputs down with tf.name_scope("projection"): projection_width = int(total_output_width/(2*len(hidden_outputs))) w_p = tf_utils.initialize_weights([total_output_width, projection_width], "w_p", init_type="xavier") b_p = tf.get_variable("b_p", initializer=tf.constant(0.01, shape=[projection_width])) projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected") projected_nonlinearity = tf_utils.apply_nonlinearity(projected, self.nonlinearity) return projected_nonlinearity, projection_width # only use projection if we wanted to, and only apply middle dropout here if projection input_to_pred, proj_width = do_projection() if self.projection else (h_drop, total_output_width) input_to_pred_drop = tf.nn.dropout(input_to_pred, middle_dropout_keep_prob) if self.projection else input_to_pred # Final (unnormalized) scores and predictions with tf.name_scope("output"+block_name_suff): w_o = tf_utils.initialize_weights([proj_width, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable("b_o", initializer=tf.constant(0.01, shape=[self.num_classes])) self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(input_to_pred_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape(scores, tf.stack([self.batch_size, max_seq_len, self.num_classes])) block_unflat_scores.append(unflat_scores) return block_unflat_scores, h_concat_squeeze
def feature_layers(embeddings, reuse=True): """ 基于空洞卷积的特征提取 :param embeddings: :return: """ block_unflat_scores = [] l2_loss = tf.constant(0.0) # emeddings dim is [batch_size, max_seq_len, emb_dim] with tf.variable_scope("feature_layers", reuse=reuse): input_size = params["char_filters"] + params[ "dim"] # word-embeddings + char-embeddings-conv initial_filter_width = params["layers_map"]["conv0"]['width'] initial_num_filters = params["layers_map"]["conv0"]['filters'] filter_shape = [ 1, initial_filter_width, input_size, initial_num_filters ] # [h,w,in_channels, out_channels] initial_layer_name = "conv0" if not reuse: print("Adding initial layer %s: width: %d; filters: %d" % (initial_layer_name, initial_filter_width, initial_num_filters)) # [batch_size, max_seq_len, input_size] -> [batch_size, 1, max_seq_len, input_size] = [N, H, W, in_channels] input_feats_expanded = tf.expand_dims(embeddings, 1) # input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, params["input_dropout_keep_prob"]) print("input feats expanded ", input_feats_expanded.get_shape()) # first projection of embeddings w = tf_utils.initialize_weights(filter_shape, initial_layer_name + "_w", init_type='xavier', gain='relu') b = tf.get_variable(initial_layer_name + "_b", initializer=tf.constant( 0.01, shape=[initial_num_filters])) # conv0 shape = [batch_size, 1, max_seq_len, dim] conv0 = tf.nn.conv2d(input_feats_expanded, w, strides=[1, 1, 1, 1], padding="SAME", name=initial_layer_name) h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add( conv0, b), 'relu') # shape [batch_size, 1, max_seq_len, dim] initial_inputs = [h0] last_dims = initial_num_filters # Stacked atrous convolutions last_output = tf.concat( axis=3, values=initial_inputs) # [batch_size, 1, max_seq_len, last_dims] layer_name_sorted = sorted(params["layers_map"].keys()) for block in range(params["block_repeats"]): print("last out shape", last_output.get_shape()) print("last dims", last_dims) hidden_outputs = [] total_output_width = 0 reuse_block = (block != 0 and params["share_repeats"]) or reuse block_name_suff = "" if params["share_repeats"] else str(block) inner_last_dims = last_dims inner_last_output = last_output with tf.variable_scope("block" + block_name_suff, reuse=reuse_block): for layer_name in layer_name_sorted: # for layer_name, layer in params["layers_map"].items(): if layer_name == "conv0": continue layer = layers_map[layer_name] dilation = layer['dilation'] filter_width = layer['width'] num_filters = layer['filters'] initialization = layer['initialization'] take_layer = layer['take'] if not reuse: print( "Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % (layer_name, dilation, filter_width, num_filters, take_layer)) with tf.name_scope("atrous-conv-%s" % layer_name): # [filter_height, filter_width, in_channels, out_channels] filter_shape = [ 1, filter_width, inner_last_dims, num_filters ] w = tf_utils.initialize_weights( filter_shape, layer_name + "_w", init_type=initialization, gain=params["nonlinearity"], divisor=params["num_classes"]) b = tf.get_variable( layer_name + "_b", initializer=tf.constant( 0.0 if initialization == "identity" or initialization == "varscale" else 0.001, shape=[num_filters])) # h = tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, self.batch_norm, layer_name + "_r", # self.batch_size, max_seq_len, self.res_activation, self.training) \ # if last_output != input_feats_expanded_drop \ # else tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, False, layer_name + "_r", # self.batch_size, max_seq_len, 0, self.training) # conv shape: [batch, height, width, out_channels] -> [batch_size, 1, max_seq_len, last_dims] conv = tf.nn.atrous_conv2d(inner_last_output, w, rate=dilation, padding="SAME", name=layer_name) conv_b = tf.nn.bias_add(conv, b) # h shape: [batch_size, 1, max_seq_len, last_dims] h = tf_utils.apply_nonlinearity( conv_b, params["nonlinearity"]) # so, only apply "take" to last block (may want to change this later) if take_layer: hidden_outputs.append(h) total_output_width += num_filters inner_last_dims = num_filters inner_last_output = h h_concat = tf.concat(axis=3, values=hidden_outputs) last_output = tf.nn.dropout(h_concat, params["middle_dropout_keep_prob"]) last_dims = total_output_width h_concat_squeeze = tf.squeeze(h_concat, [1]) h_concat_flat = tf.reshape(h_concat_squeeze, [-1, total_output_width]) # Add dropout with tf.name_scope("hidden_dropout"): h_drop = tf.nn.dropout(h_concat_flat, params["hidden_dropout_keep_prob"]) def do_projection(): # Project raw outputs down with tf.name_scope("projection"): projection_width = int(total_output_width / (2 * len(hidden_outputs))) w_p = tf_utils.initialize_weights( [total_output_width, projection_width], "w_p", init_type="xavier") b_p = tf.get_variable("b_p", initializer=tf.constant( 0.01, shape=[projection_width])) projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected") projected_nonlinearity = tf_utils.apply_nonlinearity( projected, params["nonlinearity"]) return projected_nonlinearity, projection_width # only use projection if we wanted to, and only apply middle dropout here if projection input_to_pred, proj_width = do_projection( ) if params["projection"] else (h_drop, total_output_width) input_to_pred_drop = tf.nn.dropout(input_to_pred, params["middle_dropout_keep_prob"]) \ if params["projection"] else input_to_pred # Final (unnormalized) scores and predictions with tf.name_scope("output" + block_name_suff): w_o = tf_utils.initialize_weights( [proj_width, params["num_classes"]], "w_o", init_type="xavier") b_o = tf.get_variable("b_o", initializer=tf.constant( 0.01, shape=[params["num_classes"]])) l2_loss += tf.nn.l2_loss(w_o) l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(input_to_pred_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape( scores, tf.stack( [params["batch_size"], -1, params["num_classes"]])) block_unflat_scores.append(unflat_scores) return block_unflat_scores, h_concat_squeeze, l2_loss
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size # todo add embeddings for all discrete features if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size if self.use_geometric_feats: # it's giving some issue with the typing, so I'm just casting everythint ot be the same input_list.append(tf.cast(self.widths, tf.float32)) input_list.append(tf.cast(self.heights, tf.float32)) input_list.append(tf.cast(self.wh_ratios, tf.float32)) input_list.append(tf.cast(self.x_coords, tf.float32)) input_list.append(tf.cast(self.y_coords, tf.float32)) # input_list.append(tf.cast(self.pages, tf.float32)) input_list.append(tf.cast(self.lines, tf.float32)) input_list.append(tf.cast(self.zones, tf.float32)) input_size += 7 if self.use_lexicons: lex_embeddings_shape = (1, self.lex_size) # params for lexicon embeddings w_place = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_place") w_dept = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_dept") w_uni = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_uni") w_person = tf_utils.initialize_embeddings(lex_embeddings_shape, name="w_person") # embedding lookup tables place_embeddings = tf.nn.embedding_lookup( w_place, self.place_scores) dept_embeddings = tf.nn.embedding_lookup( w_dept, self.department_scores) uni_embeddings = tf.nn.embedding_lookup( w_uni, self.university_scores) person_embeddings = tf.nn.embedding_lookup( w_person, self.person_scores) # add lex embeddings to input list input_list.append(place_embeddings) input_list.append(dept_embeddings) input_list.append(uni_embeddings) input_list.append(person_embeddings) input_size += self.shape_size # input_list.append(tf.cast(self.place_scores, tf.float32)) # input_list.append(tf.cast(self.department_scores, tf.float32)) # input_list.append(tf.cast(self.university_scores, tf.float32)) # input_list.append(tf.cast(self.person_scores, tf.float32)) input_size += 4 * self.lex_size # print(input.get_shape()) # (w, h) = self.widths.get_shape() # print(tf.reshape(self.widths, (w, h, 1)).get_shape()) input_feats = tf.concat(2, input_list) print(input_feats.get_shape()) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = 2 * self.hidden_dim with tf.name_scope("bilstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) bwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn( cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, # inputs = input_feats, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) hidden_outputs = tf.concat(2, lstm_outputs) # concatenate the results of the forward and backward cells h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights( [total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant( 0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss += tf.nn.l2_loss(w_tanh) self.l2_loss += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): w_o = tf_utils.initialize_weights( [self.hidden_dim, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable(initializer=tf.constant( 0.01, shape=[self.num_classes]), name="b_o") self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape( scores, tf.pack([self.batch_size, max_seq_len, self.num_classes])) return unflat_scores
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size if self.use_geometric_feats: # TODO: add other features to input list, concat them to end of input_feats # todo this is the wrong shape to be concatenated # it's giving some issue with the typing, so I'm just casting everythint ot be the same input_list.append(tf.cast(self.widths, tf.float32)) input_list.append(tf.cast(self.heights, tf.float32)) input_list.append(tf.cast(self.wh_ratios, tf.float32)) input_list.append(tf.cast(self.x_coords, tf.float32)) input_list.append(tf.cast(self.y_coords, tf.float32)) input_list.append(tf.cast(self.pages, tf.float32)) input_list.append(tf.cast(self.lines, tf.float32)) input_list.append(tf.cast(self.zones, tf.float32)) input_size += 8 # print(input.get_shape()) # (w, h) = self.widths.get_shape() # print(tf.reshape(self.widths, (w, h, 1)).get_shape()) input_feats = tf.concat(2, input_list) print(input_feats.get_shape()) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = self.hidden_dim # 2*self.hidden_dim with tf.name_scope("lstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) lstm_outputs, _ = tf.nn.dynamic_rnn(cell=fwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, # inputs = input_feats, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) # hidden_outputs = tf.concat(2, lstm_outputs) # flatten the outputs of the lstm? is this necessary? h_concat_flat = tf.reshape(lstm_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss += tf.nn.l2_loss(w_tanh) self.l2_loss += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): w_o = tf_utils.initialize_weights([self.hidden_dim, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes]), name="b_o") self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape(scores, tf.pack([self.batch_size, max_seq_len, self.num_classes])) return unflat_scores
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True): word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1) with tf.variable_scope("forward", reuse=reuse): input_list = [word_embeddings] input_size = self.embedding_size if self.use_characters: input_list.append(self.char_embeddings) input_size += self.char_size if self.use_shape: shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size) w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s") shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2) input_list.append(shape_embeddings) input_size += self.shape_size input_feats = tf.concat(axis=2, values=input_list) # self.input_feats_expanded = tf.expand_dims(self.input_feats, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob) total_output_width = 2*self.hidden_dim with tf.name_scope("bilstm"): # selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch) fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) bwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True) lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32, inputs=input_feats_expanded_drop, parallel_iterations=50, sequence_length=self.flat_sequence_lengths) hidden_outputs = tf.concat(axis=2, values=lstm_outputs) h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width]) # Add dropout with tf.name_scope("middle_dropout"): h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob) # second projection with tf.name_scope("tanh_proj"): w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier") b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh") self.l2_loss += tf.nn.l2_loss(w_tanh) self.l2_loss += tf.nn.l2_loss(b_tanh) h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh") h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity) # Add dropout with tf.name_scope("hidden_dropout"): h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): w_o = tf_utils.initialize_weights([self.hidden_dim, self.num_classes], "w_o", init_type="xavier") b_o = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes]), name="b_o") self.l2_loss += tf.nn.l2_loss(w_o) self.l2_loss += tf.nn.l2_loss(b_o) scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores") unflat_scores = tf.reshape(scores, tf.stack([self.batch_size, max_seq_len, self.num_classes])) return unflat_scores, hidden_outputs
def forward(self, input_x1, input_dropout_keep_prob, reuse=True): with tf.variable_scope("char-forward", reuse=reuse): char_embeddings_lookup = tf.nn.embedding_lookup( self.char_embeddings, input_x1) print(char_embeddings_lookup.get_shape()) char_embeddings_flat = tf.reshape( char_embeddings_lookup, tf.pack([ self.batch_size * self.max_seq_len, self.max_tok_len, self.embedding_size ])) print(char_embeddings_flat.get_shape()) tok_lens_flat = tf.reshape(self.token_lengths, [self.batch_size * self.max_seq_len]) print(tok_lens_flat.get_shape()) # # input_list = [char_embeddings_lookup] # # input_size = self.embedding_size # # input_feats = tf.concat(2, input_list) input_feats_expanded = tf.expand_dims(char_embeddings_flat, 1) input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, input_dropout_keep_prob) with tf.name_scope("char-cnn"): filter_shape = [ 1, self.filter_width, self.embedding_size, self.hidden_dim ] w = tf_utils.initialize_weights(filter_shape, "conv0_w", init_type='xavier', gain='relu') b = tf.get_variable("conv0_b", initializer=tf.constant( 0.01, shape=[self.hidden_dim])) conv0 = tf.nn.conv2d(input_feats_expanded_drop, w, strides=[1, self.filter_width, 1, 1], padding="SAME", name="conv0") print("conv0", conv0.get_shape()) h_squeeze = tf.squeeze(conv0, [1]) print("squeeze", h_squeeze.get_shape()) hidden_outputs = tf.reduce_max(h_squeeze, 1) print("max", hidden_outputs.get_shape()) # h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv0, b), 'relu') # last_dims = self.embedding_size # last_output = input_feats_expanded_drop # hidden_outputs = [] # total_output_width = 0 # for layer_name, layer in self.layers_map: # dilation = layer['dilation'] # filter_width = layer['width'] # num_filters = layer['filters'] # initialization = layer['initialization'] # take_layer = layer['take'] # print("Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % ( # layer_name, dilation, filter_width, num_filters, take_layer)) # with tf.name_scope("atrous-conv-%s" % layer_name): # # [filter_height, filter_width, in_channels, out_channels] # filter_shape = [1, filter_width, last_dims, num_filters] # w = tf_utils.initialize_weights(filter_shape, layer_name + "_w", init_type=initialization, gain='relu') # b = tf.get_variable(layer_name + "_b", initializer=tf.constant(0.0 if initialization == 'identity' else 0.01, shape=[num_filters])) # if dilation > 1: # conv = tf.nn.atrous_conv2d( # last_output, # w, # rate=dilation, # padding="SAME", # name=layer_name) # else: # conv = tf.nn.conv2d(last_output, w, strides=[1, filter_width, 1, 1], padding="SAME", name=layer_name) # h = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv, b), 'relu') # if take_layer: # hidden_outputs.append(h) # total_output_width += num_filters # last_dims = num_filters # last_output = h # # h_concat = tf.concat(3, hidden_outputs) # h_concat = tf.squeeze(h_concat, [1]) # # fw_output = tf_utils.last_relevant(h_concat, tok_lens_flat) # bw_output = h_concat[:, 0, :] # hidden_outputs = tf.concat(1, [fw_output, bw_output]) # # hidden_outputs = tf.concat(2, lstm_outputs) # # hidden_outputs = tf.Print(hidden_outputs, [tf.shape(hidden_outputs)], message='hidden outputs:') hidden_outputs_unflat = tf.reshape( hidden_outputs, tf.pack( [self.batch_size, self.max_seq_len, self.hidden_dim])) return hidden_outputs_unflat