def forward(self, input_x1, input_dropout_keep_prob, reuse=True):
with tf.variable_scope("char-forward", reuse=reuse):
char_embeddings_lookup = tf.nn.embedding_lookup(self.char_embeddings, input_x1)
print(char_embeddings_lookup.get_shape())
char_embeddings_flat = tf.reshape(char_embeddings_lookup, tf.stack([self.batch_size*self.max_seq_len, self.max_tok_len, self.embedding_size]))
print(char_embeddings_flat.get_shape())
tok_lens_flat = tf.reshape(self.token_lengths, [self.batch_size*self.max_seq_len])
print(tok_lens_flat.get_shape())
input_feats_expanded = tf.expand_dims(char_embeddings_flat, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, input_dropout_keep_prob)
with tf.name_scope("char-cnn"):
filter_shape = [1, self.filter_width, self.embedding_size, self.hidden_dim]
w = tf_utils.initialize_weights(filter_shape, "conv0_w", init_type='xavier', gain='relu')
b = tf.get_variable("conv0_b", initializer=tf.constant(0.01, shape=[self.hidden_dim]))
conv0 = tf.nn.conv2d(input_feats_expanded_drop, w, strides=[1, 1, 1, 1], padding="SAME", name="conv0")
print("conv0", conv0.get_shape())
h_squeeze = tf.squeeze(conv0, [1])
print("squeeze", h_squeeze.get_shape())
hidden_outputs = tf.reduce_max(h_squeeze, 1)
print("max", hidden_outputs.get_shape())
hidden_outputs_unflat = tf.reshape(hidden_outputs, tf.stack([self.batch_size, self.max_seq_len, self.hidden_dim]))
return hidden_outputs_unflat
def do_projection():
# Project raw outputs down
with tf.name_scope("projection"):
projection_width = int(total_output_width/(2*len(hidden_outputs)))
w_p = tf_utils.initialize_weights([total_output_width, projection_width], "w_p", init_type="xavier")
b_p = tf.get_variable("b_p", initializer=tf.constant(0.01, shape=[projection_width]))
projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected")
projected_nonlinearity = tf_utils.apply_nonlinearity(projected, self.nonlinearity)
return projected_nonlinearity, projection_width
def forward(self, hidden_dropout_keep_prob, input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True):
word_embeddings = tf.nn.embedding_lookup(self.w_e, self.input_x1)
with tf.variable_scope("forward", reuse=reuse):
input_list = [word_embeddings]
input_size = self.embedding_size
if self.use_characters:
input_list.append(self.char_embeddings)
input_size += self.char_size
if self.use_shape:
shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size)
w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s")
shape_embeddings = tf.nn.embedding_lookup(w_s, self.input_x2)
input_list.append(shape_embeddings)
input_size += self.shape_size
input_feats = tf.concat(axis=2, values=input_list)
# self.input_feats_expanded = tf.expand_dims(self.input_feats, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob)
total_output_width = 2*self.hidden_dim
with tf.name_scope("bilstm"):
# selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch)
fwd_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_dim, state_is_tuple=True, reuse=reuse)
bwd_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_dim, state_is_tuple=True, reuse=reuse)
lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32,
inputs=input_feats_expanded_drop,
parallel_iterations=50,
sequence_length=self.flat_sequence_lengths)
hidden_outputs = tf.concat(axis=2, values=lstm_outputs)
h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width])
# Add dropout
with tf.name_scope("middle_dropout"):
h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob)
# second projection
with tf.name_scope("tanh_proj"):
w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier")
b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh")
self.l2_loss_A += tf.nn.l2_loss(w_tanh)
self.l2_loss_A += tf.nn.l2_loss(b_tanh)
self.l2_loss_B += tf.nn.l2_loss(w_tanh)
self.l2_loss_B += tf.nn.l2_loss(b_tanh)
h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh")
h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity)
# Add dropout
with tf.name_scope("hidden_dropout"):
h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob)
return h2_drop
def calc_unflat_scores_B(self, lstm_output, reuse=True):
with tf.variable_scope("forward_B", reuse=reuse):
# Final (unnormalized) scores and predictions
with tf.name_scope("output_B"):
w_o_B = tf_utils.initialize_weights([self.hidden_dim, self.num_classes_B], "w_o_B", init_type="xavier")
b_o_B = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes_B]), name="b_o_B")
self.l2_loss_B += tf.nn.l2_loss(w_o_B)
self.l2_loss_B += tf.nn.l2_loss(b_o_B)
scores_B = tf.nn.xw_plus_b(lstm_output, w_o_B, b_o_B, name="scores")
unflat_scores_B = tf.reshape(scores_B, tf.stack([self.batch_size, self.max_seq_len, self.num_classes_B]))
return unflat_scores_B
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob, input_dropout_keep_prob,
middle_dropout_keep_prob, reuse=True):
block_unflat_scores = []
with tf.variable_scope("forward", reuse=reuse):
word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1)
input_list = [word_embeddings]
input_size = self.embedding_size
if self.use_characters:
char_embeddings_masked = tf.multiply(self.char_embeddings, tf.expand_dims(self.input_mask, 2))
input_list.append(char_embeddings_masked)
input_size += self.char_size
if self.use_shape:
shape_embeddings_shape = (self.shape_domain_size-1, self.shape_size)
w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s")
shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2)
input_list.append(shape_embeddings)
input_size += self.shape_size
initial_filter_width = self.layers_map[0][1]['width']
initial_num_filters = self.layers_map[0][1]['filters']
filter_shape = [1, initial_filter_width, input_size, initial_num_filters]
initial_layer_name = "conv0"
if not reuse:
print(input_list)
print("Adding initial layer %s: width: %d; filters: %d" % (
initial_layer_name, initial_filter_width, initial_num_filters))
input_feats = tf.concat(axis=2, values=input_list)
input_feats_expanded = tf.expand_dims(input_feats, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, input_dropout_keep_prob)
print("input feats expanded drop", input_feats_expanded_drop.get_shape())
# first projection of embeddings
w = tf_utils.initialize_weights(filter_shape, initial_layer_name + "_w", init_type='xavier', gain='relu')
b = tf.get_variable(initial_layer_name + "_b", initializer=tf.constant(0.01, shape=[initial_num_filters]))
conv0 = tf.nn.conv2d(input_feats_expanded_drop, w, strides=[1, 1, 1, 1], padding="SAME", name=initial_layer_name)
h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv0, b), 'relu')
initial_inputs = [h0]
last_dims = initial_num_filters
# Stacked atrous convolutions
last_output = tf.concat(axis=3, values=initial_inputs)
for block in range(self.repeats):
print("last out shape", last_output.get_shape())
print("last dims", last_dims)
hidden_outputs = []
total_output_width = 0
reuse_block = (block != 0 and self.share_repeats) or reuse
block_name_suff = "" if self.share_repeats else str(block)
inner_last_dims = last_dims
inner_last_output = last_output
with tf.variable_scope("block" + block_name_suff, reuse=reuse_block):
for layer_name, layer in self.layers_map:
dilation = layer['dilation']
filter_width = layer['width']
num_filters = layer['filters']
initialization = layer['initialization']
take_layer = layer['take']
if not reuse:
print("Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % (
layer_name, dilation, filter_width, num_filters, take_layer))
with tf.name_scope("atrous-conv-%s" % layer_name):
# [filter_height, filter_width, in_channels, out_channels]
filter_shape = [1, filter_width, inner_last_dims, num_filters]
w = tf_utils.initialize_weights(filter_shape, layer_name + "_w", init_type=initialization, gain=self.nonlinearity, divisor=self.num_classes)
b = tf.get_variable(layer_name + "_b", initializer=tf.constant(0.0 if initialization == "identity" or initialization == "varscale" else 0.001, shape=[num_filters]))
# h = tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, self.batch_norm, layer_name + "_r",
# self.batch_size, max_seq_len, self.res_activation, self.training) \
# if last_output != input_feats_expanded_drop \
# else tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, False, layer_name + "_r",
# self.batch_size, max_seq_len, 0, self.training)
conv = tf.nn.atrous_conv2d(inner_last_output, w, rate=dilation, padding="SAME", name=layer_name)
conv_b = tf.nn.bias_add(conv, b)
h = tf_utils.apply_nonlinearity(conv_b, self.nonlinearity)
# so, only apply "take" to last block (may want to change this later)
if take_layer:
hidden_outputs.append(h)
total_output_width += num_filters
inner_last_dims = num_filters
inner_last_output = h
h_concat = tf.concat(axis=3, values=hidden_outputs)
last_output = tf.nn.dropout(h_concat, middle_dropout_keep_prob)
last_dims = total_output_width
h_concat_squeeze = tf.squeeze(h_concat, [1])
h_concat_flat = tf.reshape(h_concat_squeeze, [-1, total_output_width])
# Add dropout
with tf.name_scope("hidden_dropout"):
h_drop = tf.nn.dropout(h_concat_flat, hidden_dropout_keep_prob)
def do_projection():
# Project raw outputs down
with tf.name_scope("projection"):
projection_width = int(total_output_width/(2*len(hidden_outputs)))
w_p = tf_utils.initialize_weights([total_output_width, projection_width], "w_p", init_type="xavier")
b_p = tf.get_variable("b_p", initializer=tf.constant(0.01, shape=[projection_width]))
projected = tf.nn.xw_plus_b(h_drop, w_p, b_p, name="projected")
projected_nonlinearity = tf_utils.apply_nonlinearity(projected, self.nonlinearity)
return projected_nonlinearity, projection_width
# only use projection if we wanted to, and only apply middle dropout here if projection
input_to_pred, proj_width = do_projection() if self.projection else (h_drop, total_output_width)
input_to_pred_drop = tf.nn.dropout(input_to_pred, middle_dropout_keep_prob) if self.projection else input_to_pred
# Final (unnormalized) scores and predictions
with tf.name_scope("output"+block_name_suff):
w_o = tf_utils.initialize_weights([proj_width, self.num_classes], "w_o", init_type="xavier")
b_o = tf.get_variable("b_o", initializer=tf.constant(0.01, shape=[self.num_classes]))
self.l2_loss += tf.nn.l2_loss(w_o)
self.l2_loss += tf.nn.l2_loss(b_o)
scores = tf.nn.xw_plus_b(input_to_pred_drop, w_o, b_o, name="scores")
unflat_scores = tf.reshape(scores, tf.stack([self.batch_size, max_seq_len, self.num_classes]))
block_unflat_scores.append(unflat_scores)
return block_unflat_scores, h_concat_squeeze
def feature_layers(embeddings, reuse=True):
"""
基于空洞卷积的特征提取
:param embeddings:
:return:
"""
block_unflat_scores = []
l2_loss = tf.constant(0.0)
# emeddings dim is [batch_size, max_seq_len, emb_dim]
with tf.variable_scope("feature_layers", reuse=reuse):
input_size = params["char_filters"] + params[
"dim"] # word-embeddings + char-embeddings-conv
initial_filter_width = params["layers_map"]["conv0"]['width']
initial_num_filters = params["layers_map"]["conv0"]['filters']
filter_shape = [
1, initial_filter_width, input_size, initial_num_filters
] # [h,w,in_channels, out_channels]
initial_layer_name = "conv0"
if not reuse:
print("Adding initial layer %s: width: %d; filters: %d" %
(initial_layer_name, initial_filter_width,
initial_num_filters))
# [batch_size, max_seq_len, input_size] -> [batch_size, 1, max_seq_len, input_size] = [N, H, W, in_channels]
input_feats_expanded = tf.expand_dims(embeddings, 1)
# input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded, params["input_dropout_keep_prob"])
print("input feats expanded ", input_feats_expanded.get_shape())
# first projection of embeddings
w = tf_utils.initialize_weights(filter_shape,
initial_layer_name + "_w",
init_type='xavier',
gain='relu')
b = tf.get_variable(initial_layer_name + "_b",
initializer=tf.constant(
0.01, shape=[initial_num_filters]))
# conv0 shape = [batch_size, 1, max_seq_len, dim]
conv0 = tf.nn.conv2d(input_feats_expanded,
w,
strides=[1, 1, 1, 1],
padding="SAME",
name=initial_layer_name)
h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add(
conv0, b), 'relu') # shape [batch_size, 1, max_seq_len, dim]
initial_inputs = [h0]
last_dims = initial_num_filters
# Stacked atrous convolutions
last_output = tf.concat(
axis=3,
values=initial_inputs) # [batch_size, 1, max_seq_len, last_dims]
layer_name_sorted = sorted(params["layers_map"].keys())
for block in range(params["block_repeats"]):
print("last out shape", last_output.get_shape())
print("last dims", last_dims)
hidden_outputs = []
total_output_width = 0
reuse_block = (block != 0 and params["share_repeats"]) or reuse
block_name_suff = "" if params["share_repeats"] else str(block)
inner_last_dims = last_dims
inner_last_output = last_output
with tf.variable_scope("block" + block_name_suff,
reuse=reuse_block):
for layer_name in layer_name_sorted:
# for layer_name, layer in params["layers_map"].items():
if layer_name == "conv0": continue
layer = layers_map[layer_name]
dilation = layer['dilation']
filter_width = layer['width']
num_filters = layer['filters']
initialization = layer['initialization']
take_layer = layer['take']
if not reuse:
print(
"Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r"
% (layer_name, dilation, filter_width, num_filters,
take_layer))
with tf.name_scope("atrous-conv-%s" % layer_name):
# [filter_height, filter_width, in_channels, out_channels]
filter_shape = [
1, filter_width, inner_last_dims, num_filters
]
w = tf_utils.initialize_weights(
filter_shape,
layer_name + "_w",
init_type=initialization,
gain=params["nonlinearity"],
divisor=params["num_classes"])
b = tf.get_variable(
layer_name + "_b",
initializer=tf.constant(
0.0 if initialization == "identity"
or initialization == "varscale" else 0.001,
shape=[num_filters]))
# h = tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, self.batch_norm, layer_name + "_r",
# self.batch_size, max_seq_len, self.res_activation, self.training) \
# if last_output != input_feats_expanded_drop \
# else tf_utils.residual_layer(inner_last_output, w, b, dilation, self.nonlinearity, False, layer_name + "_r",
# self.batch_size, max_seq_len, 0, self.training)
# conv shape: [batch, height, width, out_channels] -> [batch_size, 1, max_seq_len, last_dims]
conv = tf.nn.atrous_conv2d(inner_last_output,
w,
rate=dilation,
padding="SAME",
name=layer_name)
conv_b = tf.nn.bias_add(conv, b)
# h shape: [batch_size, 1, max_seq_len, last_dims]
h = tf_utils.apply_nonlinearity(
conv_b, params["nonlinearity"])
# so, only apply "take" to last block (may want to change this later)
if take_layer:
hidden_outputs.append(h)
total_output_width += num_filters
inner_last_dims = num_filters
inner_last_output = h
h_concat = tf.concat(axis=3, values=hidden_outputs)
last_output = tf.nn.dropout(h_concat,
params["middle_dropout_keep_prob"])
last_dims = total_output_width
h_concat_squeeze = tf.squeeze(h_concat, [1])
h_concat_flat = tf.reshape(h_concat_squeeze,
[-1, total_output_width])
# Add dropout
with tf.name_scope("hidden_dropout"):
h_drop = tf.nn.dropout(h_concat_flat,
params["hidden_dropout_keep_prob"])
def do_projection():
# Project raw outputs down
with tf.name_scope("projection"):
projection_width = int(total_output_width /
(2 * len(hidden_outputs)))
w_p = tf_utils.initialize_weights(
[total_output_width, projection_width],
"w_p",
init_type="xavier")
b_p = tf.get_variable("b_p",
initializer=tf.constant(
0.01,
shape=[projection_width]))
projected = tf.nn.xw_plus_b(h_drop,
w_p,
b_p,
name="projected")
projected_nonlinearity = tf_utils.apply_nonlinearity(
projected, params["nonlinearity"])
return projected_nonlinearity, projection_width
# only use projection if we wanted to, and only apply middle dropout here if projection
input_to_pred, proj_width = do_projection(
) if params["projection"] else (h_drop, total_output_width)
input_to_pred_drop = tf.nn.dropout(input_to_pred, params["middle_dropout_keep_prob"]) \
if params["projection"] else input_to_pred
# Final (unnormalized) scores and predictions
with tf.name_scope("output" + block_name_suff):
w_o = tf_utils.initialize_weights(
[proj_width, params["num_classes"]],
"w_o",
init_type="xavier")
b_o = tf.get_variable("b_o",
initializer=tf.constant(
0.01,
shape=[params["num_classes"]]))
l2_loss += tf.nn.l2_loss(w_o)
l2_loss += tf.nn.l2_loss(b_o)
scores = tf.nn.xw_plus_b(input_to_pred_drop,
w_o,
b_o,
name="scores")
unflat_scores = tf.reshape(
scores,
tf.stack(
[params["batch_size"], -1, params["num_classes"]]))
block_unflat_scores.append(unflat_scores)
return block_unflat_scores, h_concat_squeeze, l2_loss
def forward(self,
input_x1,
input_x2,
max_seq_len,
hidden_dropout_keep_prob,
input_dropout_keep_prob,
middle_dropout_keep_prob,
reuse=True):
word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1)
with tf.variable_scope("forward", reuse=reuse):
input_list = [word_embeddings]
input_size = self.embedding_size
if self.use_characters:
input_list.append(self.char_embeddings)
input_size += self.char_size
# todo add embeddings for all discrete features
if self.use_shape:
shape_embeddings_shape = (self.shape_domain_size - 1,
self.shape_size)
w_s = tf_utils.initialize_embeddings(shape_embeddings_shape,
name="w_s")
shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2)
input_list.append(shape_embeddings)
input_size += self.shape_size
if self.use_geometric_feats:
# it's giving some issue with the typing, so I'm just casting everythint ot be the same
input_list.append(tf.cast(self.widths, tf.float32))
input_list.append(tf.cast(self.heights, tf.float32))
input_list.append(tf.cast(self.wh_ratios, tf.float32))
input_list.append(tf.cast(self.x_coords, tf.float32))
input_list.append(tf.cast(self.y_coords, tf.float32))
# input_list.append(tf.cast(self.pages, tf.float32))
input_list.append(tf.cast(self.lines, tf.float32))
input_list.append(tf.cast(self.zones, tf.float32))
input_size += 7
if self.use_lexicons:
lex_embeddings_shape = (1, self.lex_size)
# params for lexicon embeddings
w_place = tf_utils.initialize_embeddings(lex_embeddings_shape,
name="w_place")
w_dept = tf_utils.initialize_embeddings(lex_embeddings_shape,
name="w_dept")
w_uni = tf_utils.initialize_embeddings(lex_embeddings_shape,
name="w_uni")
w_person = tf_utils.initialize_embeddings(lex_embeddings_shape,
name="w_person")
# embedding lookup tables
place_embeddings = tf.nn.embedding_lookup(
w_place, self.place_scores)
dept_embeddings = tf.nn.embedding_lookup(
w_dept, self.department_scores)
uni_embeddings = tf.nn.embedding_lookup(
w_uni, self.university_scores)
person_embeddings = tf.nn.embedding_lookup(
w_person, self.person_scores)
# add lex embeddings to input list
input_list.append(place_embeddings)
input_list.append(dept_embeddings)
input_list.append(uni_embeddings)
input_list.append(person_embeddings)
input_size += self.shape_size
# input_list.append(tf.cast(self.place_scores, tf.float32))
# input_list.append(tf.cast(self.department_scores, tf.float32))
# input_list.append(tf.cast(self.university_scores, tf.float32))
# input_list.append(tf.cast(self.person_scores, tf.float32))
input_size += 4 * self.lex_size
# print(input.get_shape())
# (w, h) = self.widths.get_shape()
# print(tf.reshape(self.widths, (w, h, 1)).get_shape())
input_feats = tf.concat(2, input_list)
print(input_feats.get_shape())
# self.input_feats_expanded = tf.expand_dims(self.input_feats, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats,
input_dropout_keep_prob)
total_output_width = 2 * self.hidden_dim
with tf.name_scope("bilstm"):
# selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch)
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim,
state_is_tuple=True)
bwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim,
state_is_tuple=True)
lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(
cell_fw=fwd_cell,
cell_bw=bwd_cell,
dtype=tf.float32,
inputs=input_feats_expanded_drop,
# inputs = input_feats,
parallel_iterations=50,
sequence_length=self.flat_sequence_lengths)
hidden_outputs = tf.concat(2, lstm_outputs)
# concatenate the results of the forward and backward cells
h_concat_flat = tf.reshape(hidden_outputs,
[-1, total_output_width])
# Add dropout
with tf.name_scope("middle_dropout"):
h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob)
# second projection
with tf.name_scope("tanh_proj"):
w_tanh = tf_utils.initialize_weights(
[total_output_width, self.hidden_dim],
"w_tanh",
init_type="xavier")
b_tanh = tf.get_variable(initializer=tf.constant(
0.01, shape=[self.hidden_dim]),
name="b_tanh")
self.l2_loss += tf.nn.l2_loss(w_tanh)
self.l2_loss += tf.nn.l2_loss(b_tanh)
h2_concat_flat = tf.nn.xw_plus_b(h_drop,
w_tanh,
b_tanh,
name="h2_tanh")
h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat,
self.nonlinearity)
# Add dropout
with tf.name_scope("hidden_dropout"):
h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob)
# Final (unnormalized) scores and predictions
with tf.name_scope("output"):
w_o = tf_utils.initialize_weights(
[self.hidden_dim, self.num_classes],
"w_o",
init_type="xavier")
b_o = tf.get_variable(initializer=tf.constant(
0.01, shape=[self.num_classes]),
name="b_o")
self.l2_loss += tf.nn.l2_loss(w_o)
self.l2_loss += tf.nn.l2_loss(b_o)
scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores")
unflat_scores = tf.reshape(
scores,
tf.pack([self.batch_size, max_seq_len, self.num_classes]))
return unflat_scores
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob,
input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True):
word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1)
with tf.variable_scope("forward", reuse=reuse):
input_list = [word_embeddings]
input_size = self.embedding_size
if self.use_characters:
input_list.append(self.char_embeddings)
input_size += self.char_size
if self.use_shape:
shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size)
w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s")
shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2)
input_list.append(shape_embeddings)
input_size += self.shape_size
if self.use_geometric_feats:
# TODO: add other features to input list, concat them to end of input_feats
# todo this is the wrong shape to be concatenated
# it's giving some issue with the typing, so I'm just casting everythint ot be the same
input_list.append(tf.cast(self.widths, tf.float32))
input_list.append(tf.cast(self.heights, tf.float32))
input_list.append(tf.cast(self.wh_ratios, tf.float32))
input_list.append(tf.cast(self.x_coords, tf.float32))
input_list.append(tf.cast(self.y_coords, tf.float32))
input_list.append(tf.cast(self.pages, tf.float32))
input_list.append(tf.cast(self.lines, tf.float32))
input_list.append(tf.cast(self.zones, tf.float32))
input_size += 8
# print(input.get_shape())
# (w, h) = self.widths.get_shape()
# print(tf.reshape(self.widths, (w, h, 1)).get_shape())
input_feats = tf.concat(2, input_list)
print(input_feats.get_shape())
# self.input_feats_expanded = tf.expand_dims(self.input_feats, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob)
total_output_width = self.hidden_dim # 2*self.hidden_dim
with tf.name_scope("lstm"):
# selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch)
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True)
lstm_outputs, _ = tf.nn.dynamic_rnn(cell=fwd_cell, dtype=tf.float32,
inputs=input_feats_expanded_drop,
# inputs = input_feats,
parallel_iterations=50,
sequence_length=self.flat_sequence_lengths)
# hidden_outputs = tf.concat(2, lstm_outputs)
# flatten the outputs of the lstm? is this necessary?
h_concat_flat = tf.reshape(lstm_outputs, [-1, total_output_width])
# Add dropout
with tf.name_scope("middle_dropout"):
h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob)
# second projection
with tf.name_scope("tanh_proj"):
w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier")
b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh")
self.l2_loss += tf.nn.l2_loss(w_tanh)
self.l2_loss += tf.nn.l2_loss(b_tanh)
h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh")
h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity)
# Add dropout
with tf.name_scope("hidden_dropout"):
h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob)
# Final (unnormalized) scores and predictions
with tf.name_scope("output"):
w_o = tf_utils.initialize_weights([self.hidden_dim, self.num_classes], "w_o", init_type="xavier")
b_o = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes]), name="b_o")
self.l2_loss += tf.nn.l2_loss(w_o)
self.l2_loss += tf.nn.l2_loss(b_o)
scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores")
unflat_scores = tf.reshape(scores, tf.pack([self.batch_size, max_seq_len, self.num_classes]))
return unflat_scores
def forward(self, input_x1, input_x2, max_seq_len, hidden_dropout_keep_prob,
input_dropout_keep_prob, middle_dropout_keep_prob, reuse=True):
word_embeddings = tf.nn.embedding_lookup(self.w_e, input_x1)
with tf.variable_scope("forward", reuse=reuse):
input_list = [word_embeddings]
input_size = self.embedding_size
if self.use_characters:
input_list.append(self.char_embeddings)
input_size += self.char_size
if self.use_shape:
shape_embeddings_shape = (self.shape_domain_size - 1, self.shape_size)
w_s = tf_utils.initialize_embeddings(shape_embeddings_shape, name="w_s")
shape_embeddings = tf.nn.embedding_lookup(w_s, input_x2)
input_list.append(shape_embeddings)
input_size += self.shape_size
input_feats = tf.concat(axis=2, values=input_list)
# self.input_feats_expanded = tf.expand_dims(self.input_feats, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats, input_dropout_keep_prob)
total_output_width = 2*self.hidden_dim
with tf.name_scope("bilstm"):
# selected_col_embeddings = tf.nn.embedding_lookup(token_embeddings, self.token_batch)
fwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True)
bwd_cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_dim, state_is_tuple=True)
lstm_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fwd_cell, cell_bw=bwd_cell, dtype=tf.float32,
inputs=input_feats_expanded_drop,
parallel_iterations=50,
sequence_length=self.flat_sequence_lengths)
hidden_outputs = tf.concat(axis=2, values=lstm_outputs)
h_concat_flat = tf.reshape(hidden_outputs, [-1, total_output_width])
# Add dropout
with tf.name_scope("middle_dropout"):
h_drop = tf.nn.dropout(h_concat_flat, middle_dropout_keep_prob)
# second projection
with tf.name_scope("tanh_proj"):
w_tanh = tf_utils.initialize_weights([total_output_width, self.hidden_dim], "w_tanh", init_type="xavier")
b_tanh = tf.get_variable(initializer=tf.constant(0.01, shape=[self.hidden_dim]), name="b_tanh")
self.l2_loss += tf.nn.l2_loss(w_tanh)
self.l2_loss += tf.nn.l2_loss(b_tanh)
h2_concat_flat = tf.nn.xw_plus_b(h_drop, w_tanh, b_tanh, name="h2_tanh")
h2_tanh = tf_utils.apply_nonlinearity(h2_concat_flat, self.nonlinearity)
# Add dropout
with tf.name_scope("hidden_dropout"):
h2_drop = tf.nn.dropout(h2_tanh, hidden_dropout_keep_prob)
# Final (unnormalized) scores and predictions
with tf.name_scope("output"):
w_o = tf_utils.initialize_weights([self.hidden_dim, self.num_classes], "w_o", init_type="xavier")
b_o = tf.get_variable(initializer=tf.constant(0.01, shape=[self.num_classes]), name="b_o")
self.l2_loss += tf.nn.l2_loss(w_o)
self.l2_loss += tf.nn.l2_loss(b_o)
scores = tf.nn.xw_plus_b(h2_drop, w_o, b_o, name="scores")
unflat_scores = tf.reshape(scores, tf.stack([self.batch_size, max_seq_len, self.num_classes]))
return unflat_scores, hidden_outputs
def forward(self, input_x1, input_dropout_keep_prob, reuse=True):
with tf.variable_scope("char-forward", reuse=reuse):
char_embeddings_lookup = tf.nn.embedding_lookup(
self.char_embeddings, input_x1)
print(char_embeddings_lookup.get_shape())
char_embeddings_flat = tf.reshape(
char_embeddings_lookup,
tf.pack([
self.batch_size * self.max_seq_len, self.max_tok_len,
self.embedding_size
]))
print(char_embeddings_flat.get_shape())
tok_lens_flat = tf.reshape(self.token_lengths,
[self.batch_size * self.max_seq_len])
print(tok_lens_flat.get_shape())
#
# input_list = [char_embeddings_lookup]
# # input_size = self.embedding_size
#
# input_feats = tf.concat(2, input_list)
input_feats_expanded = tf.expand_dims(char_embeddings_flat, 1)
input_feats_expanded_drop = tf.nn.dropout(input_feats_expanded,
input_dropout_keep_prob)
with tf.name_scope("char-cnn"):
filter_shape = [
1, self.filter_width, self.embedding_size, self.hidden_dim
]
w = tf_utils.initialize_weights(filter_shape,
"conv0_w",
init_type='xavier',
gain='relu')
b = tf.get_variable("conv0_b",
initializer=tf.constant(
0.01, shape=[self.hidden_dim]))
conv0 = tf.nn.conv2d(input_feats_expanded_drop,
w,
strides=[1, self.filter_width, 1, 1],
padding="SAME",
name="conv0")
print("conv0", conv0.get_shape())
h_squeeze = tf.squeeze(conv0, [1])
print("squeeze", h_squeeze.get_shape())
hidden_outputs = tf.reduce_max(h_squeeze, 1)
print("max", hidden_outputs.get_shape())
# h0 = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv0, b), 'relu')
# last_dims = self.embedding_size
# last_output = input_feats_expanded_drop
# hidden_outputs = []
# total_output_width = 0
# for layer_name, layer in self.layers_map:
# dilation = layer['dilation']
# filter_width = layer['width']
# num_filters = layer['filters']
# initialization = layer['initialization']
# take_layer = layer['take']
# print("Adding layer %s: dilation: %d; width: %d; filters: %d; take: %r" % (
# layer_name, dilation, filter_width, num_filters, take_layer))
# with tf.name_scope("atrous-conv-%s" % layer_name):
# # [filter_height, filter_width, in_channels, out_channels]
# filter_shape = [1, filter_width, last_dims, num_filters]
# w = tf_utils.initialize_weights(filter_shape, layer_name + "_w", init_type=initialization, gain='relu')
# b = tf.get_variable(layer_name + "_b", initializer=tf.constant(0.0 if initialization == 'identity' else 0.01, shape=[num_filters]))
# if dilation > 1:
# conv = tf.nn.atrous_conv2d(
# last_output,
# w,
# rate=dilation,
# padding="SAME",
# name=layer_name)
# else:
# conv = tf.nn.conv2d(last_output, w, strides=[1, filter_width, 1, 1], padding="SAME", name=layer_name)
# h = tf_utils.apply_nonlinearity(tf.nn.bias_add(conv, b), 'relu')
# if take_layer:
# hidden_outputs.append(h)
# total_output_width += num_filters
# last_dims = num_filters
# last_output = h
#
# h_concat = tf.concat(3, hidden_outputs)
# h_concat = tf.squeeze(h_concat, [1])
#
# fw_output = tf_utils.last_relevant(h_concat, tok_lens_flat)
# bw_output = h_concat[:, 0, :]
# hidden_outputs = tf.concat(1, [fw_output, bw_output])
# # hidden_outputs = tf.concat(2, lstm_outputs)
# # hidden_outputs = tf.Print(hidden_outputs, [tf.shape(hidden_outputs)], message='hidden outputs:')
hidden_outputs_unflat = tf.reshape(
hidden_outputs,
tf.pack(
[self.batch_size, self.max_seq_len, self.hidden_dim]))
return hidden_outputs_unflat
