def task_metalearn(inp, reuse=True):
""" Perform gradient descent for one task in the meta-batch. """
inputa, inputb, labela, labelb = inp
task_outputbs, task_lossesb = [], []
if self.classification:
task_accuraciesb = []
task_outputa = self.forward(
inputa, weights,
reuse=reuse) # only reuse on the first iter
task_lossa = self.loss_func(task_outputa, labela)
grads = tf.gradients(task_lossa, list(weights.values()))
if FLAGS.stop_grad:
grads = [tf.stop_gradient(grad) for grad in grads]
gradients = dict(zip(weights.keys(), grads))
fast_weights = dict(
zip(weights.keys(), [
weights[key] - self.update_lr * gradients[key]
for key in weights.keys()
]))
output = self.forward(inputb, fast_weights, reuse=True)
task_outputbs.append(output)
task_lossesb.append(self.loss_func(output, labelb))
for j in range(num_updates - 1):
loss = self.loss_func(
self.forward(inputa, fast_weights, reuse=True), labela)
grads = tf.gradients(loss, list(fast_weights.values()))
if FLAGS.stop_grad:
grads = [tf.stop_gradient(grad) for grad in grads]
gradients = dict(zip(fast_weights.keys(), grads))
fast_weights = dict(
zip(fast_weights.keys(), [
fast_weights[key] - self.update_lr * gradients[key]
for key in fast_weights.keys()
]))
output = self.forward(inputb, fast_weights, reuse=True)
task_outputbs.append(output)
task_lossesb.append(self.loss_func(output, labelb))
task_output = [
task_outputa, task_outputbs, task_lossa, task_lossesb
]
if self.classification:
task_accuracya = contrib_metrics.accuracy(
tf.argmax(tf.nn.softmax(task_outputa), 1),
tf.argmax(labela, 1))
for j in range(num_updates):
task_accuraciesb.append(
contrib_metrics.accuracy(
tf.argmax(tf.nn.softmax(task_outputbs[j]), 1),
tf.argmax(labelb, 1)))
task_output.extend([task_accuracya, task_accuraciesb])
return task_output
def build_net(self):
# 数据
image_placeholder = tf.placeholder(dtype=tf.float32, shape=(None, self.input_size[0], self.input_size[1], 4))
label_segment_placeholder = tf.placeholder(dtype=tf.int32, shape=(None, self.input_size[0] // self.ratio,
self.input_size[1] // self.ratio, 1))
label_classes_placeholder = tf.placeholder(dtype=tf.int32, shape=(None,))
# 网络
net = PSPNet({'data': image_placeholder}, is_training=True, num_classes=self.num_classes,
num_segment=self.num_segment, last_pool_size=self.last_pool_size, filter_number=self.filter_number)
raw_output_segment = net.layers['conv6_n_4']
raw_output_classes = net.layers['class_attention_fc']
# Predictions
prediction = tf.reshape(raw_output_segment, [-1, self.num_segment])
pred_segment = tf.cast(tf.expand_dims(tf.argmax(raw_output_segment, axis=-1), axis=-1), tf.int32)
pred_classes = tf.cast(tf.argmax(raw_output_classes, axis=-1), tf.int32)
# label
label_batch = tf.image.resize_nearest_neighbor(label_segment_placeholder,
tf.stack(raw_output_segment.get_shape()[1:3]))
label_batch = tf.reshape(label_batch, [-1, ])
# 当前批次的准确率：accuracy
accuracy_segment = tcm.accuracy(pred_segment, label_segment_placeholder)
accuracy_classes = tcm.accuracy(pred_classes, label_classes_placeholder)
# loss
loss_segment = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label_batch,
logits=prediction))
# 分类损失
loss_classes = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label_classes_placeholder,
logits=raw_output_classes))
# 总损失
loss = tf.add_n([loss_segment, 0.1 * loss_classes])
# 学习率策略
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(tf.constant(self.learning_rate), tf.pow((1 - step_ph / self.num_steps), 0.9))
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
# 单独训练最后的分类
classes_trainable = [v for v in tf.trainable_variables() if 'class_attention' in v.name]
train_classes_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, var_list=classes_trainable)
return (image_placeholder, label_segment_placeholder, label_classes_placeholder,
raw_output_segment, raw_output_classes, pred_segment, pred_classes,
loss_segment, loss_classes, loss, accuracy_segment, accuracy_classes,
step_ph, train_op, train_classes_op, learning_rate)
def accuracy_(score, labels):
'''
assumes score will be logits (ie unbounded real numbers)
'''
pred = tf.cast(tf.round(tf.sigmoid(score)), 'int8')
_labels = tf.cast(labels, 'int8')
return accuracy(pred, _labels)
def build_model(inputs, labels, num_classes):
weight_decay = 1e-3
conv1sz = 16
conv2sz = 32
fc3sz = 512
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=5, stride=1, padding='SAME', activation_fn=tf.nn.relu,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)
):
net = layers.convolution2d(inputs, conv1sz, scope='conv1')
# ostatak konvolucijskih i pooling slojeva
pool1 = layers.max_pool2d(inputs=net, kernel_size=[2, 2], stride=2)
conv2 = layers.convolution2d(pool1, conv2sz, scope="conv2")
pool2 = layers.max_pool2d(inputs=conv2, kernel_size=2, stride=2)
with tf.contrib.framework.arg_scope(
[layers.fully_connected],
activation_fn=tf.nn.relu,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)
):
# sada definiramo potpuno povezane slojeve
# ali najprije prebacimo 4D tenzor u matricu
net = layers.flatten(pool2) # originally inputs...?
net = layers.fully_connected(net, fc3sz, scope='fc3')
logits = layers.fully_connected(net, num_classes, activation_fn=None, scope='logits')
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(labels, logits))
accuracy = metrics.accuracy(tf.argmax(logits, 1), tf.argmax(labels, 1))
return logits, loss, accuracy
def build_trainer(self, timestamps, mmsis):
labels = self.multihot_labels(mmsis)
with tf.variable_scope("custom-loss"):
# Normal args are the totals for each correct value (as used in standard cross entropy)
normal_args = tf.reduce_sum(tf.to_float(labels) * self.prediction,
reduction_indices=[1])
# To encourage self consistency in the face of noise we also use a component where the args
# is the largest prediction.
consistent_args = tf.reduce_max(self.prediction, axis=[1])
#
positives = (1 - self.epsilon) * tf.log(
normal_args) + self.epsilon * tf.log(consistent_args)
raw_loss = -tf.reduce_mean(positives)
mask = tf.to_float(tf.equal(tf.reduce_sum(labels, 1), 1))
int_labels = tf.to_int32(tf.argmax(labels, 1))
int_predictions = tf.to_int32(tf.argmax(self.prediction, 1))
accuracy = metrics.accuracy(int_labels, int_predictions, weights=mask)
loss = raw_loss * self.loss_weight
update_ops = []
update_ops.append(
tf.summary.scalar('%s/Training-loss' % self.name, raw_loss))
update_ops.append(
tf.summary.scalar('%s/Training-accuracy' % self.name, accuracy))
return Trainer(loss, update_ops)
def predict_loss_acc(logits, real):
predict = tf.to_int32(tf.argmax(logits, real.shape.ndims))
loss = tf.losses.sparse_softmax_cross_entropy(logits = logits, labels = real, scope="loss")
#予測がラベルに一致する頻度を計算する
acc = tf_metrics.accuracy(predict, real)
return predict, loss, acc
def _compute_accuracy(logits, targets, weights=None):
if self._n_classes > 2:
_, predictions = nn.top_k(logits, 1)
else:
predictions = array_ops.reshape(logits, [-1])
predictions = math_ops.greater(predictions,
array_ops.zeros_like(predictions))
targets = array_ops.reshape(targets, [-1])
return metrics_lib.accuracy(
math_ops.to_int32(predictions), math_ops.to_int32(targets), weights)
def _compute_accuracy(logits, targets, weights=None):
if self._n_classes > 2:
_, predictions = nn.top_k(logits, 1)
else:
predictions = array_ops.reshape(logits, [-1])
predictions = math_ops.greater(
predictions, array_ops.zeros_like(predictions))
targets = array_ops.reshape(targets, [-1])
return metrics_lib.accuracy(math_ops.to_int32(predictions),
math_ops.to_int32(targets), weights)
def accuracy(self, params, data, labels):
"""Computes the accuracy (fraction of correct classifications).
Args:
params: List of parameter tensors or variables
data: Batch of features with samples along the first dimension
labels: Vector of labels with the same number of samples as the data
Returns:
accuracy: Fraction of correct classifications across the batch
"""
predictions = self.argmax(self.inference(params, data))
return contrib_metrics.accuracy(predictions, tf.cast(labels, tf.int32))
def accuracy_top_1(labels, logits=None, probs=None, decay=0.01):
""" calculate moving accuracy for classification
Arguments:
labels : [batch_size, nb_classes], must be one-hot
logits or probs : [batch_size, nb_classes]
decay : float in range [0, 1]
"""
if probs is not None:
acc = tcm.accuracy(predictions=tf.argmax(probs, axis=-1),
labels=tf.argmax(labels, axis=-1))
elif logits is not None:
acc = tcm.accuracy(predictions=tf.argmax(logits, axis=-1),
labels=tf.argmax(labels, axis=-1))
else:
raise Exception(
'in metric accuracy, the probability vector cannot be None')
if decay == 1.0:
return acc
else:
var = tf.Variable(0.0, name='acc_top_1')
return _assign_moving_average(var, acc, decay)
def construct(self, hidden_layer_size):
with self.session.graph.as_default():
with tf.name_scope("inputs"):
self.images = tf.placeholder(
tf.float32, [None, self.WIDTH, self.HEIGHT, 1],
name="images")
self.labels = tf.placeholder(tf.int64, [None], name="labels")
flattened_images = tf_layers.flatten(self.images,
scope="preprocessing")
hidden_layer = tf_layers.fully_connected(
flattened_images,
num_outputs=hidden_layer_size,
activation_fn=tf.nn.relu,
scope="hidden_layer")
output_layer = tf_layers.fully_connected(hidden_layer,
num_outputs=self.LABELS,
activation_fn=None,
scope="output_layer")
self.predictions = tf.argmax(output_layer, 1)
loss = tf_losses.sparse_softmax_cross_entropy(output_layer,
self.labels,
scope="loss")
self.global_step = tf.Variable(0,
dtype=tf.int64,
trainable=False,
name="global_step")
self.training = tf.train.AdamOptimizer().minimize(
loss, global_step=self.global_step)
self.accuracy = tf_metrics.accuracy(self.predictions, self.labels)
# Summaries
self.summaries = {
"training":
tf.merge_summary([
tf.scalar_summary("train/loss", loss),
tf.scalar_summary("train/accuracy", self.accuracy)
])
}
for dataset in ["dev", "test"]:
self.summaries[dataset] = tf.scalar_summary(
dataset + "/accuracy", self.accuracy)
# Initialize variables
self.session.run(tf.initialize_all_variables())
# Finalize graph and log it if requested
self.session.graph.finalize()
if self.summary_writer:
self.summary_writer.add_graph(self.session.graph)
def build_model(num_classes):
weight_decay = 1e-3
conv1sz = 16
conv2sz = 32
fc3sz = 256
fc4sz = 128
inputs = tf.placeholder(tf.float32, [None, 32, 32, 3])
labels = tf.placeholder(tf.int32, [None])
one_hot = tf.one_hot(labels, num_classes)
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=5, stride=1, padding='SAME', activation_fn=tf.nn.relu,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)
):
net = layers.convolution2d(inputs, conv1sz, scope='conv1')
# ostatak konvolucijskih i pooling slojeva
pool1 = layers.max_pool2d(inputs=net, kernel_size=3, stride=2)
conv2 = layers.convolution2d(pool1, conv2sz, scope="conv2")
pool2 = layers.max_pool2d(inputs=conv2, kernel_size=3, stride=2)
with tf.contrib.framework.arg_scope(
[layers.fully_connected],
activation_fn=tf.nn.relu,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)
):
# sada definiramo potpuno povezane slojeve
# ali najprije prebacimo 4D tenzor u matricu
net = layers.flatten(pool2) # originally inputs...?
net = layers.fully_connected(net, fc3sz, scope='fc3')
net = layers.fully_connected(net, fc4sz, scope='fc4')
logits = layers.fully_connected(net, num_classes, activation_fn=None, scope='logits')
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(one_hot, logits))
accuracy = metrics.accuracy(tf.argmax(logits, 1), tf.argmax(one_hot, 1))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-2)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return logits, loss, train_op, accuracy, inputs, labels
def build_trainer(self, timestamps, mmsis):
labels = self.multihot_labels(mmsis)
with tf.variable_scope("custom-loss"):
positives = tf.reduce_sum(tf.to_float(labels) * self.prediction,
reduction_indices=[1])
raw_loss = -tf.reduce_mean(tf.log(positives))
mask = tf.to_float(tf.equal(tf.reduce_sum(labels, 1), 1))
int_labels = tf.to_int32(tf.argmax(labels, 1))
int_predictions = tf.to_int32(tf.argmax(self.prediction, 1))
accuracy = metrics.accuracy(int_labels, int_predictions, weights=mask)
loss = raw_loss * self.loss_weight
update_ops = []
update_ops.append(
tf.summary.scalar('%s/Training-loss' % self.name, raw_loss))
update_ops.append(
tf.summary.scalar('%s/Training-accuracy' % self.name, accuracy))
return Trainer(loss, update_ops)
def __init__(self,
batch_size,
last_pool_size,
input_size,
log_dir,
data_root_path,
train_list,
data_path,
annotation_path,
class_path,
model_name="model.ckpt",
is_test=False):
# 和保存模型相关的参数
self.log_dir = Tools.new_dir(log_dir)
self.model_name = model_name
self.checkpoint_path = os.path.join(self.log_dir, self.model_name)
# 和数据相关的参数
self.input_size = input_size
self.batch_size = batch_size
self.num_classes = 21
self.num_segment = 4 # 解码通道数：其他对象、attention、边界、背景
self.segment_attention = 1 # 当解码的通道数是4时，attention所在的位置
self.attention_module_num = 2 # attention模块中，解码通道数是2（背景、attention）的模块个数
# 和模型相关的参数：必须保证input_size大于8倍的last_pool_size
self.ratio = 8
self.last_pool_size = last_pool_size
self.filter_number = 32
# 和模型训练相关的参数
self.learning_rate = 5e-3
self.num_steps = 500001
self.print_step = 5 if is_test else 25
# 读取数据
self.data_reader = Data(data_root_path=data_root_path,
data_list=train_list,
data_path=data_path,
annotation_path=annotation_path,
class_path=class_path,
batch_size=self.batch_size,
image_size=self.input_size,
is_test=is_test,
has_255=True)
# 数据
self.image_placeholder = tf.placeholder(dtype=tf.float32,
shape=(None,
self.input_size[0],
self.input_size[1], 4))
self.label_segment_placeholder = tf.placeholder(
dtype=tf.int32,
shape=(None, self.input_size[0] // self.ratio,
self.input_size[1] // self.ratio, 1))
self.label_attention_placeholder = tf.placeholder(
dtype=tf.int32,
shape=(None, self.input_size[0] // self.ratio,
self.input_size[1] // self.ratio, 1))
self.label_classes_placeholder = tf.placeholder(dtype=tf.int32,
shape=(None, ))
# 网络
self.net = BAISNet(self.image_placeholder,
is_training=True,
num_classes=self.num_classes,
num_segment=self.num_segment,
segment_attention=self.segment_attention,
last_pool_size=self.last_pool_size,
filter_number=self.filter_number,
attention_module_num=self.attention_module_num)
self.segments, self.attentions, self.classes = self.net.build()
self.final_segment_logit = self.segments[0]
self.final_class_logit = self.classes[0]
# Predictions
self.pred_segment = tf.cast(
tf.expand_dims(tf.argmax(self.final_segment_logit, axis=-1),
axis=-1), tf.int32)
self.pred_classes = tf.cast(tf.argmax(self.final_class_logit, axis=-1),
tf.int32)
# loss
self.label_batch = tf.image.resize_nearest_neighbor(
self.label_segment_placeholder,
tf.stack(self.final_segment_logit.get_shape()[1:3]))
self.label_attention_batch = tf.image.resize_nearest_neighbor(
self.label_attention_placeholder,
tf.stack(self.final_segment_logit.get_shape()[1:3]))
self.loss, self.loss_segment_all, self.loss_class_all, self.loss_segments, self.loss_classes = self.cal_loss(
self.segments,
self.classes,
self.label_batch,
self.label_attention_batch,
self.label_classes_placeholder,
self.num_segment,
attention_module_num=self.attention_module_num)
# 当前批次的准确率：accuracy
self.accuracy_segment = tcm.accuracy(self.pred_segment,
self.label_segment_placeholder)
self.accuracy_classes = tcm.accuracy(self.pred_classes,
self.label_classes_placeholder)
with tf.name_scope("train"):
# 学习率策略
self.step_ph = tf.placeholder(dtype=tf.float32, shape=())
self.learning_rate = tf.scalar_mul(
tf.constant(self.learning_rate),
tf.pow((1 - self.step_ph / self.num_steps), 0.9))
self.train_op = tf.train.GradientDescentOptimizer(
self.learning_rate).minimize(self.loss)
# 单独训练最后的attention
attention_trainable = [
v for v in tf.trainable_variables()
if 'attention' in v.name or "class_attention" in v.name
]
print(len(attention_trainable))
self.train_attention_op = tf.train.GradientDescentOptimizer(
self.learning_rate).minimize(self.loss,
var_list=attention_trainable)
pass
# summary 1
with tf.name_scope("loss"):
tf.summary.scalar("loss", self.loss)
tf.summary.scalar("loss_segment", self.loss_segment_all)
tf.summary.scalar("loss_class", self.loss_class_all)
for loss_segment_index, loss_segment in enumerate(
self.loss_segments):
tf.summary.scalar("loss_segment_{}".format(loss_segment_index),
loss_segment)
for loss_class_index, loss_class in enumerate(self.loss_classes):
tf.summary.scalar("loss_class_{}".format(loss_class_index),
loss_class)
pass
with tf.name_scope("accuracy"):
tf.summary.scalar("accuracy_segment", self.accuracy_segment)
tf.summary.scalar("accuracy_classes", self.accuracy_classes)
pass
with tf.name_scope("label"):
split = tf.split(self.image_placeholder,
num_or_size_splits=4,
axis=3)
tf.summary.image("0-mask", split[3])
tf.summary.image("1-image", tf.concat(split[0:3], axis=3))
tf.summary.image(
"2-label",
tf.cast(self.label_segment_placeholder * 85, dtype=tf.uint8))
tf.summary.image(
"3-attention",
tf.cast(self.label_attention_placeholder * 255,
dtype=tf.uint8))
pass
with tf.name_scope("predict"):
tf.summary.image("predict",
tf.cast(self.pred_segment * 85, dtype=tf.uint8))
pass
with tf.name_scope("attention"):
# attention
for attention_index, attention in enumerate(self.attentions):
tf.summary.image("{}-attention".format(attention_index),
attention)
pass
pass
with tf.name_scope("sigmoid"):
for segment_index, segment in enumerate(self.segments):
if segment_index < self.attention_module_num:
split = tf.split(segment,
num_or_size_splits=self.num_segment,
axis=3)
tf.summary.image("{}-other".format(segment_index),
split[0])
tf.summary.image("{}-attention".format(segment_index),
split[1])
tf.summary.image("{}-border".format(segment_index),
split[2])
tf.summary.image("{}-background".format(segment_index),
split[-1])
else:
split = tf.split(segment, num_or_size_splits=2, axis=3)
tf.summary.image("{}-background".format(segment_index),
split[0])
tf.summary.image("{}-attention".format(segment_index),
split[1])
pass
pass
pass
self.summary_op = tf.summary.merge_all()
# sess 和 saver
self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True)))
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(var_list=tf.global_variables(),
max_to_keep=10)
# summary 2
self.summary_writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
pass
def conv_model_eval_metrics(classes, Y_, mode):
return {'accuracy': metrics.accuracy(classes, Y_)} \
if mode == learn.ModeKeys.TRAIN or mode == learn.ModeKeys.EVAL else None
# In[3]:
outputs, v_list = model_func(x_train_batch)
for v in v_list:
tf.add_to_collection('vars', v)
tf.add_to_collection('vars', outputs)
loss = tf.losses.sparse_softmax_cross_entropy(y_train_batch , outputs)
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_op = slim.learning.create_train_op(loss, optimizer)
# In[4]:
acc = accuracy(tf.cast(tf.arg_max(outputs,1),tf.int32) , y_train_batch)
# In[5]:
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# In[6]:
_ = tf.train.start_queue_runners(sess=sess)
# ## Train
def _build_network(self, input_width):
with self.session.graph.as_default():
if self.rnn_cell_type == "LSTM":
self.rnn_cell = tf.contrib.rnn.LSTMCell(self.rnn_cell_dim)
elif self.rnn_cell_type == "GRU":
self.rnn_cell = tf.contrib.rnn.GRUCell(self.rnn_cell_dim)
else:
raise ValueError("Unknown rnn_cell {}".format(rnn_cell))
self.global_step = tf.Variable(0,
dtype=tf.int64,
trainable=False,
name='global_step')
self.tokens = tf.placeholder(tf.int32, [None, None, None],
name="tokens")
self.token_lens = tf.placeholder(tf.int32, [None, None],
name="token_lens")
self.features = tf.placeholder(tf.float32, [None, None],
name="features")
self.labels = tf.placeholder(tf.int64, [None], name="labels")
self.alphabet_size = len(self.char_vocabulary.classes_)
self.dropout_keep = tf.placeholder(tf.float32)
self.input_width = input_width
char_embedding_matrix = tf.get_variable(
"char_embeddings", [self.alphabet_size, self.EMBEDDING_SIZE],
initializer=tf.random_normal_initializer(stddev=0.01),
dtype=tf.float32)
with tf.variable_scope("token_encoder"):
tokens_flat = tf.reshape(self.tokens,
[-1, tf.shape(self.tokens)[-1]])
token_lens_flat = tf.reshape(self.token_lens, [-1])
char_embeddings = tf.nn.embedding_lookup(
char_embedding_matrix, tokens_flat)
hidden_states, final_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=self.rnn_cell,
cell_bw=self.rnn_cell,
inputs=char_embeddings,
sequence_length=token_lens_flat,
dtype=tf.float32,
scope="char_BiRNN")
tokens_encoded = tf_layers.linear(tf.concat(final_states, 1),
self.EMBEDDING_SIZE,
scope="tokens_encoded")
tokens_encoded = tf.reshape(tokens_encoded,
[tf.shape(self.features)[0], -1])
self.input_layer = tf.concat((tokens_encoded, self.features), 1)
self.input_layer = tf.reshape(self.input_layer,
[-1, self.input_width])
# input transform
self.hidden_layer = tf.nn.dropout(
tf_layers.fully_connected(self.input_layer,
num_outputs=self.h_width,
activation_fn=None,
scope="input_layer"),
self.dropout_keep)
# hidden layers
for i in range(self.h_depth):
if self.layer_type == "FeedForward":
self.hidden_layer = tf.nn.dropout(
tf_layers.fully_connected(
self.hidden_layer,
num_outputs=self.h_width,
activation_fn=tf.nn.relu,
scope="ff_layer_{}".format(i)), self.dropout_keep)
elif self.layer_type == "Highway":
self.hidden_layer = tf.nn.dropout(
highway_layer(self.hidden_layer,
num_outputs=self.h_width,
activation_fn=tf.nn.relu,
scope="highway_layer_{}".format(i)),
self.dropout_keep)
else:
raise ValueError("Unknown hidden layer type.")
self.output_layer = tf_layers.fully_connected(
self.hidden_layer,
num_outputs=len(self.target_encoder.classes_),
activation_fn=None,
scope="output_layer")
self.predictions = tf.argmax(self.output_layer, 1)
self.loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.output_layer, labels=self.labels),
name="loss")
self.training = tf.train.AdamOptimizer().minimize(
self.loss, global_step=self.global_step)
self.accuracy = tf_metrics.accuracy(self.predictions, self.labels)
self.summary = tf.summary.merge([
tf.summary.scalar("train/loss", self.loss),
tf.summary.scalar("train/accuracy", self.accuracy)
])
self._initialize_variables()
def conv_model_eval_metrics(classes, Y_, mode):
# You can name the fields of your metrics dictionary as you like.
return {'accuracy': metrics.accuracy(classes, Y_)} \
if mode == learn.ModeKeys.TRAIN or mode == learn.ModeKeys.EVAL else None
def __init__(self,
rnn_cell,
rnn_cell_dim,
method,
words,
logdir,
expname,
threads=1,
seed=42,
character_count=50,
embedding_matrix=None,
max_sentence_length=None,
embedding_dim=100,
distinct_tags=2):
# Create an empty graph and a session
graph = tf.Graph()
graph.seed = seed
self.session = tf.Session(graph=graph,
config=tf.ConfigProto(
inter_op_parallelism_threads=threads,
intra_op_parallelism_threads=threads))
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
# Construct the graph
with self.session.graph.as_default():
if rnn_cell == "LSTM":
rnn_cell = tf.contrib.rnn.LSTMCell(rnn_cell_dim)
elif rnn_cell == "GRU":
rnn_cell = tf.contrib.rnn.GRUCell(rnn_cell_dim)
else:
raise ValueError("Unknown rnn_cell {}".format(rnn_cell))
self.embedding_matrix = embedding_matrix
self.global_step = tf.Variable(0,
dtype=tf.int64,
trainable=False,
name="global_step")
self.sentence_lens = tf.placeholder(tf.int32, [None])
self.forms = tf.placeholder(tf.int32, [None, None])
self.tags = tf.placeholder(tf.int64, [None])
self.char_ids = tf.placeholder(tf.int32, [None, None])
self.charseq_lens = tf.placeholder(tf.int32, [None])
self.charseqs = tf.placeholder(tf.int32,
[None, max_sentence_length])
self.is_first = tf.placeholder(tf.bool)
self.embedding_placeholder = tf.placeholder_with_default(
tf.random_normal([words, embedding_dim]),
[words, embedding_dim])
alpha = .3
if method == "learned_we":
embeddings = tf.Variable(initial_value=tf.random_normal(
[words, embedding_dim]),
dtype=tf.float32)
encoded_inputs = tf.nn.embedding_lookup(embeddings,
ids=self.forms)
elif method == "updated_pretrained_we":
emb_W = tf.Variable(tf.constant(0.0,
shape=[words, embedding_dim]),
trainable=True,
name="W")
embeddings = cond(self.is_first,
lambda: emb_W.assign(self.embedding_matrix),
lambda: emb_W)
encoded_inputs = tf.nn.embedding_lookup(embeddings,
ids=self.forms)
elif method == "only_pretrained_we":
emb_W = tf.Variable(tf.constant(0.0,
shape=[words, embedding_dim]),
trainable=False,
name="W")
embeddings = cond(self.is_first,
lambda: emb_W.assign(self.embedding_matrix),
lambda: emb_W)
encoded_inputs = tf.nn.embedding_lookup(embeddings,
ids=self.forms)
elif method == "char_rnn":
embeddings = self.compute_embedding_rnn(
self.charseqs, self.charseq_lens, character_count)
encoded_inputs = tf.nn.embedding_lookup(
embeddings, self.char_ids)
elif method == "char_conv":
embeddings = self.compute_embedding_conv(
self.charseqs, character_count)
encoded_inputs = tf.nn.embedding_lookup(
embeddings, self.char_ids)
with tf.variable_scope("RNN"):
(outputs, states) = tf.nn.bidirectional_dynamic_rnn(
rnn_cell,
rnn_cell,
encoded_inputs,
sequence_length=self.sentence_lens,
time_major=False,
dtype=tf.float32)
self.logits = tf_layers.linear(tf.concat(states, 1), distinct_tags)
# self.logits = tf.Print(self.logits, [self.logits], message="This is a: ")
self.labels = tf.one_hot(self.tags, distinct_tags, dtype=tf.int64)
mask = tf.sequence_mask(self.sentence_lens,
tf.reduce_max(self.sentence_lens))
logits = alpha * outputs[0] + (1 - alpha) * outputs[1]
self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.logits, labels=self.tags)
self.optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
self.training = self.optimizer.minimize(self.loss)
self.predictions = tf.argmax(self.logits, 1)
self.accuracy = tf_metrics.accuracy(self.predictions,
tf.argmax(self.labels, 1))
self.dataset_name = tf.placeholder(tf.string, [])
# Initialize variables
self.session.run(tf.initialize_all_variables())
def __init__(self,
encoder, decoder,
rnn_cell, rnn_cell_dim,
chars_size, words_size, tags_size,
bow_char, eow_char,
logdir, expname,
threads=1, seed=42):
# Create an empty graph and a session
graph = tf.Graph()
graph.seed = seed
self.session = tf.Session(
graph=graph,
config=tf.ConfigProto(
inter_op_parallelism_threads=threads,
intra_op_parallelism_threads=threads))
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.summary_writer = tf.summary.FileWriter("{}/{}-{}".format(logdir, timestamp, expname), flush_secs=10)
# Construct the graph
with self.session.graph.as_default():
if rnn_cell == "LSTM":
rnn_cell = tf.contrib.rnn.LSTMCell(rnn_cell_dim)
elif rnn_cell == "GRU":
rnn_cell = tf.contrib.rnn.GRUCell(rnn_cell_dim)
else:
raise ValueError("Unknown rnn_cell {}".format(rnn_cell))
self.global_step = tf.Variable(0, dtype=tf.int64, trainable=False, name="global_step")
self.sentence_lens = tf.placeholder(tf.int32, [None], name="sent_lens")
self.lemma_ids = tf.placeholder(tf.int32, [None, None], name="lemma_ids")
self.lemmas = tf.placeholder(tf.int64, [None, None], name="lemmas")
self.lemma_lens = tf.placeholder(tf.int32, [None], name="lemma_lens")
self.tag_ids = tf.placeholder(tf.int32, [None, None], name="tag_ids")
self.tags = tf.placeholder(tf.int64, [None, None], name="tags")
self.tag_lens = tf.placeholder(tf.int32, [None], name="tag_lens")
self.form_ids = tf.placeholder(tf.int32, [None, None], name="form_ids")
self.forms = tf.placeholder(tf.int64, [None, None], name="forms")
self.form_lens = tf.placeholder(tf.int32, [None], name="form_lens")
self.alphabet_len = chars_size
self.word_vocab_len = words_size
self.tag_vocab_len = tags_size
self.dummy_inputs = tf.zeros([tf.shape(self.sentence_lens)[0], self.MAX_GEN_LEN], name="inference_shape")
self.char_embedding_matrix = tf.get_variable(
"char_embeddings",
[self.alphabet_len, self.EMBEDDING_SIZE],
initializer=tf.random_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.we_lookup_matrix = tf.get_variable(
"we_lookup_matrix",
[self.word_vocab_len, self.EMBEDDING_SIZE],
initializer=tf.random_normal_initializer(stddev=0.01),
dtype=tf.float32,
trainable=True)
self.tag_lookup_matrix = tf.get_variable(
"tag_lookup_matrix",
[self.tag_vocab_len, self.EMBEDDING_SIZE],
initializer=tf.random_normal_initializer(stddev=0.01),
dtype=tf.float32,
trainable=True)
# Encode words
with tf.variable_scope("encoder"):
self.char_embeddings = tf.nn.embedding_lookup(self.char_embedding_matrix, self.lemmas)
ch_rnn_cell = tf.contrib.rnn.GRUCell(rnn_cell_dim)
hidden_states, final_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=ch_rnn_cell,
cell_bw=ch_rnn_cell,
inputs=self.char_embeddings,
sequence_length=self.lemma_lens,
dtype=tf.float32,
scope="char_BiRNN")
self.sentence_mask = tf.sequence_mask(self.sentence_lens)
# Create decoder input
self.we_encoder_matrix = tf_layers.linear(
tf.concat(axis=1, values=final_states),
self.EMBEDDING_SIZE,
scope="we_encoder_matrix")
self.encoder_output = tf.nn.embedding_lookup(self.we_encoder_matrix, self.lemma_ids)
self.encoder_output = tf.reshape(
tf.boolean_mask(self.encoder_output, self.sentence_mask),
[-1, self.EMBEDDING_SIZE],
name="encoder_output_flat")
# Encode tags
self.tags_embedded = tf.nn.embedding_lookup(self.tag_lookup_matrix, self.tag_ids)
self.tags_embedded = tf.reshape(
tf.boolean_mask(self.tags_embedded, self.sentence_mask),
[-1, self.EMBEDDING_SIZE],
name="tag_embeddings_flat")
# Combine encoder_output with tag embedding
self.encoder_output = tf_layers.linear(
tf.concat(axis=1, values=[self.encoder_output, self.tags_embedded]),
self.EMBEDDING_SIZE,
scope="encoder_output_with_tags")
# Create annotations for attention
self.annot_matrix = tf_layers.linear(
tf.concat(axis=2, values=hidden_states),
self.EMBEDDING_SIZE,
scope="annot_matrix")
self.annotations = tf.nn.embedding_lookup(self.annot_matrix, self.lemma_ids)
self.annotations = tf.reshape(
tf.boolean_mask(self.annotations, self.sentence_mask),
[-1, tf.shape(self.annot_matrix)[1], self.EMBEDDING_SIZE],
name="annotations_flat")
# Reshape form values
self.forms_flat = tf.nn.embedding_lookup(self.forms, self.form_ids)
self.forms_flat = tf.reshape(
tf.boolean_mask(self.forms_flat, self.sentence_mask),
[-1, tf.shape(self.forms)[1]],
name="forms_flat")
self.forms_flat_lens = tf.nn.embedding_lookup(self.form_lens, self.form_ids)
self.forms_flat_lens = tf.reshape(
tf.boolean_mask(self.forms_flat_lens, self.sentence_mask),
[-1],
name="lemmas_flat_lens")
self.attention_fn = None
if decoder in ["individual", "individual_attention", "combined_attention", "combined_attention_birnn"]:
if decoder in ["individual_attention", "combined_attention", "combined_attention_birnn"]:
#self.attention_fn = self.attention_fn_builder(self.annotations)
if decoder == "combined_attention":
word_embeddings = tf.nn.embedding_lookup(self.we_lookup_matrix, self.lemma_ids)
word_embeddings = tf.reshape(
tf.boolean_mask(word_embeddings, self.sentence_mask),
[-1, self.EMBEDDING_SIZE],
name="word_embeddings_flat")
self.encoder_output = tf_layers.linear(
tf.concat(axis=1, values=[self.encoder_output, word_embeddings]),
self.EMBEDDING_SIZE,
scope="combined_encoder_output")
if decoder == "combined_attention_rnn":
else:
raise ValueError("Unknown decoder ({}).".format(decoder))
# Decoder training
with tf.variable_scope("decoder"):
if decoder == "individual":
self.training_logits, states = tf_seq2seq.rnn_decoder(
decoder_inputs=self.forms_flat,
initial_state=self.encoder_output,
cell=rnn_cell)
else:
self.training_logits, states = tf_seq2seq.attention_decoder(
decoder_inputs=self.forms_flat,
initial_state=self.encoder_output,
attention_states=self.annotations,
cell=rnn_cell)
#self.training_logits, states = tf_seq2seq.dynamic_rnn_decoder(
#cell=rnn_cell,
#decoder_fn=self.decoder_fn_train(
# self.encoder_output,
# self.output_fn_builder(),
# self.input_fn_builder(self.char_embedding_matrix, self.attention_fn)),
#inputs=tf.expand_dims(self.forms_flat, -1),
#sequence_length=self.forms_flat_lens)
# Decoder inference
with tf.variable_scope("decoder", reuse=True):
if decoder == "individual":
self.training_logits, states = tf_seq2seq.rnn_decoder(
decoder_inputs=self.dummy_inputs,
initial_state=self.encoder_output,
cell=rnn_cell,
loop_function=decoder_fn)
else:
self.training_logits, states = tf_seq2seq.attention_decoder(
decoder_inputs=self.dummy_inputs,
initial_state=self.encoder_output,
attention_states=self.annotations,
cell=rnn_cell,
loop_function=decoder_fn)
#self.inference_logits, states = tf_seq2seq.dynamic_rnn_decoder(
#cell=rnn_cell,
#decoder_fn=self.decoder_fn_inference(
# self.encoder_output,
# self.output_fn_builder(),
# self.input_fn_builder(self.char_embedding_matrix, self.attention_fn),
#bow_char,
#eow_char,
#self.MAX_GEN_LEN))
self.predictions = tf.argmax(self.inference_logits, 2)
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.training_logits, labels=self.forms_flat[:,1:]))
self.training = tf.train.AdamOptimizer().minimize(loss, global_step=self.global_step)
self.forms_flat = tf.cond(
tf.reduce_max(self.forms_flat_lens) > self.MAX_GEN_LEN,
lambda: tf.slice(self.forms_flat, [0, 0], [-1, self.MAX_GEN_LEN]),
lambda: self.forms_flat)
self.pred_padded = tf.pad(
self.predictions,
[[0,0],[0, self.MAX_GEN_LEN - tf.shape(self.predictions)[1]]],
mode="CONSTANT")
self.forms_padded = tf.pad(
self.forms_flat,
[[0,0],[0, self.MAX_GEN_LEN - tf.shape(self.forms_flat)[1] + 1]],
mode="CONSTANT")
self.char_accuracy = tf_metrics.accuracy(self.pred_padded, self.forms_padded[:,1:])
self.word_accuracy = tf.reduce_mean(tf.reduce_min(tf.cast(tf.equal(self.pred_padded, self.forms_padded[:,1:]), tf.float32), axis=1))
self.summary = {}
for dataset_name in ["train", "dev"]:
self.summary[dataset_name] = tf.summary.merge([tf.summary.scalar(dataset_name+"/loss", loss),
tf.summary.scalar(dataset_name+"/char_accuracy", self.char_accuracy),
tf.summary.scalar(dataset_name+"/word_accuracy", self.word_accuracy)])
# Initialize variables
self.session.run(tf.global_variables_initializer())
if self.summary_writer:
self.summary_writer.add_graph(self.session.graph)
# Simple decoder for training
def decoder_fn_train(self, encoder_state, output_fn, input_fn, name=None):
def decoder_fn(time, cell_state, next_id, cell_output, context_state):
cell_output = output_fn(cell_output)
reuse = True
if cell_state is None: # first call, return encoder_state
cell_state = encoder_state
reuse = None
next_input = input_fn(tf.squeeze(next_id, [1]), cell_state, reuse)
return (None, cell_state, next_input, cell_output, context_state)
return decoder_fn
# TODO: Beam search
# Simple decoder for inference
def decoder_fn_inference(self, encoder_state, output_fn, input_fn,
beginning_of_word="<bow>", end_of_word="<eow>", maximum_length=MAX_GEN_LEN):
batch_size = tf.shape(encoder_state)[0]
def decoder_fn(time, cell_state, cell_input, cell_output, context_state):
cell_output = output_fn(cell_output)
if cell_state is None:
cell_state = encoder_state
next_id = tf.tile([beginning_of_word], [batch_size])
done = tf.zeros([batch_size], dtype=tf.bool)
else:
next_id = tf.argmax(cell_output, 1)
done = tf.equal(next_id, end_of_word)
done = tf.cond(
tf.greater_equal(time, maximum_length), # return true if time >= maxlen
lambda: tf.ones([batch_size], dtype=tf.bool),
lambda: done)
next_input = input_fn(next_id, cell_state, True)
return (done, cell_state, next_input, cell_output, context_state)
return decoder_fn
def decoder_fn_builder(self, encoder_state, output_fn, input_fn,
beginning_of_word="<bow>", end_of_word="<eow>", maximum_length=MAX_GEN_LEN):
def decoder_fn(cell_output, i):
cell_output = output_fn(cell_output)
next_input = tf.argmax(cell_output, 1)
next_input = input_fn(next_input)
return decoder_fn
# TODO: dropout
def attention_fn_builder(self, annotations):
def attention_fn(state):
batch_size = tf.shape(state)[0]
annot_len = tf.shape(annotations)[1]
annot_dim = annotations.get_shape().as_list()[2]
state_dim = state.get_shape().as_list()[1]
e_dim = self.ALIGNMENT_SIZE
a = tf.reshape(annotations, [-1, annot_dim])
U = tf.get_variable(
"annot_weight",
shape=[annot_dim, e_dim],
initializer=tf.random_normal_initializer(stddev=0.1),
trainable=True)
U_b = tf.get_variable(
"annot_bias",
shape=[e_dim],
initializer=tf.constant_initializer(0.1))
W = tf.get_variable(
"state_weight",
shape=[state_dim, e_dim],
initializer=tf.random_normal_initializer(stddev=0.1),
trainable=True)
W_b = tf.get_variable(
"state_bias",
shape=[e_dim],
initializer=tf.constant_initializer(0.1))
v = tf.get_variable(
"lin_combo",
shape=[e_dim, 1],
initializer=tf.random_normal_initializer(stddev=0.1),
trainable=True)
w_res = tf.matmul(state, W) + W_b
w_res = tf.tile(tf.reshape(w_res, [-1, 1]), [1, annot_len])
u_res = tf.matmul(a, U) + U_b
u_res = tf.reshape(u_res, [-1, annot_len])
e = tf.matmul(tf.tanh(tf.reshape(w_res + u_res, [-1, e_dim])), v)
e = tf.reshape(e, [batch_size, -1])
alpha = tf.nn.softmax(e)
alpha = tf.tile(tf.reshape(alpha, [-1, 1]), [1, annot_dim])
c = tf.multiply(alpha, a)
c = tf.reduce_sum(tf.reshape(c, [batch_size, -1, annot_dim]), 1)
C = tf.get_variable(
"attention_weight",
shape=[state_dim, state_dim],
initializer=tf.random_normal_initializer(stddev=0.1),
trainable=True)
C_b = tf.get_variable(
"attention_bias",
shape=[state_dim],
initializer=tf.constant_initializer(0.1))
return tf.add(tf.matmul(c, C), C_b)
return attention_fn
# Output function builder (makes logits out of rnn outputs)
def output_fn_builder(self):
def output_fn(cell_output):
if cell_output is None:
return tf.zeros([self.alphabet_len], tf.float32) # only used for shape inference
else:
return tf_layers.linear(
cell_output,
num_outputs=self.alphabet_len,
scope="decoder_output")
return output_fn
# Input function builder (makes rnn input from word id and cell state)
def input_fn_builder(self, embeddings):
def input_fn(next_id):
return tf.nn.embedding_lookup(embeddings, next_id)
return input_fn
# Input function builder (makes rnn input from word id and cell state)
#def input_fn_builder(self, embeddings, attention_fn=None):
# def input_fn(next_id, cell_state, reuse=True):
# if attention_fn is not None:
# with tf.variable_scope("attention", reuse=reuse):
# return tf.add(
# tf.nn.embedding_lookup(embeddings, next_id),
# attention_fn(cell_state))
# else:
# return tf.nn.embedding_lookup(embeddings, next_id)
#
# return input_fn
@property
def training_step(self):
return self.session.run(self.global_step)
def train(self,
sentence_lens,
forms, form_ids, form_lens,
tags, tag_ids, tag_lens,
lemmas, lemma_ids, lemma_lens):
try:
_, summary, pred = self.session.run([self.training, self.summary, self.predictions],
{self.sentence_lens: sentence_lens,
self.forms: forms,
self.form_ids: form_ids,
self.form_lens: form_lens,
self.tags: tags,
self.tag_ids: tag_ids,
self.tag_lens: tag_lens,
self.lemmas: lemmas,
self.lemma_ids: lemma_ids,
self.lemma_lens: lemma_lens})
except Exception as e:
import pdb; pdb.set_trace()
raise e
self.summary_writer.add_summary(summary["train"], self.training_step)
def evaluate(self,
sentence_lens,
forms, form_ids, form_lens,
tags, tag_ids, tag_lens,
lemmas, lemma_ids, lemma_lens):
try:
ch_acc, w_acc, summary, pred = self.session.run([self.char_accuracy, self.word_accuracy, self.summary, self.predictions],
{self.sentence_lens: sentence_lens,
self.forms: forms,
self.form_ids: form_ids,
self.form_lens: form_lens,
self.tags: tags,
self.tag_ids: tag_ids,
self.tag_lens: tag_lens,
self.lemmas: lemmas,
self.lemma_ids: lemma_ids,
self.lemma_lens: lemma_lens})
except Exception as e:
import pdb; pdb.set_trace()
raise e
self.summary_writer.add_summary(summary["dev"], self.training_step)
return ch_acc, w_acc
def predict(self,
sentence_lens,
lemmas, lemma_ids, lemma_lens,
tags, tag_ids, tag_lens):
predictions = self.session.run([self.predictions],
{self.sentence_lens: sentence_lens,
self.lemmas: lemmas,
self.lemma_ids: lemma_ids,
self.lemma_lens: lemma_lens,
self.tags: tags,
self.tag_ids: tag_ids,
self.tag_lens: tag_lens})
return predictions
def conv_model_eval_metrics(classes, Y_, mode):
# You can name the fields of your metrics dictionary as you like.
return {'accuracy': metrics.accuracy(classes, Y_)} \
if mode == learn.ModeKeys.TRAIN or mode == learn.ModeKeys.EVAL else None
def __init__(self, batch_size, input_size, log_dir, data_root_path, train_list, data_path,
annotation_path, class_path, model_name="model.ckpt", pretrain=None, is_test=False):
# 和保存模型相关的参数
self.log_dir = Tools.new_dir(log_dir)
self.model_name = model_name
self.checkpoint_path = os.path.join(self.log_dir, self.model_name)
self.pretrain = pretrain
# 和数据相关的参数
self.input_size = input_size
self.batch_size = batch_size
self.num_classes = 21
# 和模型训练相关的参数
self.learning_rate = 5e-4
self.num_steps = 500001
self.print_step = 10 if is_test else 100
self.cal_step = 100 if is_test else 1000
# 读取数据
self.data_reader = Data(data_root_path=data_root_path, data_list=train_list,
data_path=data_path, annotation_path=annotation_path, class_path=class_path,
batch_size=self.batch_size, image_size=self.input_size, is_test=is_test)
# 数据
self.image_placeholder = tf.placeholder(tf.float32, shape=(None, self.input_size[0], self.input_size[1], 3))
self.mask_placeholder = tf.placeholder(tf.float32, shape=(None, self.input_size[0], self.input_size[1], 1))
self.label_seg_placeholder = tf.placeholder(tf.int32, shape=(None, self.input_size[0], self.input_size[1], 1))
self.label_cls_placeholder = tf.placeholder(tf.int32, shape=(None,))
# 网络
self.net = BAISNet(self.image_placeholder, self.mask_placeholder, True, num_classes=self.num_classes)
self.segments, self.attentions, self.classes = self.net.build()
# loss
self.loss, self.loss_segment_all, self.loss_class_all, self.loss_segments, self.loss_classes = self.cal_loss(
self.segments, self.attentions, self.classes, self.label_seg_placeholder, self.label_cls_placeholder)
# 当前批次的准确率：accuracy
self.pred_classes = tf.cast(tf.argmax(self.classes[0], axis=-1), tf.int32)
self.accuracy_classes = tcm.accuracy(self.pred_classes, self.label_cls_placeholder)
with tf.name_scope("train"):
# 学习率策略
self.step_ph = tf.placeholder(dtype=tf.float32, shape=())
self.learning_rate = tf.scalar_mul(tf.constant(self.learning_rate),
tf.pow((1 - self.step_ph / self.num_steps), 0.9))
self.train_op = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(self.loss)
# 单独训练最后的attention
attention_trainable = [v for v in tf.trainable_variables()
if 'attention' in v.name or "class_attention" in v.name]
print(len(attention_trainable))
self.train_attention_op = tf.train.GradientDescentOptimizer(
self.learning_rate).minimize(self.loss, var_list=attention_trainable)
pass
# summary 1
with tf.name_scope("loss"):
tf.summary.scalar("loss", self.loss)
tf.summary.scalar("loss_segment", self.loss_segment_all)
tf.summary.scalar("loss_class", self.loss_class_all)
for loss_segment_index, loss_segment in enumerate(self.loss_segments):
tf.summary.scalar("loss_segment_{}".format(loss_segment_index), loss_segment)
for loss_class_index, loss_class in enumerate(self.loss_classes):
tf.summary.scalar("loss_class_{}".format(loss_class_index), loss_class)
pass
with tf.name_scope("accuracy"):
tf.summary.scalar("accuracy_classes", self.accuracy_classes)
pass
with tf.name_scope("label"):
tf.summary.image("1-image", self.image_placeholder)
tf.summary.image("2-segment", tf.cast(self.label_seg_placeholder * 255, dtype=tf.uint8))
tf.summary.image("3-mask", tf.cast(self.mask_placeholder * 255, dtype=tf.uint8))
pass
with tf.name_scope("result"):
# segment
for segment_index, segment in enumerate(self.segments):
segment = tf.split(segment, num_or_size_splits=2, axis=3)
# tf.summary.image("predict-{}-0".format(segment_index), tf.nn.sigmoid(segment[0]))
tf.summary.image("predict-{}-1".format(segment_index), tf.nn.sigmoid(segment[1]))
# attention
for attention_index, attention in enumerate(self.attentions):
attention = tf.split(attention, num_or_size_splits=2, axis=3)
# tf.summary.image("attention-{}-0".format(attention_index), tf.nn.sigmoid(attention[0]))
tf.summary.image("attention-{}-1".format(attention_index), tf.nn.sigmoid(attention[1]))
pass
pass
self.summary_op = tf.summary.merge_all()
# sess 和 saver
self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)))
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
# summary 2
self.summary_writer = tf.summary.FileWriter(self.log_dir, self.sess.graph)
pass
def __init__(self,
rnn_cell,
rnn_cell_dim,
method,
data_train,
logdir,
expname,
threads,
restore_path,
seed=42):
n_words = len(data_train.factors[data_train.FORMS]['words'])
n_tags = len(data_train.factors[data_train.TAGS]['words'])
n_lemmas = len(data_train.factors[data_train.LEMMAS]['words'])
n_mwe = len(data_train.factors[data_train.MWE]['words'])
# Create an empty graph and a session
graph = tf.Graph()
graph.seed = seed
self.session = tf.Session(graph=graph,
config=tf.ConfigProto(
inter_op_parallelism_threads=threads,
intra_op_parallelism_threads=threads))
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.summary_writer = tf.train.SummaryWriter("{}/{}-{}".format(
logdir, timestamp, expname),
flush_secs=10)
# Construct the graph
with self.session.graph.as_default():
if rnn_cell == "LSTM":
rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_cell_dim)
elif rnn_cell == "GRU":
rnn_cell = tf.nn.rnn_cell.GRUCell(rnn_cell_dim)
else:
raise ValueError("Unknown rnn_cell {}".format(rnn_cell))
self.global_step = tf.Variable(0,
dtype=tf.int64,
trainable=False,
name="global_step")
self.sentence_lens = tf.placeholder(tf.int32, [None])
self.forms = tf.placeholder(tf.int32, [None, None])
self.tags = tf.placeholder(tf.int32, [None, None])
self.lemmas = tf.placeholder(tf.int32, [None, None])
self.mwe = tf.placeholder(tf.int32, [None, None])
embeddings_forms = tf.get_variable("word_embedding_matrix",
[n_words, rnn_cell_dim],
dtype=tf.float32)
embeddings_lemmas = tf.get_variable("lemma_embedding_matrix",
[n_lemmas, rnn_cell_dim],
dtype=tf.float32)
embeddings_tags = tf.get_variable("tag_embedding_matrix",
[n_tags, rnn_cell_dim],
dtype=tf.float32)
embedding_in_forms = tf.nn.embedding_lookup(
embeddings_forms, self.forms)
embedding_in_lemmas = tf.nn.embedding_lookup(
embeddings_lemmas, self.lemmas)
embedding_in_tags = tf.nn.embedding_lookup(embeddings_tags,
self.tags)
#TODO concatenata tf.cvoncat emb_form emb_lemma emb_tags (zkusit udelat one-hot na tag - nebo zapomenout na to)
embedding_in = tf.concat(
2,
[embedding_in_forms, embedding_in_lemmas, embedding_in_tags
]) #mail Milanovi - je to tak???
self.input_keep_dropout = tf.placeholder_with_default(
1.0, None, name="input_keep")
self.hidden_keep_dropout = tf.placeholder_with_default(
1.0, None, name="hidden_keep")
hidden_layer = tf.nn.dropout(embedding_in, self.input_keep_dropout)
bi_out, bi_out_states = tf.nn.bidirectional_dynamic_rnn(
rnn_cell,
rnn_cell,
embedding_in,
self.sentence_lens,
dtype=tf.float32) # concatenate embedding
layer1 = tf.concat(2, bi_out)
layer = tf_layers.fully_connected(layer1,
n_mwe,
activation_fn=None,
scope="output_layer")
mask = tf.sequence_mask(self.sentence_lens)
masked_mwe = tf.boolean_mask(self.mwe, mask)
self.predictions = tf.cast(tf.argmax(layer, 2, name="predictions"),
dtype=tf.int32)
masked_predictions = tf.boolean_mask(self.predictions, mask)
loss = tf_losses.sparse_softmax_cross_entropy(tf.boolean_mask(
layer, mask),
masked_mwe,
scope="loss")
self.training = tf.train.AdamOptimizer().minimize(
loss, global_step=self.global_step)
self.accuracy = tf_metrics.accuracy(
masked_predictions, masked_mwe
) #vysledni scotre bude jiny !!! cely score na tech MWE
self.dataset_name = tf.placeholder(tf.string, [])
self.summary = tf.merge_summary([
tf.scalar_summary(self.dataset_name + "/loss", loss),
tf.scalar_summary(self.dataset_name + "/accuracy",
self.accuracy)
])
self.saver = tf.train.Saver(max_to_keep=None)
# Initialize variables
self.session.run(tf.initialize_all_variables())
if restore_path is not None:
self.saver.restore(self.session, restore_path)
