def sg_log(tensor, opt):
r"""Log transform a dense tensor
See `tf.log()` in tensorflow.
Args:
tensor: A `Tensor` ( automatically given by chain )
opt:
name: If provided, replace current tensor's name.
Returns:
A `Tensor`.
"""
return tf.log(tensor + tf.sg_eps, name=opt.name)
def ner_cost(tensor, opt):
one_hot_labels = tf.one_hot(opt.target - 1, opt.num_classes, dtype=tf.float32)
cross_entropy = one_hot_labels * tf.log(tensor)
cross_entropy = -tf.reduce_sum(cross_entropy, reduction_indices=2)
mask = tf.sign(tf.abs(opt.target))
cross_entropy *= tf.cast(mask, tf.float32)
cross_entropy = tf.reduce_sum(cross_entropy, reduction_indices=1)
length = tf.cast(tf.reduce_sum(tf.sign(opt.target), reduction_indices=1), tf.int32)
cross_entropy /= tf.cast(length, tf.float32)
out = tf.reduce_mean(cross_entropy, name='ner_cost')
# add summary
tf.sg_summary_loss(out, name=opt.name)
return out
def ner_cost(tensor, opt):
one_hot_labels = tf.one_hot(opt.target - 1, opt.num_classes, dtype=tf.float32)
cross_entropy = one_hot_labels * tf.log(tensor)
cross_entropy = -tf.reduce_sum(cross_entropy, reduction_indices=2)
mask = tf.sign(tf.reduce_max(tf.abs(one_hot_labels), reduction_indices=2))
cross_entropy *= tf.cast(mask, tf.float32)
cross_entropy = tf.reduce_sum(cross_entropy, reduction_indices=1)
length = tf.cast(tf.reduce_sum(tf.sign(opt.target), reduction_indices=1), tf.int32)
cross_entropy /= tf.cast(length, tf.float32)
out = tf.reduce_mean(cross_entropy, name='ner_cost')
# add summary
tf.sg_summary_loss(out, name=opt.name)
return out
tf.random_normal([n_hidden_units_two, n_hidden_units_three],
mean=0,
stddev=sd))
b_3 = tf.Variable(tf.random_normal([n_hidden_units_three], mean=0, stddev=sd))
h_3 = tf.nn.sigmoid(tf.matmul(h_2, W_3) + b_3)
W = tf.Variable(
tf.random_normal([n_hidden_units_three, num_classes], mean=0, stddev=sd))
b = tf.Variable(tf.random_normal([num_classes], mean=0, stddev=sd))
with tf.name_scope('out'):
y_ = tf.nn.softmax(tf.matmul(h_3, W) + b, name="out")
init = tf.global_variables_initializer()
cost_function = tf.reduce_mean(
-tf.reduce_sum(Y * tf.log(y_), reduction_indices=[1]))
#optimizer = tf.train.RMSPropOptimizer(learning_rate,decay=0.9,momentum=0.9,centered=True).minimize(cost_function)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(
cost_function)
correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
cost_history = np.empty(shape=[1], dtype=float)
acc_history = np.empty(shape=[1], dtype=float)
t_cost_history = np.empty(shape=[1], dtype=float)
t_acc_history = np.empty(shape=[1], dtype=float)
y_true, y_pred = None, None
with tf.Session() as session:
session.run(init)
def __init__(self,
x,
y,
num_batch,
vocab_size,
emb_dim,
hidden_dim,
max_ep=240,
infer_shape=(1, 1),
mode="train"):
self.num_batch = num_batch
self.emb_dim = emb_dim
self.hidden_dim = hidden_dim
self.vocab_size = vocab_size
self.max_len_infer = 512
self.max_ep = max_ep
# reuse = len([t for t in tf.global_variables() if t.name.startswith('gen')]) > 0
reuse = (mode == 'infer')
if mode == "train":
self.x = x
self.y = y
elif mode == "infer":
self.x = tf.placeholder(tf.int32, shape=infer_shape)
self.y = tf.placeholder(tf.int32, shape=infer_shape)
with tf.variable_scope("gen_embs", reuse=reuse):
self.emb_x = tf.get_variable("emb_x",
[self.vocab_size, self.emb_dim])
self.emb_y = tf.get_variable("emb_y",
[self.vocab_size, self.emb_dim])
self.X = tf.nn.embedding_lookup(self.emb_x, self.x)
self.Y = tf.nn.embedding_lookup(self.emb_y, self.y)
with tf.sg_context(name='gen', reuse=reuse):
# self.emb_x = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_x")
# self.emb_y = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_y")
# self.emb_x = tf.sg_emb(name='emb_x', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16)
# self.emb_y = tf.sg_emb(name='emb_y', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16)
# self.X = self.x.sg_lookup(emb=self.emb_x) # (8,63,16)
# self.Y = self.y.sg_lookup(emb=self.emb_y) # (8,63,16)
if mode == "train":
self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
dim=self.vocab_size,
name="lstm") # (8, 63, 68)
self.test = self.lstm_layer.sg_softmax(name="testtt")
print "mazum??"
print self.test
elif mode == "infer":
self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
dim=self.vocab_size,
last_only=True,
name="lstm")
self.log_prob = tf.log(self.lstm_layer)
# next_token: select by distribution probability, preds: select by argmax
self.multinormed = tf.multinomial(self.log_prob, 1)
self.next_token = tf.cast(
tf.reshape(tf.multinomial(self.log_prob, 1),
[1, infer_shape[0]]), tf.int32)
self.preds = self.lstm_layer.sg_argmax()
if mode == "train":
self.loss = self.lstm_layer.sg_ce(target=self.y)
self.istarget = tf.not_equal(self.y, 0).sg_float()
self.reduced_loss = (self.loss.sg_sum()) / (
self.istarget.sg_sum() + 0.0000001)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")