def build_model(sess, graph, loss_model):
"""
Builds a tensor graph model
"""
model = None
with graph.as_default():
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Input data.
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
global_step = tf.Variable(0, trainable=False)
# Look up embeddings for inputs.
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0,
1.0))
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
sm_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
# sm_weights = tf.Print(sm_weights, [sm_weights], 'smweights: ', summarize=32)
# Get context embeddings from lables
true_w = tf.nn.embedding_lookup(sm_weights, train_labels)
true_w = tf.reshape(true_w, [-1, embedding_size])
# Construct the variables for the NCE loss
nce_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
if loss_model == 'cross_entropy':
loss = tf.reduce_mean(tf_func.cross_entropy_loss(embed, true_w))
else:
#sample negative examples with unigram probability
sample = np.random.choice(vocabulary_size,
num_sampled,
p=unigram_prob,
replace=False)
loss = tf.reduce_mean(
tf_func.nce_loss(embed, nce_weights, nce_biases, train_labels,
sample, unigram_prob))
# tf.summary.scalar('loss', loss)
# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(
loss, global_step=global_step)
# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
valid_dataset)
similarity = tf.matmul(valid_embeddings,
normalized_embeddings,
transpose_b=True)
saver = tf.train.Saver(tf.global_variables())
# Save summary
# summary = tf.summary.merge_all()
# summary_writer = tf.summary.FileWriter(summary_path + '/summary', sess.graph)
summary = None
summary_writer = None
tf.global_variables_initializer().run()
print("Initialized")
model = Word2Vec(train_inputs, train_labels, loss, optimizer, global_step,
embeddings, normalized_embeddings, valid_embeddings,
similarity, saver, summary, summary_writer)
return model
predict_train = tf.argmax(tf.nn.softmax(decoder_output), axis=-1)
predict_eval = tf.argmax(tf.nn.softmax(decoder_output_eval), axis=-1)
ground_truth = tf.argmax(y, axis=-1)
accuracy_train = tf.reduce_sum(
tf.cast(tf.equal(predict_train, ground_truth), tf.float32))
accuracy_eval = tf.reduce_sum(
tf.cast(tf.equal(predict_eval, ground_truth), tf.float32))
l2_reg_loss = 0
for tf_var in tf.trainable_variables():
#print(tf_var.name)
if not ("bias" in tf_var.name or "output_project" in tf_var.name):
l2_reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var))
loss = l.cross_entropy_loss(decoder_output, y) + l2_reg_loss
loss_eval = l.cross_entropy_loss(decoder_output_eval, y) + l2_reg_loss
#train_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss)
train_op = tf.train.RMSPropOptimizer(1e-4, 0.9).minimize(loss)
with tf.name_scope('train_summary'):
tf.summary.scalar('cross_entropy_loss', loss, collections=['train'])
tf.summary.scalar('accuracy', accuracy_train, collections=['train'])
merged_summary_train = tf.summary.merge_all('train')
with tf.name_scope('validatin_summary'):
tf.summary.scalar('cross_entropy_loss',
loss_eval,
collections=['validatin'])
tf.summary.scalar('accuracy', accuracy_eval, collections=['validatin'])
merged_summary_val = tf.summary.merge_all('validatin')
def Enc_gru_cls(conf, steps, train, validation, weights=None):
c['input_step'] = conf['input_step']
c['batch_size'] = conf['batch_size']
c['n_lstm_hidden_units'] = conf['n_lstm_hidden_units']
with tf.Graph().as_default():
if c['feature_size'] == None: c['feature_size'] = train.shape[-1]
#x = tf.placeholder(tf.float32, [None, c['input_step'], train.shape[-1]])
x = tf.placeholder(tf.float32,
[None, c['input_step'], c['feature_size']])
y = tf.placeholder(tf.float32, [None, c['predict_step'], 3])
decoder_output = mz.model_zoo(c, x, y, dropout=0.6,
is_train=True).decoder_output
decoder_output_eval = mz.model_zoo(c,
x,
y,
dropout=1.0,
is_train=False).decoder_output
predict_train = tf.argmax(tf.nn.softmax(decoder_output), axis=-1)
predict_eval = tf.argmax(tf.nn.softmax(decoder_output_eval), axis=-1)
ground_truth = tf.argmax(y, axis=-1)
accuracy_train = tf.reduce_mean(
tf.cast(tf.equal(predict_train, ground_truth), tf.float32))
accuracy_eval = tf.reduce_mean(
tf.cast(tf.equal(predict_eval, ground_truth), tf.float32))
l2_reg_loss = 0
for tf_var in tf.trainable_variables():
#print(tf_var.name)
if not ("bias" in tf_var.name or "output_project" in tf_var.name):
l2_reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var))
loss = l.cross_entropy_loss(decoder_output, y)
loss_eval = l.cross_entropy_loss(decoder_output_eval, y)
#train_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss)
train_op = tf.train.RMSPropOptimizer(1e-2, 0.9).minimize(loss)
Nbatch = Ndata // c['batch_size']
with tf.Session() as sess:
variables = ray.experimental.TensorFlowVariables(loss, sess)
sess.run(tf.global_variables_initializer())
if weights is not None:
variables.set_weights(weights)
for i in range(1, steps + 1):
for j in range(Nbatch):
train_data, train_label = get_batch_random_cls(
train, c['input_step'], c['batch_size'], 0,
c['feature_size'])
sess.run(train_op,
feed_dict={
x: train_data,
y: train_label
})
val_data, val_label = get_batch_random_cls(validation,
c['input_step'],
c['batch_size'], 0,
c['feature_size'])
acc = sess.run(accuracy_eval,
feed_dict={
x: val_data,
y: val_label
})
return acc
predict_train = tf.argmax(tf.nn.softmax(decoder_output), axis=-1)
predict_eval = tf.argmax(tf.nn.softmax(decoder_output_eval), axis=-1)
ground_truth = tf.argmax(y, axis=-1)
accuracy_train = tf.reduce_sum(
tf.cast(tf.equal(predict_train, ground_truth), tf.float32))
accuracy_eval = tf.reduce_sum(
tf.cast(tf.equal(predict_eval, ground_truth), tf.float32))
l2_reg_loss = 0
for tf_var in tf.trainable_variables():
#print(tf_var.name)
if not ("bias" in tf_var.name or "output_project" in tf_var.name):
l2_reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var))
loss = l.cross_entropy_loss(decoder_output, y)
loss_eval = l.cross_entropy_loss(decoder_output_eval, y)
#train_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss)
train_op = tf.train.RMSPropOptimizer(1e-4, 0.9).minimize(loss)
#plt.scatter(validation[0:55,:,0], validation[0:55,:,1])
batchsize = 32
data_log = []
loss_log = []
loss_eval_log = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
def build_model(sess, graph, loss_model):
"""
Builds a tensor graph model
"""
model = None
with graph.as_default():
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Input data.
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
global_step = tf.Variable(0, trainable=False)
# Look up embeddings for inputs.
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0,
1.0))
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
sm_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
# Get context embeddings from lables
true_w = tf.nn.embedding_lookup(sm_weights, train_labels)
true_w = tf.reshape(true_w, [-1, embedding_size])
# Construct the variables for the NCE loss
nce_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
if loss_model == 'cross_entropy':
loss = tf.reduce_mean(tf_func.cross_entropy_loss(embed, true_w))
else:
# sample negative examples with unigram probability
sample = np.random.choice(vocabulary_size,
num_sampled,
p=unigram_prob,
replace=False)
# negative samples for max accuracy nce loss mode
# [ 1717,3137, 6,448,56, 614,2857, 115,2799,18, 3,44,36,13,20, 1,1614,23, 932, 299,1585,3663, 422,2153, 2,5224,10,22,3320,24,1463,79,31,222,15986,3178,\
# 20188,569, 102, 5,6145,27,57, 9,2251,5545,1449, 758
# , 8,1772,47,237, 0,32, 13934, 224,29,6628,15,16
# ,4105,339,3310,597]
loss = tf.reduce_mean(
tf_func.nce_loss(embed, nce_weights, nce_biases, train_labels,
sample, unigram_prob))
# tf.summary.scalar('loss', loss)
# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(
loss, global_step=global_step)
# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
valid_dataset)
similarity = tf.matmul(valid_embeddings,
normalized_embeddings,
transpose_b=True)
saver = tf.train.Saver(tf.global_variables())
# Save summary
# summary = tf.summary.merge_all()
# summary_writer = tf.summary.FileWriter(summary_path + '/summary', sess.graph)
summary = None
summary_writer = None
tf.global_variables_initializer().run()
print("Initialized")
model = Word2Vec(train_inputs, train_labels, loss, optimizer, global_step,
embeddings, normalized_embeddings, valid_embeddings,
similarity, saver, summary, summary_writer)
return model
