def build_model(sess, graph, loss_model): """ Builds a tensor graph model """ model = None with graph.as_default(): # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Input data. train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) global_step = tf.Variable(0, trainable=False) # Look up embeddings for inputs. embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) sm_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) # sm_weights = tf.Print(sm_weights, [sm_weights], 'smweights: ', summarize=32) # Get context embeddings from lables true_w = tf.nn.embedding_lookup(sm_weights, train_labels) true_w = tf.reshape(true_w, [-1, embedding_size]) # Construct the variables for the NCE loss nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) nce_biases = tf.Variable(tf.zeros([vocabulary_size])) if loss_model == 'cross_entropy': loss = tf.reduce_mean(tf_func.cross_entropy_loss(embed, true_w)) else: #sample negative examples with unigram probability sample = np.random.choice(vocabulary_size, num_sampled, p=unigram_prob, replace=False) loss = tf.reduce_mean( tf_func.nce_loss(embed, nce_weights, nce_biases, train_labels, sample, unigram_prob)) # tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. optimizer = tf.train.GradientDescentOptimizer(1.0).minimize( loss, global_step=global_step) # Compute the cosine similarity between minibatch examples and all embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) saver = tf.train.Saver(tf.global_variables()) # Save summary # summary = tf.summary.merge_all() # summary_writer = tf.summary.FileWriter(summary_path + '/summary', sess.graph) summary = None summary_writer = None tf.global_variables_initializer().run() print("Initialized") model = Word2Vec(train_inputs, train_labels, loss, optimizer, global_step, embeddings, normalized_embeddings, valid_embeddings, similarity, saver, summary, summary_writer) return model
predict_train = tf.argmax(tf.nn.softmax(decoder_output), axis=-1) predict_eval = tf.argmax(tf.nn.softmax(decoder_output_eval), axis=-1) ground_truth = tf.argmax(y, axis=-1) accuracy_train = tf.reduce_sum( tf.cast(tf.equal(predict_train, ground_truth), tf.float32)) accuracy_eval = tf.reduce_sum( tf.cast(tf.equal(predict_eval, ground_truth), tf.float32)) l2_reg_loss = 0 for tf_var in tf.trainable_variables(): #print(tf_var.name) if not ("bias" in tf_var.name or "output_project" in tf_var.name): l2_reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var)) loss = l.cross_entropy_loss(decoder_output, y) + l2_reg_loss loss_eval = l.cross_entropy_loss(decoder_output_eval, y) + l2_reg_loss #train_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss) train_op = tf.train.RMSPropOptimizer(1e-4, 0.9).minimize(loss) with tf.name_scope('train_summary'): tf.summary.scalar('cross_entropy_loss', loss, collections=['train']) tf.summary.scalar('accuracy', accuracy_train, collections=['train']) merged_summary_train = tf.summary.merge_all('train') with tf.name_scope('validatin_summary'): tf.summary.scalar('cross_entropy_loss', loss_eval, collections=['validatin']) tf.summary.scalar('accuracy', accuracy_eval, collections=['validatin']) merged_summary_val = tf.summary.merge_all('validatin')
def Enc_gru_cls(conf, steps, train, validation, weights=None): c['input_step'] = conf['input_step'] c['batch_size'] = conf['batch_size'] c['n_lstm_hidden_units'] = conf['n_lstm_hidden_units'] with tf.Graph().as_default(): if c['feature_size'] == None: c['feature_size'] = train.shape[-1] #x = tf.placeholder(tf.float32, [None, c['input_step'], train.shape[-1]]) x = tf.placeholder(tf.float32, [None, c['input_step'], c['feature_size']]) y = tf.placeholder(tf.float32, [None, c['predict_step'], 3]) decoder_output = mz.model_zoo(c, x, y, dropout=0.6, is_train=True).decoder_output decoder_output_eval = mz.model_zoo(c, x, y, dropout=1.0, is_train=False).decoder_output predict_train = tf.argmax(tf.nn.softmax(decoder_output), axis=-1) predict_eval = tf.argmax(tf.nn.softmax(decoder_output_eval), axis=-1) ground_truth = tf.argmax(y, axis=-1) accuracy_train = tf.reduce_mean( tf.cast(tf.equal(predict_train, ground_truth), tf.float32)) accuracy_eval = tf.reduce_mean( tf.cast(tf.equal(predict_eval, ground_truth), tf.float32)) l2_reg_loss = 0 for tf_var in tf.trainable_variables(): #print(tf_var.name) if not ("bias" in tf_var.name or "output_project" in tf_var.name): l2_reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var)) loss = l.cross_entropy_loss(decoder_output, y) loss_eval = l.cross_entropy_loss(decoder_output_eval, y) #train_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss) train_op = tf.train.RMSPropOptimizer(1e-2, 0.9).minimize(loss) Nbatch = Ndata // c['batch_size'] with tf.Session() as sess: variables = ray.experimental.TensorFlowVariables(loss, sess) sess.run(tf.global_variables_initializer()) if weights is not None: variables.set_weights(weights) for i in range(1, steps + 1): for j in range(Nbatch): train_data, train_label = get_batch_random_cls( train, c['input_step'], c['batch_size'], 0, c['feature_size']) sess.run(train_op, feed_dict={ x: train_data, y: train_label }) val_data, val_label = get_batch_random_cls(validation, c['input_step'], c['batch_size'], 0, c['feature_size']) acc = sess.run(accuracy_eval, feed_dict={ x: val_data, y: val_label }) return acc
predict_train = tf.argmax(tf.nn.softmax(decoder_output), axis=-1) predict_eval = tf.argmax(tf.nn.softmax(decoder_output_eval), axis=-1) ground_truth = tf.argmax(y, axis=-1) accuracy_train = tf.reduce_sum( tf.cast(tf.equal(predict_train, ground_truth), tf.float32)) accuracy_eval = tf.reduce_sum( tf.cast(tf.equal(predict_eval, ground_truth), tf.float32)) l2_reg_loss = 0 for tf_var in tf.trainable_variables(): #print(tf_var.name) if not ("bias" in tf_var.name or "output_project" in tf_var.name): l2_reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var)) loss = l.cross_entropy_loss(decoder_output, y) loss_eval = l.cross_entropy_loss(decoder_output_eval, y) #train_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss) train_op = tf.train.RMSPropOptimizer(1e-4, 0.9).minimize(loss) #plt.scatter(validation[0:55,:,0], validation[0:55,:,1]) batchsize = 32 data_log = [] loss_log = [] loss_eval_log = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer())
def build_model(sess, graph, loss_model): """ Builds a tensor graph model """ model = None with graph.as_default(): # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Input data. train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) global_step = tf.Variable(0, trainable=False) # Look up embeddings for inputs. embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) sm_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) # Get context embeddings from lables true_w = tf.nn.embedding_lookup(sm_weights, train_labels) true_w = tf.reshape(true_w, [-1, embedding_size]) # Construct the variables for the NCE loss nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) nce_biases = tf.Variable(tf.zeros([vocabulary_size])) if loss_model == 'cross_entropy': loss = tf.reduce_mean(tf_func.cross_entropy_loss(embed, true_w)) else: # sample negative examples with unigram probability sample = np.random.choice(vocabulary_size, num_sampled, p=unigram_prob, replace=False) # negative samples for max accuracy nce loss mode # [ 1717,3137, 6,448,56, 614,2857, 115,2799,18, 3,44,36,13,20, 1,1614,23, 932, 299,1585,3663, 422,2153, 2,5224,10,22,3320,24,1463,79,31,222,15986,3178,\ # 20188,569, 102, 5,6145,27,57, 9,2251,5545,1449, 758 # , 8,1772,47,237, 0,32, 13934, 224,29,6628,15,16 # ,4105,339,3310,597] loss = tf.reduce_mean( tf_func.nce_loss(embed, nce_weights, nce_biases, train_labels, sample, unigram_prob)) # tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. optimizer = tf.train.GradientDescentOptimizer(1.0).minimize( loss, global_step=global_step) # Compute the cosine similarity between minibatch examples and all embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) saver = tf.train.Saver(tf.global_variables()) # Save summary # summary = tf.summary.merge_all() # summary_writer = tf.summary.FileWriter(summary_path + '/summary', sess.graph) summary = None summary_writer = None tf.global_variables_initializer().run() print("Initialized") model = Word2Vec(train_inputs, train_labels, loss, optimizer, global_step, embeddings, normalized_embeddings, valid_embeddings, similarity, saver, summary, summary_writer) return model