def model_fn(features, labels, mode): # 先构建神经网络 logits = neural_network(features) # 预测结果 pred_calsses = tf.argmax(logits, axis=1) pred_probas = tf.nn.softmax(logits) # 如果是 预测模式,早早的返回 if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode, predictions=pred_calsses) # 定义损失函数和优化函数 loss_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=tf.cast( labels, dtype=tf.int32))) optimizer = tf.train.GradientDescentOptimizer(learning_rate) train_op = optimizer.minimize(loss_op, global_step=tf.train.get_global_step()) # 验证模型的acc acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_calsses) # TF Estimators 需要返回一个EstimatorSpec,从而确定是(训练,预测)哪个操作 estim_specs = tf.estimator.EstimatorSpec( mode=mode, predictions=pred_calsses, loss=loss_op, train_op=train_op, eval_metric_ops={'accuracy': acc_op}) return estim_specs
def loss(logits, labels): # 对所有的可训练参数增加 l2 loss sparse_labels = tf.reshape(labels, [FLAGS.batch_size, 1]) indices = tf.reshape(tf.range(FLAGS.batch_size), [FLAGS.batch_size, 1]) concated = tf.concat([indices, sparse_labels], 1) dense_labels = tf.sparse_to_dense(concated, [FLAGS.batch_size, NUM_CLASSES], 1.0, 0.0) # 计算均方误差 cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2( logits, dense_labels, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) return tf.add_n(tf.get_collection('losses'), name='total_loss')
def average_gradients(tower_grads): average_grads = [] for grad_and_vars in zip(*tower_grads): # 每个grad_and_vars参数都是(gpu0,梯度0),....,(gpuN,梯度N) grads = [] for g, _ in grad_and_vars: # Add 0 dimension to the gradients to represent the tower. expanded_g = tf.expand_dims(g, 0) grads.append(expanded_g) # 对所有的梯度取平均 grad = tf.concat(grads, 0) grad = tf.reduce_mean(grad, 0) # 由于参数变量在上面的list中是共享的,所以只返回第一个towers中的Variable v = grad_and_vars[0][1] grad_and_vars = (grad, v) average_grads.append(grad_and_vars) return average_grads
def model_fn(features, labels, mode): logits_train = conv_network(features, num_classes, dropout, reuse=False, is_training=True) logits_test = conv_network(features, num_classes, dropout, reuse=True, is_training=False) pred_classes = tf.argmax(logits_test, axis=1) pred_probs = tf.nn.softmax(logits_test) if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode, predictions=pred_classes) loss_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits_train, labels=tf.cast( labels, dtype=tf.int32))) optimizer = tf.train.AdamOptimizer(learning_rate) train_op = optimizer.minimize(loss_op, global_step=tf.train.get_global_step()) # 验证模型的acc acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_classes) estim_spec = tf.estimator.EstimatorSpec( mode=mode, predictions=pred_classes, loss=loss_op, train_op=train_op, eval_metric_ops={'accuracy': acc_op}) return estim_spec
num_clusters=k, distance_metric='cosine', use_mini_batch=True) # 构建 KMeans 的图 training_graph = kmeans.training_graph() if len(training_graph) > 6: (all_scores, cluster_idx, scores, cluster_centers_initialized, cluster_centers_var, init_op, train_op) = training_graph else: (all_scores, cluster_idx, scores, cluster_centers_initialized, cluster_centers_var, init_op, train_op) = training_graph cluster_idx = cluster_idx[0] avg_distance = tf.reduce_mean(scores) init_vars = tf.global_variables_initializer() sess = tf.Session() sess.run(init_vars, feed_dict={X: full_data_x}) sess.run(init_op, feed_dict={X: full_data_x}) # training for i in range(1, num_steps + 1): _, d, idx = sess.run([train_op, avg_distance, cluster_idx], feed_dict={X: full_data_x}) if i % 10 == 0 or i == 1: print("Step %i, Avg Distance: %f" % (i, d))
# 由于dropout在训练和预测阶段是不同的行为,所以需要创建两个独立的图来共享相同参数 logits_train = conv_net(_x, num_classes, dropout, reuse=reuse_vars, is_training=True) logits_test = conv_net(_x, num_classes, dropout, reuse=True, is_training=False) # 定义loss和opts,带上logits_train 以使dropout生效 loss_op = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits_train, labels=_y)) optimizer = tf.train.AdamOptimizer(learning_rate) grads = optimizer.compute_gradients(loss_op) # 只用其中一个gpu计算acc if i == 0: # Evaluate model (with test logits, for dropout to be disabled) correct_pred = tf.equal(tf.argmax(logits_test, 1), tf.argmax(_y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) reuse_vars = True tower_grads.append(grads) tower_grads = average_gradients(tower_grads) train_op = optimizer.apply_gradients(tower_grads)
layer_2 = tf.nn.sigmoid( tf.add(tf.matmul(layer_1, weights['decoder_h2']), biases['decoder_b2'])) return layer_2 # 构建图的操作 encoder_op = encoder(X) decoder_op = decoder(encoder_op) # prediction y_pred = decoder_op y_true = X loss = tf.reduce_mean(tf.pow(y_true - y_pred, 2)) optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) for i in range(1, num_steps + 1): batch_x, _ = mnist.train.next_batch(batch_size) # 运行optimizer op(反向传播) 和 loss _, l = sess.run([optimizer, loss], feed_dict={X: batch_x}) if i % display_step == 0 or i == 1: print('Step %i: Minibatch Loss: %f' % (i, l)) # Testing n = 4
X = tf.placeholder(tf.int32, shape=[None]) Y = tf.placeholder(tf.int32, shape=[None, 1]) with tf.device("/cpu:0"): embedding = tf.Variable(tf.random_normal([vocab_size, embedding_size])) X_embed = tf.nn.embedding_lookup(embedding, X) # 为NCE loss 构造变量 nce_weights = tf.Variable(tf.random_normal([vocab_size, embedding_size])) nce_biases = tf.Variable(tf.zeros([vocab_size])) # 为每个batch计算平均的nce loss loss_op = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=Y, inputs=X_embed, num_sampled=num_sampled, num_classes=vocab_size)) # 定义优化器,优化器的作用是求导,反向传播。 optimizer = tf.train.GradientDescentOptimizer(learning_rate) train_op = optimizer.minimize(loss_op) # 验证/评价 # 计算输入数据embedding 和每个embedding向量的cosine相似度 X_embed_norm = X_embed / tf.sqrt(tf.reduce_sum(tf.square(X_embed))) embedding_norm = embedding / tf.sqrt( tf.reduce_sum(tf.square(embedding), 1, keep_dims=True)) cosine_sim_op = tf.matmul(X_embed_norm, embedding_norm, transpose_b=True)
def word2vec_basic(log_dir): # 创建tensorboard的可视化目录 if not os.path.exists(log_dir): os.makedirs(log_dir) # 第一步,下载数据 url = 'http://mattmahoney.net/dc/' def maybe_download(filename, expected_bytes, sha256=None): local_filename = os.path.join(gettempdir(), filename) if not os.path.exists(local_filename): local_filename, _ = urllib.request.urlretrieve( url + filename, local_filename) statinfo = os.stat(local_filename) if sha256 and _hash_file(local_filename) != sha256: raise Exception('Failed to verify ' + local_filename + ' due to hash ' 'mismatch. Can you get to it with a browser?') if statinfo.st_size == expected_bytes: print("found and verified", filename) else: print(statinfo.st_size) raise Exception('Failed to verify ' + local_filename + '. Can you get to it with a browser?') return local_filename filename = maybe_download( 'text8.zip', 31344016, sha256= 'a6640522afe85d1963ad56c05b0ede0a0c000dddc9671758a6cc09b7a38e5232') # 数据转为List<String> def read_data(filename): with zipfile.ZipFile(filename) as f: data = tf.compat.as_str(f.read(f.namelist()[0])).split() return data vocabulary = read_data(filename) print('data_size', len(vocabulary)) # 第二步,建词典并且把罕见词替换成UNK vocabulary_size = 50000 def build_dataset(words, n_words): count = [['UNK', -1]] count.extend(collections.Counter(words).most_common(n_words - 1)) dictionary = {word: index for index, (word, _) in enumerate(count)} data = [] unk_count = 0 for word in words: index = dictionary.get(word, 0) if index == 0: # dictionary['UNK'] unk_count += 1 data.append(index) count[0][1] = unk_count reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) return data, count, dictionary, reversed_dictionary # data: 词表中的所有的词的id # count: 单词和出现次数的map # dictionary: 单词-->index 的映射 # reverse_dictionary:index -->单词 data, count, dictionary, reversed_dictionary = build_dataset( vocabulary, vocabulary_size) del vocabulary print('Most common words (+UNK)', count[:5]) print('Sample data', data[:10], [reversed_dictionary[i] for i in data[:10]]) # 针对skip-gram模型生成batch数据 def generate_batch(batch_size, num_skips, skip_window): global data_index assert batch_size % num_skips == 0 assert num_skips <= 2 * skip_window batch = np.ndarray(shape=(batch_size), dtype=np.int32) labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) # skip的范围 span = 2 * skip_window + 1 buffer = collections.deque(maxlen=span) if data_index + span > len(data): data_index = 0 buffer.extend(data[data_index:data_index + span]) # 向后取一个窗口内的结果 data_index += span for i in range(batch_size // num_skips): context_words = [w for w in range(span) if w != skip_window] words_to_use = random.sample(context_words, num_skips) for j, context_words in enumerate(words_to_use): batch[i * num_skips + j] = buffer[skip_window] labels[i * num_skips + j, 0] = buffer[context_words] if data_index == len(data): buffer.extend(data[0:span]) data_index = span else: buffer.append(data[data_index]) data_index += 1 # Backtrack a little bit to avoid skipping words in the end of a batch data_index = (data_index - span) % len(data) return batch, labels batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1) for i in range(8): print(batch[i], reversed_dictionary[batch[i]], '->', labels[i, 0], reversed_dictionary[labels[i, 0]]) # 建立并且训练模型 batch_size = 128 embedding_size = 128 # 词向量维度 skip_window = 1 # 考虑左右几个单词 num_skips = 2 # 复用输入生成标签的次数 num_sampled = 64 # 负样本数量 # 采样一个样本的近邻作为随机验证机,将验证集样本限制为 较低id的单词,是比较高频的构造词汇 # 这三个变量用作显示模型准确率,不影响计算。 valid_size = 16 # 用于评估相似性的随机单词集合 valid_window = 100 # valid_examples = np.random.choice(valid_window, valid_size, replace=False) graph = tf.Graph() with graph.as_default(): # 输入数据 with tf.name_scope('input'): train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) # 操作op和变量variables 固定在CPU上。 with tf.device('/cpu:0'): with tf.name_scope('embeddings'): embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) # 构造NCE损失的变量 with tf.name_scope('weights'): nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size])) # 计算该批次的平均nce损失,当评估损失的时候,自动绘制一个新的负样本。 with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=vocabulary_size)) # 汇总损失 tf.summary.scalar('loss', loss) # 构造SGD with tf.name_scope('opytimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # 计算小批次样本和所有样本之间的余弦相似度 norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) # merge all summary merged = tf.summary.merge_all() init = tf.global_variables_initializer() saver = tf.train.Saver() # 开始训练 num_steps = 1000001 with tf.compat.v1.Session(graph=graph) as session: # 写入摘要 writer = tf.summary.FileWriter(log_dir, session.graph) init.run() print('inited..') average_loss = 0 for step in range(num_steps): batch_inputs, batch_labels = generate_batch( batch_size, num_skips, skip_window) feed_dict = { train_inputs: batch_inputs, train_labels: batch_labels } # 定义元变量 run_metadata = tf.RunMetadata() _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val writer.add_summary(summary, step) if step == (num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 # 平均损失是对最近的2000个批次样本的估计。 print('Average loss at step ', step, ': ', average_loss) average_loss = 0 if step % 10000 == 0: sim = similarity.eval() for i in range(valid_size): valid_word = reversed_dictionary[valid_examples[i]] top_k = 8 nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word print( log_str, ', '.join([ reversed_dictionary[nearest[k]] for k in range(top_k) ])) final_embeddings = normalized_embeddings.eval() # 写下embedding的相应标签 with open(log_dir + '/metadata.tsv', 'w') as f: for i in range(vocabulary_size): f.write(reversed_dictionary[i] + '\n') # 保存checkpoint saver.save(session, os.path.join(log_dir, 'model.ckpt')) # 配置Tensorboard config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() # Step 6: Visualize the embeddings. # pylint: disable=missing-docstring # Function to draw visualization of distance between embeddings. def plot_with_labels(low_dim_embs, labels, filename): assert low_dim_embs.shape[0] >= len( labels), 'More labels than embeddings' plt.figure(figsize=(18, 18)) # in inches for i, label in enumerate(labels): x, y = low_dim_embs[i, :] plt.scatter(x, y) plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(filename) try: # pylint: disable=g-import-not-at-top from sklearn.manifold import TSNE import matplotlib.pyplot as plt tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact') plot_only = 500 low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :]) labels = [reversed_dictionary[i] for i in xrange(plot_only)] plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(), 'tsne.png')) except ImportError as ex: print( 'Please install sklearn, matplotlib, and scipy to show embeddings.' ) print(ex)
# outputs 是每个timesteps的输出列表,打包成[batch_size,n_step,n_inputs] outputs = tf.stack(outputs) outputs = tf.transpose(outputs, [1, 0, 2]) batch_size = tf.shape(outputs)[0] # 每个样本的起始索引 index = tf.range(0, batch_size) * seq_max_len + (seqlen - 1) outputs = tf.gather(tf.reshape(outputs, [-1, n_hidden]), index) return tf.matmul(outputs, weights['out']) + biases['out'] pred = dynamicRNN(x, seqlen, weights, biases) cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred, labels=y)) optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) for step in range(1, training_steps + 1): batch_x, batch_y, batch_seqlen = trainset.next(batch_size) sess.run(optimizer, feed_dict={
W = tf.Variable(tf.zeros([784,10])) b = tf.Variable(tf.zeros([10])) sess.run(tf.initialize_all_variables()) y = tf.nn.softmax(tf.matmul(x,W)+b) cross_entropy = -tf.reduce_sum(y_*tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) for i in range(1000): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x:batch[0],y_:batch[1]}) correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction,"float")) print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
def accuarcy_fn(interface_fn,inputs,labels): prediction = tf.nn.softmax(interface_fn(inputs)) correct_pred = tf.equal(tf.argmax(prediction,1),labels) # 计算预测值和标签是否相等 return tf.reduce_mean(tf.cast(correct_pred,tf.float32))
Y = tf.placeholder(tf.float32, shape=[None]) hparams = tensor_forest.ForestHParams(num_classes=num_classes, num_features=num_features, num_trees=num_trees, max_nodes=max_nodes).fill() # 构建随机森林 forgest_graph = tensor_forest.RandomForestGraphs(hparams) # 获取训练图和损失 train_op = forgest_graph.training_graph(X, Y) loss_op = forgest_graph.training_loss(X, Y) # 衡量准确率 infer_op, _, _ = forgest_graph.inference_graph(X) correct_prediction = tf.equal(tf.argmax(infer_op, 1), tf.cast(Y, tf.int64)) accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) init_vars = tf.group( tf.global_variables_initializer(), resources.initialize_resources(resources.shared_resources())) sess = tf.Session() sess.run(init_vars) for i in range(1, num_steps + 1): batch_x, batch_y = mnist.train.next_batch(batch_size) _, l = sess.run([train_op, loss_op], feed_dict={X: batch_x, Y: batch_y}) if i % 50 == 0 or i == 1: acc = sess.run(accuracy_op, feed_dict={X: batch_x, Y: batch_y}) print('Step %i, Loss: %f, Acc: %f' % (i, l, acc))
# Store layers weight & bias weights = { 'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])), 'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])), 'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes])) } biases = { 'b1': tf.Variable(tf.random_normal([n_hidden_1])), 'b2': tf.Variable(tf.random_normal([n_hidden_2])), 'out': tf.Variable(tf.random_normal([n_classes])) } pred = multilayer_perceptron(x, weights, biases) cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y)) optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) init = tf.global_variables_initializer() saver = tf.train.Saver() print("start 1st session..") with tf.Session() as sess: sess.run(init) for epoch in range(3): avg_cost = 0. total_batch = int(mnist.train.num_examples / batch_size) for i in range(total_batch):
def loss_fn(inference_fn, inputs, labels): return tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=inference_fn(inputs), labels=labels))
train_epochs = 25 batch_size = 100 display_step = 1 # 定义placeholder x = tf.placeholder(tf.float32, [None, 784]) y = tf.placeholder(tf.float32, [None, 10]) # weights bias W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) pred = tf.nn.softmax(tf.matmul(x, W) + b) # 最小化交叉熵 cost = tf.reduce_mean(-tf.reduce_sum(y * tf.log(pred), reduction_indices=1)) # 梯度下降 optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) # 初始化所有参数 init = tf.global_variables_initializer() with tf.Session() as sess: # 运行初始化 sess.run(init) for epoch in range(train_epochs): avg_cost = 0. total_batch = int(mnist.train.num_examples / batch_size) # 循环所有的batchs for i in range(total_batch):
def accuracy_fn(inference_fn, inputs, labels): prediction = tf.nn.softmax(inference_fn(inputs)) correct_pred = tf.equal(tf.argmax(prediction, 1), labels) return tf.reduce_mean(tf.cast(correct_pred, tf.float32))
display_epoch = 1 logs_path = './logs/' x = tf.placeholder(tf.float32, [None, 784], name='InputData') y = tf.placeholder(tf.float32, [None, 10], name='LabelData') W = tf.Variable(tf.zeros([784, 10]), name='Weights') b = tf.Variable(tf.zeros([10]), name='Bias') # 构造模型并将所有操作封装到scope中,方便tensorboard可视化。 with tf.name_scope('Model'): pred = tf.nn.softmax(tf.matmul(x, W) + b) with tf.name_scope('Loss'): cost = tf.reduce_mean(-tf.reduce_sum(y * tf.log(pred), reduction_indices=1)) with tf.name_scope('SGD'): optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) with tf.name_scope('Accuracy'): acc = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) acc = tf.reduce_mean(tf.cast(acc, tf.float32)) init = tf.global_variables_initializer() tf.summary.scalar("loss", cost) tf.summary.scalar("accuracy", acc) merged_summary_op = tf.summary.merge_all()