def model_fn(features, labels, mode):
# 先构建神经网络
logits = neural_network(features)
# 预测结果
pred_calsses = tf.argmax(logits, axis=1)
pred_probas = tf.nn.softmax(logits)
# 如果是 预测模式，早早的返回
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode, predictions=pred_calsses)
# 定义损失函数和优化函数
loss_op = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=tf.cast(
labels,
dtype=tf.int32)))
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss_op,
global_step=tf.train.get_global_step())
# 验证模型的acc
acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_calsses)
# TF Estimators 需要返回一个EstimatorSpec，从而确定是(训练，预测)哪个操作
estim_specs = tf.estimator.EstimatorSpec(
mode=mode,
predictions=pred_calsses,
loss=loss_op,
train_op=train_op,
eval_metric_ops={'accuracy': acc_op})
return estim_specs
def loss(logits, labels):
# 对所有的可训练参数增加 l2 loss
sparse_labels = tf.reshape(labels, [FLAGS.batch_size, 1])
indices = tf.reshape(tf.range(FLAGS.batch_size), [FLAGS.batch_size, 1])
concated = tf.concat([indices, sparse_labels], 1)
dense_labels = tf.sparse_to_dense(concated,
[FLAGS.batch_size, NUM_CLASSES], 1.0,
0.0)
# 计算均方误差
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(
logits, dense_labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def average_gradients(tower_grads):
average_grads = []
for grad_and_vars in zip(*tower_grads):
# 每个grad_and_vars参数都是(gpu0,梯度0)，....,(gpuN,梯度N)
grads = []
for g, _ in grad_and_vars:
# Add 0 dimension to the gradients to represent the tower.
expanded_g = tf.expand_dims(g, 0)
grads.append(expanded_g)
# 对所有的梯度取平均
grad = tf.concat(grads, 0)
grad = tf.reduce_mean(grad, 0)
# 由于参数变量在上面的list中是共享的，所以只返回第一个towers中的Variable
v = grad_and_vars[0][1]
grad_and_vars = (grad, v)
average_grads.append(grad_and_vars)
return average_grads
def model_fn(features, labels, mode):
logits_train = conv_network(features,
num_classes,
dropout,
reuse=False,
is_training=True)
logits_test = conv_network(features,
num_classes,
dropout,
reuse=True,
is_training=False)
pred_classes = tf.argmax(logits_test, axis=1)
pred_probs = tf.nn.softmax(logits_test)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode, predictions=pred_classes)
loss_op = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits_train,
labels=tf.cast(
labels,
dtype=tf.int32)))
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.minimize(loss_op,
global_step=tf.train.get_global_step())
# 验证模型的acc
acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_classes)
estim_spec = tf.estimator.EstimatorSpec(
mode=mode,
predictions=pred_classes,
loss=loss_op,
train_op=train_op,
eval_metric_ops={'accuracy': acc_op})
return estim_spec
num_clusters=k,
distance_metric='cosine',
use_mini_batch=True)
# 构建 KMeans 的图
training_graph = kmeans.training_graph()
if len(training_graph) > 6:
(all_scores, cluster_idx, scores, cluster_centers_initialized,
cluster_centers_var, init_op, train_op) = training_graph
else:
(all_scores, cluster_idx, scores, cluster_centers_initialized,
cluster_centers_var, init_op, train_op) = training_graph
cluster_idx = cluster_idx[0]
avg_distance = tf.reduce_mean(scores)
init_vars = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init_vars, feed_dict={X: full_data_x})
sess.run(init_op, feed_dict={X: full_data_x})
# training
for i in range(1, num_steps + 1):
_, d, idx = sess.run([train_op, avg_distance, cluster_idx],
feed_dict={X: full_data_x})
if i % 10 == 0 or i == 1:
print("Step %i, Avg Distance: %f" % (i, d))
# 由于dropout在训练和预测阶段是不同的行为，所以需要创建两个独立的图来共享相同参数
logits_train = conv_net(_x,
num_classes,
dropout,
reuse=reuse_vars,
is_training=True)
logits_test = conv_net(_x,
num_classes,
dropout,
reuse=True,
is_training=False)
# 定义loss和opts，带上logits_train 以使dropout生效
loss_op = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits_train,
labels=_y))
optimizer = tf.train.AdamOptimizer(learning_rate)
grads = optimizer.compute_gradients(loss_op)
# 只用其中一个gpu计算acc
if i == 0:
# Evaluate model (with test logits, for dropout to be disabled)
correct_pred = tf.equal(tf.argmax(logits_test, 1),
tf.argmax(_y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
reuse_vars = True
tower_grads.append(grads)
tower_grads = average_gradients(tower_grads)
train_op = optimizer.apply_gradients(tower_grads)
layer_2 = tf.nn.sigmoid(
tf.add(tf.matmul(layer_1, weights['decoder_h2']),
biases['decoder_b2']))
return layer_2
# 构建图的操作
encoder_op = encoder(X)
decoder_op = decoder(encoder_op)
# prediction
y_pred = decoder_op
y_true = X
loss = tf.reduce_mean(tf.pow(y_true - y_pred, 2))
optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for i in range(1, num_steps + 1):
batch_x, _ = mnist.train.next_batch(batch_size)
# 运行optimizer op(反向传播) 和 loss
_, l = sess.run([optimizer, loss], feed_dict={X: batch_x})
if i % display_step == 0 or i == 1:
print('Step %i: Minibatch Loss: %f' % (i, l))
# Testing
n = 4
X = tf.placeholder(tf.int32, shape=[None])
Y = tf.placeholder(tf.int32, shape=[None, 1])
with tf.device("/cpu:0"):
embedding = tf.Variable(tf.random_normal([vocab_size, embedding_size]))
X_embed = tf.nn.embedding_lookup(embedding, X)
# 为NCE loss 构造变量
nce_weights = tf.Variable(tf.random_normal([vocab_size, embedding_size]))
nce_biases = tf.Variable(tf.zeros([vocab_size]))
# 为每个batch计算平均的nce loss
loss_op = tf.reduce_mean(
tf.nn.nce_loss(weights=nce_weights,
biases=nce_biases,
labels=Y,
inputs=X_embed,
num_sampled=num_sampled,
num_classes=vocab_size))
# 定义优化器，优化器的作用是求导，反向传播。
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss_op)
# 验证/评价
# 计算输入数据embedding 和每个embedding向量的cosine相似度
X_embed_norm = X_embed / tf.sqrt(tf.reduce_sum(tf.square(X_embed)))
embedding_norm = embedding / tf.sqrt(
tf.reduce_sum(tf.square(embedding), 1, keep_dims=True))
cosine_sim_op = tf.matmul(X_embed_norm, embedding_norm, transpose_b=True)
def word2vec_basic(log_dir):
# 创建tensorboard的可视化目录
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# 第一步，下载数据
url = 'http://mattmahoney.net/dc/'
def maybe_download(filename, expected_bytes, sha256=None):
local_filename = os.path.join(gettempdir(), filename)
if not os.path.exists(local_filename):
local_filename, _ = urllib.request.urlretrieve(
url + filename, local_filename)
statinfo = os.stat(local_filename)
if sha256 and _hash_file(local_filename) != sha256:
raise Exception('Failed to verify ' + local_filename +
' due to hash '
'mismatch. Can you get to it with a browser?')
if statinfo.st_size == expected_bytes:
print("found and verified", filename)
else:
print(statinfo.st_size)
raise Exception('Failed to verify ' + local_filename +
'. Can you get to it with a browser?')
return local_filename
filename = maybe_download(
'text8.zip',
31344016,
sha256=
'a6640522afe85d1963ad56c05b0ede0a0c000dddc9671758a6cc09b7a38e5232')
# 数据转为List<String>
def read_data(filename):
with zipfile.ZipFile(filename) as f:
data = tf.compat.as_str(f.read(f.namelist()[0])).split()
return data
vocabulary = read_data(filename)
print('data_size', len(vocabulary))
# 第二步，建词典并且把罕见词替换成UNK
vocabulary_size = 50000
def build_dataset(words, n_words):
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = {word: index for index, (word, _) in enumerate(count)}
data = []
unk_count = 0
for word in words:
index = dictionary.get(word, 0)
if index == 0: # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary
# data: 词表中的所有的词的id
# count： 单词和出现次数的map
# dictionary： 单词-->index 的映射
# reverse_dictionary:index -->单词
data, count, dictionary, reversed_dictionary = build_dataset(
vocabulary, vocabulary_size)
del vocabulary
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10],
[reversed_dictionary[i] for i in data[:10]])
# 针对skip-gram模型生成batch数据
def generate_batch(batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
# skip的范围
span = 2 * skip_window + 1
buffer = collections.deque(maxlen=span)
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span]) # 向后取一个窗口内的结果
data_index += span
for i in range(batch_size // num_skips):
context_words = [w for w in range(span) if w != skip_window]
words_to_use = random.sample(context_words, num_skips)
for j, context_words in enumerate(words_to_use):
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[context_words]
if data_index == len(data):
buffer.extend(data[0:span])
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index - span) % len(data)
return batch, labels
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
print(batch[i], reversed_dictionary[batch[i]], '->', labels[i, 0],
reversed_dictionary[labels[i, 0]])
# 建立并且训练模型
batch_size = 128
embedding_size = 128 # 词向量维度
skip_window = 1 # 考虑左右几个单词
num_skips = 2 # 复用输入生成标签的次数
num_sampled = 64 # 负样本数量
# 采样一个样本的近邻作为随机验证机，将验证集样本限制为 较低id的单词，是比较高频的构造词汇
# 这三个变量用作显示模型准确率，不影响计算。
valid_size = 16 # 用于评估相似性的随机单词集合
valid_window = 100 #
valid_examples = np.random.choice(valid_window, valid_size, replace=False)
graph = tf.Graph()
with graph.as_default():
# 输入数据
with tf.name_scope('input'):
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# 操作op和变量variables 固定在CPU上。
with tf.device('/cpu:0'):
with tf.name_scope('embeddings'):
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0,
1.0))
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# 构造NCE损失的变量
with tf.name_scope('weights'):
nce_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 /
math.sqrt(embedding_size)))
with tf.name_scope('biases'):
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
# 计算该批次的平均nce损失，当评估损失的时候，自动绘制一个新的负样本。
with tf.name_scope('loss'):
loss = tf.reduce_mean(
tf.nn.nce_loss(weights=nce_weights,
biases=nce_biases,
labels=train_labels,
inputs=embed,
num_sampled=num_sampled,
num_classes=vocabulary_size))
# 汇总损失
tf.summary.scalar('loss', loss)
# 构造SGD
with tf.name_scope('opytimizer'):
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# 计算小批次样本和所有样本之间的余弦相似度
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
valid_dataset)
similarity = tf.matmul(valid_embeddings,
normalized_embeddings,
transpose_b=True)
# merge all summary
merged = tf.summary.merge_all()
init = tf.global_variables_initializer()
saver = tf.train.Saver()
# 开始训练
num_steps = 1000001
with tf.compat.v1.Session(graph=graph) as session:
# 写入摘要
writer = tf.summary.FileWriter(log_dir, session.graph)
init.run()
print('inited..')
average_loss = 0
for step in range(num_steps):
batch_inputs, batch_labels = generate_batch(
batch_size, num_skips, skip_window)
feed_dict = {
train_inputs: batch_inputs,
train_labels: batch_labels
}
# 定义元变量
run_metadata = tf.RunMetadata()
_, summary, loss_val = session.run([optimizer, merged, loss],
feed_dict=feed_dict,
run_metadata=run_metadata)
average_loss += loss_val
writer.add_summary(summary, step)
if step == (num_steps - 1):
writer.add_run_metadata(run_metadata, 'step%d' % step)
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# 平均损失是对最近的2000个批次样本的估计。
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
if step % 10000 == 0:
sim = similarity.eval()
for i in range(valid_size):
valid_word = reversed_dictionary[valid_examples[i]]
top_k = 8
nearest = (-sim[i, :]).argsort()[1:top_k + 1]
log_str = 'Nearest to %s:' % valid_word
print(
log_str, ', '.join([
reversed_dictionary[nearest[k]]
for k in range(top_k)
]))
final_embeddings = normalized_embeddings.eval()
# 写下embedding的相应标签
with open(log_dir + '/metadata.tsv', 'w') as f:
for i in range(vocabulary_size):
f.write(reversed_dictionary[i] + '\n')
# 保存checkpoint
saver.save(session, os.path.join(log_dir, 'model.ckpt'))
# 配置Tensorboard
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = embeddings.name
embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv')
projector.visualize_embeddings(writer, config)
writer.close()
# Step 6: Visualize the embeddings.
# pylint: disable=missing-docstring
# Function to draw visualization of distance between embeddings.
def plot_with_labels(low_dim_embs, labels, filename):
assert low_dim_embs.shape[0] >= len(
labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(label,
xy=(x, y),
xytext=(5, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
tsne = TSNE(perplexity=30,
n_components=2,
init='pca',
n_iter=5000,
method='exact')
plot_only = 500
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [reversed_dictionary[i] for i in xrange(plot_only)]
plot_with_labels(low_dim_embs, labels,
os.path.join(gettempdir(), 'tsne.png'))
except ImportError as ex:
print(
'Please install sklearn, matplotlib, and scipy to show embeddings.'
)
print(ex)
# outputs 是每个timesteps的输出列表，打包成[batch_size,n_step,n_inputs]
outputs = tf.stack(outputs)
outputs = tf.transpose(outputs, [1, 0, 2])
batch_size = tf.shape(outputs)[0]
# 每个样本的起始索引
index = tf.range(0, batch_size) * seq_max_len + (seqlen - 1)
outputs = tf.gather(tf.reshape(outputs, [-1, n_hidden]), index)
return tf.matmul(outputs, weights['out']) + biases['out']
pred = dynamicRNN(x, seqlen, weights, biases)
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for step in range(1, training_steps + 1):
batch_x, batch_y, batch_seqlen = trainset.next(batch_size)
sess.run(optimizer,
feed_dict={
W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.initialize_all_variables())
y = tf.nn.softmax(tf.matmul(x,W)+b)
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
for i in range(1000):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x:batch[0],y_:batch[1]})
correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction,"float"))
print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
def accuarcy_fn(interface_fn,inputs,labels):
prediction = tf.nn.softmax(interface_fn(inputs))
correct_pred = tf.equal(tf.argmax(prediction,1),labels) # 计算预测值和标签是否相等
return tf.reduce_mean(tf.cast(correct_pred,tf.float32))
Y = tf.placeholder(tf.float32, shape=[None])
hparams = tensor_forest.ForestHParams(num_classes=num_classes,
num_features=num_features,
num_trees=num_trees,
max_nodes=max_nodes).fill()
# 构建随机森林
forgest_graph = tensor_forest.RandomForestGraphs(hparams)
# 获取训练图和损失
train_op = forgest_graph.training_graph(X, Y)
loss_op = forgest_graph.training_loss(X, Y)
# 衡量准确率
infer_op, _, _ = forgest_graph.inference_graph(X)
correct_prediction = tf.equal(tf.argmax(infer_op, 1), tf.cast(Y, tf.int64))
accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init_vars = tf.group(
tf.global_variables_initializer(),
resources.initialize_resources(resources.shared_resources()))
sess = tf.Session()
sess.run(init_vars)
for i in range(1, num_steps + 1):
batch_x, batch_y = mnist.train.next_batch(batch_size)
_, l = sess.run([train_op, loss_op], feed_dict={X: batch_x, Y: batch_y})
if i % 50 == 0 or i == 1:
acc = sess.run(accuracy_op, feed_dict={X: batch_x, Y: batch_y})
print('Step %i, Loss: %f, Acc: %f' % (i, l, acc))
# Store layers weight & bias
weights = {
'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1])),
'b2': tf.Variable(tf.random_normal([n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
pred = multilayer_perceptron(x, weights, biases)
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
print("start 1st session..")
with tf.Session() as sess:
sess.run(init)
for epoch in range(3):
avg_cost = 0.
total_batch = int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
def loss_fn(inference_fn, inputs, labels):
return tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=inference_fn(inputs), labels=labels))
train_epochs = 25
batch_size = 100
display_step = 1
# 定义placeholder
x = tf.placeholder(tf.float32, [None, 784])
y = tf.placeholder(tf.float32, [None, 10])
# weights bias
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
pred = tf.nn.softmax(tf.matmul(x, W) + b)
# 最小化交叉熵
cost = tf.reduce_mean(-tf.reduce_sum(y * tf.log(pred), reduction_indices=1))
# 梯度下降
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
# 初始化所有参数
init = tf.global_variables_initializer()
with tf.Session() as sess:
# 运行初始化
sess.run(init)
for epoch in range(train_epochs):
avg_cost = 0.
total_batch = int(mnist.train.num_examples / batch_size)
# 循环所有的batchs
for i in range(total_batch):
def accuracy_fn(inference_fn, inputs, labels):
prediction = tf.nn.softmax(inference_fn(inputs))
correct_pred = tf.equal(tf.argmax(prediction, 1), labels)
return tf.reduce_mean(tf.cast(correct_pred, tf.float32))
display_epoch = 1
logs_path = './logs/'
x = tf.placeholder(tf.float32, [None, 784], name='InputData')
y = tf.placeholder(tf.float32, [None, 10], name='LabelData')
W = tf.Variable(tf.zeros([784, 10]), name='Weights')
b = tf.Variable(tf.zeros([10]), name='Bias')
# 构造模型并将所有操作封装到scope中，方便tensorboard可视化。
with tf.name_scope('Model'):
pred = tf.nn.softmax(tf.matmul(x, W) + b)
with tf.name_scope('Loss'):
cost = tf.reduce_mean(-tf.reduce_sum(y *
tf.log(pred), reduction_indices=1))
with tf.name_scope('SGD'):
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
with tf.name_scope('Accuracy'):
acc = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
acc = tf.reduce_mean(tf.cast(acc, tf.float32))
init = tf.global_variables_initializer()
tf.summary.scalar("loss", cost)
tf.summary.scalar("accuracy", acc)
merged_summary_op = tf.summary.merge_all()
