def main(argv=None):
d = ImageDataset()
d.npy_maker(globalconf.get_root() + 'transfer/small')
d.combine(globalconf.get_root() + 'transfer/small', 5, 0.1, 0.1, 0.5)
print(d.training_images.shape, d.training_labels.shape,
d.validation_images.shape, d.validation_labels.shape,
d.test_images.shape, d.test_labels.shape)
def train():
train_file = globalconf.get_root(
) + "rnn/ptb/simple-examples/data/id.train.txt"
model_path = globalconf.get_root() + "rnn/ptb/model/model.ckpt"
train_id_list = file_to_id_list(train_file)
batch_size = 20
timestep = 35
hidden_size = 300
vocab_size = 10000
data = make_batch(train_id_list, batch_size, timestep)
model = Model(batch_size, timestep, hidden_size, vocab_size, 0.9)
writer = tf.summary.FileWriter(logdir=globalconf.get_root() +
"rnn/ptb/log",
graph=tf.get_default_graph())
writer.close()
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
for epoch in range(10):
print("epoch ", epoch)
model.train_epoch(data, sess)
saver = tf.train.Saver()
saver.save(sess, save_path=model_path)
def valid():
valid_file = globalconf.get_root(
) + "rnn/ptb/simple-examples/data/id.valid.txt"
model_path = globalconf.get_root() + "rnn/ptb/model/model.ckpt"
valid_id_list = file_to_id_list(valid_file)
batch_size = 20
timestep = 35
hidden_size = 300
vocab_size = 10000
valid_data = make_batch(valid_id_list, batch_size, timestep)
model = Model(batch_size, timestep, hidden_size, vocab_size, 1.0)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
saver.restore(sess, model_path)
model.validate(valid_data, sess)
self.trg_path: ""
})
saver.restore(sess, model_path)
zh = sess.run(self.dec_ids,
feed_dict={
self.src_seq: [input],
self.src_size: [input_len],
self.keep_prob: 1.0
})
output_zh_sentense = ''.join(
[zh_id_to_word.get(w, "?") for w in zh])
print(zh)
print(output_zh_sentense)
if __name__ == "__main__":
final_en_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.en.final"
final_zh_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.zh.final"
en_vocab_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.en.vocab"
zh_vocab_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.zh.vocab"
model = Model(100, 0, 1, 10000, 10000, 1024, 2, 100)
model_path = globalconf.get_root() + "rnn/ted/seq2seq.model"
model.train(model_path, final_en_file, final_zh_file)
# model_path = globalconf.get_root() + "rnn/ted/seq2seq.model-140"
# model.eval(model_path, en_vocab_file, zh_vocab_file)
with tf.variable_scope("layer5", initializer=initializer):
w = tf.get_variable("w", shape=[128, 10], dtype=tf.float32)
b = tf.get_variable("b", shape=[10], dtype=tf.float32)
logits = tf.matmul(a, w) + b
with tf.variable_scope("optimize", initializer=initializer):
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
logits=logits))
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logits, axis=1, output_type=tf.int32), y),
tf.float32))
train_op = tf.train.AdamOptimizer(0.001).minimize(loss)
log_dir = globalconf.get_root() + "one_cnn/straight_foeward"
writer = tf.summary.FileWriter(logdir=log_dir, graph=tf.get_default_graph())
writer.close()
with tf.Session() as s:
tf.global_variables_initializer().run()
for epoch in range(10):
start = 0
batch_size = 64
while start < x_train.shape[0]:
end = min(x_train.shape[0], start + batch_size)
lo, acc, _ = s.run([loss, accuracy, train_op],
feed_dict={
x: x_train[start:end],
y: y_train[start:end]
})
while start < x_valid.shape[0]:
end = min(start + batch_size, x_valid.shape[0])
sum += sess.run(correct_predict,
feed_dict={
self.x: x_valid[start:end],
self.y: y_valid[start:end]
})
start = end
return sum / x_valid.shape[0]
conv_layers = [[[5, 5], [16], [2, 2]], [[5, 5], [32], [2, 2]]]
dense_layers = [512, 128, 10]
from keras.datasets import cifar10
import numpy as np
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.reshape([-1, 32, 32, 3]) / 255.0
x_test = x_test.reshape([-1, 32, 32, 3]) / 255.0
y_train = y_train.reshape(-1).astype(np.int32)
y_test = y_test.reshape(-1).astype(np.int32)
a = Model(32, 32, 3, conv_layers, dense_layers)
import globalconf
model_dir = globalconf.get_root() + "/one_cnn/model/"
model_name = "model.ckpt"
with tf.Session() as s:
tf.global_variables_initializer().run()
a.train(s, x_train, y_train, 64, 100, model_dir + model_name)
with tf.Session() as s:
tf.global_variables_initializer().run()
print(a.eval(s, x_test, y_test, 128, model_dir + model_name))
import sys
sys.path.append("..")
import globalconf
import tensorflow as tf
# a = tf.constant([1.0,2.0,3.0], name="a")
# b = tf.Variable(tf.random_normal([3]), name="b")
with tf.variable_scope("input1"):
a = tf.constant([1.0, 2.0, 3.0], name="a")
with tf.variable_scope("input2"):
b = tf.Variable(tf.random_normal([3]), name="b")
c = tf.add_n([a, b], name="c")
writer = tf.summary.FileWriter(globalconf.get_root() + "tensorboard/demo1",
tf.get_default_graph())
writer.close()
# -*- coding: utf-8 -*-
import sys
sys.path.append("..")
import globalconf
import tensorflow as tf
import numpy as np
from keras.datasets import mnist
batch_size = 64
training_epochs = 10
log_savepath = globalconf.get_root() + "tensorboard/monitor/log"
def add_variable_to_monitor(var, name):
with tf.variable_scope("summaries"):
tf.summary.histogram(name=name, values=var)
mean = tf.reduce_mean(var)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar("mean/" + name, mean)
tf.summary.scalar("stddev/" + name, stddev)
def dense(input_tensor, input_dim, output_dim, layer_name, act):
with tf.variable_scope(layer_name):
with tf.variable_scope("weights"):
weights = tf.get_variable(
"weights",
shape=[input_dim, output_dim],
initializer=tf.truncated_normal_initializer(stddev=0.1,
dtype=tf.float32))
add_variable_to_monitor(weights, layer_name + "/weights")
with tf.variable_scope("biases"):
record,
features={
"image": tf.FixedLenFeature([], tf.string),
"label": tf.FixedLenFeature([], tf.int64)
})
image = tf.reshape(tf.decode_raw(features["image"], tf.uint8),
[self.height, self.width, self.channel])
image = tf.image.convert_image_dtype(image, tf.float32)
label = features["label"]
return image, label
return parser
if __name__ == "__main__1":
src_root = globalconf.get_root() + "transfer/small/"
dst_root = globalconf.get_root() + "transfer/tfrecord/"
flower_id_map = {
"daisy": 0,
"dandelion": 1,
"roses": 2,
"sunflowers": 3,
"tulips": 4
}
# res_a = tf.reshape(tf.decode_raw(a.tostring(),tf.uint8), [h,w,c])
builder = ImageTFRecordBuilder(299, 299, 3, 128, src_root, dst_root,
flower_id_map, 0.1, 0.1)
builder.make_tfrecord()
if __name__ == "__main__":
dst_root = globalconf.get_root() + "transfer/tfrecord/"
X_test = X_test.astype(np.float32) / 255.0
Y_train = Y_train.astype(np.int32)
Y_test = Y_test.astype(np.int32)
def dataset_to_sprite(images):
if isinstance(images, list):
images = np.array(images)
h = images.shape[1]
w = images.shape[2]
m = int(np.ceil(np.sqrt(images.shape[0])))
sprite = np.ones(shape=[h * m, w * m], dtype=np.float32)
for i in range(m):
for j in range(m):
index = i * m + j
if index < images.shape[0]:
sprite[i * h:(i + 1) * h, j * w:(j + 1) * w] = images[index]
return sprite
sprite_image = dataset_to_sprite(1.0 - X_test)
sprite_path = globalconf.get_root() + "tensorboard/visual/sprite.png"
meta_path = globalconf.get_root() + "tensorboard/visual/meta.tsv"
plt.imsave(sprite_path, sprite_image, cmap='gray')
# plt.imshow(sprite_image, cmap='gray')
# plt.show()
with open(meta_path, 'w') as f:
f.write("Index\tLabel\n")
for index in range(Y_test.shape[0]):
f.write("%d\t%d\n" % (index, Y_test[index]))
return word_id_dict
def word_to_id(in_file, out_file, vocab):
unk_id = vocab['<unk>']
with codecs.open(in_file, mode='r', encoding='utf8') as f_in, codecs.open(out_file, mode='w', encoding='utf8') as f_out:
while True:
line = f_in.readline()
if not line: break
words = line.strip().split(' ') + ['<eos>']
ids = [vocab.get(w, unk_id) for w in words]
f_out.write(' '.join(ids) + "\n")
# http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
in_train_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/ptb.train.txt"
in_valid_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/ptb.valid.txt"
in_test_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/ptb.test.txt"
out_train_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/id.train.txt"
out_valid_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/id.valid.txt"
out_test_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/id.test.txt"
out_vacab_file = globalconf.get_root() + "rnn/ptb/simple-examples/data/vocab.txt"
vocab = gen_vocab(in_train_file)
with codecs.open(out_vacab_file, mode='w', encoding='utf8') as f_vocab:
for w, id in vocab.items():
f_vocab.write(str(id) + "\t" + w + "\n")
word_to_id(in_train_file, out_train_file, vocab)
word_to_id(in_valid_file, out_valid_file, vocab)
word_to_id(in_test_file, out_test_file, vocab)
import sys
sys.path.append("..")
sys.path.append("../mnist-demo/")
import globalconf
import tensorflow as tf
import numpy as np
import mnist_infer
import os
from keras.datasets import mnist
batch_size = 64
learning_rate_base = 0.8
learning_rate_decay = 0.99
regularization_rate = 0.0001
training_epochs = 10
moving_average_decay = 0.99
model_savepath = globalconf.get_root() + "tensorboard/mnist/model"
log_savepath = globalconf.get_root() + "tensorboard/mnist/log"
model_savepath_prv = globalconf.get_root() + "tensorboard/mnistprv/model"
log_savepath_prv = globalconf.get_root() + "tensorboard/mnistprv/log"
model_name = "mnist.ckpt"
def train(x_train, y_train):
x = tf.placeholder(tf.float32,
shape=(None, mnist_infer.N[0]),
name="input-x")
y = tf.placeholder(tf.float32,
shape=(None, mnist_infer.N[-1]),
name="input-y")
regularizer = tf.contrib.layers.l2_regularizer(regularization_rate)
z = mnist_infer.infer(x, regularizer)
import globalconf
from inception_resnet_v2 import inception_resnet_v2_arg_scope, inception_resnet_v2
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
import scipy.misc as misc
import PIL.Image as Image
# http://download.tensorflow.org/models/inception_resnet_v2_2016_08_30.tar.gz
checkpoint_path = globalconf.get_root(
) + "transfer/resnet_v2/inception_resnet_v2_2016_08_30.ckpt"
logdir = globalconf.get_root() + "transfer/resnet_v2/logdir"
height, width, channels = 299, 299, 3
x = tf.placeholder(tf.float32, shape=[None, height, width, channels])
arg_scope = inception_resnet_v2_arg_scope()
for k, v in arg_scope.items():
print(k, ":")
for k2, v2 in v.items():
print("\t", k2, ":", v2)
with slim.arg_scope(arg_scope):
logits, end_points = inception_resnet_v2(x,
is_training=False,
num_classes=1001)
features = end_points['PreLogitsFlatten']
sess = tf.Session()
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
#writer = tf.summary.FileWriter(logdir=logdir, graph=tf.get_default_graph())
#writer.close()
import globalconf
import PIL.Image as Image
import os
def image_resize(src, dst):
img = Image.open(src).convert('RGB').resize((299, 299), Image.ANTIALIAS)
img.save(dst)
def image_dir_resize(src, dst):
if os.path.isfile(src) and src.endswith(".jpg"):
Image.open(src).convert('RGB').resize((299, 299),
Image.ANTIALIAS).save(dst)
if os.path.isdir(src):
if not os.path.exists(dst): os.mkdir(dst)
for f in os.listdir(src):
image_dir_resize(os.path.join(src, f), os.path.join(dst, f))
if __name__ == "__main__":
src = globalconf.get_root() + "transfer/flower_photos"
dst = globalconf.get_root() + "transfer/small"
image_dir_resize(src, dst)
import globalconf
import mnist_infer
import numpy as np
import tensorflow as tf
from keras.datasets import mnist
from tensorflow.contrib.tensorboard.plugins import projector
REGULATIZATION_RATE = 0.0001
EMA_DECAY = 0.99
learning_rate_base = 0.1
learning_rate_decay = 0.99
batch_size = 64
TRAINING_STEP = 10000
decay_steps = 1000
log_dir = globalconf.get_root() + "tensorboard/project/"
sprite_image = globalconf.get_root() + "tensorboard/visual/sprite.png"
meta_path = globalconf.get_root() + "tensorboard/visual/meta.tsv"
def train_definer():
with tf.variable_scope("input"):
x = tf.placeholder(dtype=tf.float32, shape=[None, 784], name="input-x")
y = tf.placeholder(dtype=tf.int64, shape=[None], name="input-y")
regularizer = tf.contrib.layers.l2_regularizer(REGULATIZATION_RATE)
logits = mnist_infer.infer(x, regularizer)
global_step = tf.get_variable("global_step",
shape=[],
initializer=tf.zeros_initializer(),
trainable=False)
#global_step = tf.Variable(0, trainable=False)
import sys
sys.path.append("../../")
import globalconf
import numpy as np
import tensorflow as tf
from keras.datasets import mnist
tf.logging.set_verbosity(tf.logging.INFO)
root = globalconf.get_root()
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
X_train = X_train.reshape((X_train.shape[0], -1)).astype(np.float32) / 255.0
X_test = X_test.reshape((X_test.shape[0], -1)).astype(np.float32) / 255.0
Y_train = Y_train.astype(np.int32)
Y_test = Y_test.astype(np.int32)
image_shape = X_train.shape[1:]
print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape, image_shape)
model_dir = root + "estimator/mnist"
feature_columns = [
tf.feature_column.numeric_column("image", shape=image_shape)
]
estimator = tf.estimator.DNNClassifier(hidden_units=[500],
feature_columns=feature_columns,
model_dir=model_dir,
n_classes=10,
optimizer=tf.train.AdamOptimizer())
train_fn = tf.estimator.inputs.numpy_input_fn(x={"image": X_train},
y=Y_train,
batch_size=64,
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, model_path)
start = 0
while start < x_valid.shape[0]:
end = min(start + batch_size, x_valid.shape[0])
acc = sess.run(self.accuracy,
feed_dict={
self.images: x_valid[start:end],
self.labels: y_valid[start:end]
})
print("valid:", acc)
start = end
in_ckpt_path = globalconf.get_root(
) + "transfer/inception_v3/inception_v3.ckpt"
out_ckpt_path = globalconf.get_root(
) + "transfer/inception_v3/tuned_inception_v3.ckpt"
logdir = globalconf.get_root() + "transfer/inception_v3/logdir"
m = Model(5)
data = ImageDataset()
data.combine(globalconf.get_root() + 'transfer/small', 5, 0.1, 0.1, 0.5)
with tf.Session() as s:
m.train(s, data.training_images, data.training_labels, 16, 10,
in_ckpt_path, out_ckpt_path, logdir)
with tf.Session() as s:
m.eval(s, data.validation_images, data.validation_labels, 64,
out_ckpt_path)
sess = tf.Session()
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
],
feed_dict={self.tfrecord_paths: ''})
saver = tf.train.Saver()
saver.restore(sess, model_path)
def predictor(images):
return sess.run(self.predict, feed_dict={self.images: images})
return predictor
in_ckpt_path = globalconf.get_root(
) + "transfer/inception_v3/inception_v3.ckpt"
out_ckpt_path = globalconf.get_root(
) + "transfer/inception_v3/tuned_inception_v3.ckpt"
logdir = globalconf.get_root() + "transfer/inception_v3/logdir"
tfrecord_root = globalconf.get_root() + "transfer/tfrecord/"
m = Model(5)
# with tf.Session() as s:
# m.train(s, in_ckpt_path, out_ckpt_path, logdir,tfrecord_root+"train-*")
# with tf.Session() as s:
# m.eval(s, out_ckpt_path, tfrecord_root+"valid-*")
# exit()
predictor = m.get_predictor(out_ckpt_path)
import globalconf
import codecs
from nltk.tokenize import word_tokenize
import jieba
# https://wit3.fbk.eu/mt.php?release=2015-01
in_en_file = globalconf.get_root() + "rnn/ted/en-zh/train.tags.en-zh.en"
in_zh_file = globalconf.get_root() + "rnn/ted/en-zh/train.tags.en-zh.zh"
token_en_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.en.token"
token_zh_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.zh.token"
vocab_en_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.en.vocab"
vocab_zh_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.zh.vocab"
id_en_file = globalconf.get_root() + "rnn/ted/en-zh/train.tags.en-zh.en.id"
id_zh_file = globalconf.get_root() + "rnn/ted/en-zh/train.tags.en-zh.zh.id"
final_en_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.en.final"
final_zh_file = globalconf.get_root(
) + "rnn/ted/en-zh/train.tags.en-zh.zh.final"
def tokenize():
stop_word = [' ']
with codecs.open(in_en_file, mode='r', encoding='utf8') as f_in,\
