def vgg16(input, num_class):
x = zqtflearn.conv_2d(input, 64, 3, activation='relu', scope='conv1_1')
x = zqtflearn.conv_2d(x, 64, 3, activation='relu', scope='conv1_2')
x = zqtflearn.max_pool_2d(x, 2, strides=2, name='maxpool1')
x = zqtflearn.conv_2d(x, 128, 3, activation='relu', scope='conv2_1')
x = zqtflearn.conv_2d(x, 128, 3, activation='relu', scope='conv2_2')
x = zqtflearn.max_pool_2d(x, 2, strides=2, name='maxpool2')
x = zqtflearn.conv_2d(x, 256, 3, activation='relu', scope='conv3_1')
x = zqtflearn.conv_2d(x, 256, 3, activation='relu', scope='conv3_2')
x = zqtflearn.conv_2d(x, 256, 3, activation='relu', scope='conv3_3')
x = zqtflearn.max_pool_2d(x, 2, strides=2, name='maxpool3')
x = zqtflearn.conv_2d(x, 512, 3, activation='relu', scope='conv4_1')
x = zqtflearn.conv_2d(x, 512, 3, activation='relu', scope='conv4_2')
x = zqtflearn.conv_2d(x, 512, 3, activation='relu', scope='conv4_3')
x = zqtflearn.max_pool_2d(x, 2, strides=2, name='maxpool4')
x = zqtflearn.conv_2d(x, 512, 3, activation='relu', scope='conv5_1')
x = zqtflearn.conv_2d(x, 512, 3, activation='relu', scope='conv5_2')
x = zqtflearn.conv_2d(x, 512, 3, activation='relu', scope='conv5_3')
x = zqtflearn.max_pool_2d(x, 2, strides=2, name='maxpool5')
x = zqtflearn.fully_connected(x, 4096, activation='relu', scope='fc6')
x = zqtflearn.dropout(x, 0.5, name='dropout1')
x = zqtflearn.fully_connected(x, 4096, activation='relu', scope='fc7')
x = zqtflearn.dropout(x, 0.5, name='dropout2')
x = zqtflearn.fully_connected(x, num_class, activation='softmax', scope='fc8',
restore=False)
return x
def make_core_network(network):
dense1 = zqtflearn.fully_connected(network, 64, activation='tanh',
regularizer='L2', weight_decay=0.001, name="dense1")
dropout1 = zqtflearn.dropout(dense1, 0.8)
dense2 = zqtflearn.fully_connected(dropout1, 64, activation='tanh',
regularizer='L2', weight_decay=0.001, name="dense2")
dropout2 = zqtflearn.dropout(dense2, 0.8)
softmax = zqtflearn.fully_connected(dropout2, 10, activation='softmax', name="softmax")
return softmax
def deep_model(self, wide_inputs, n_inputs, n_nodes=[100, 50], use_dropout=False):
'''
Model - deep, i.e. two-layer fully connected network model
'''
cc_input_var = {}
cc_embed_var = {}
flat_vars = []
if self.verbose:
print ("--> deep model: %s categories, %d continuous" % (len(self.categorical_columns), n_inputs))
for cc, cc_size in self.categorical_columns.items():
cc_input_var[cc] = zqtflearn.input_data(shape=[None, 1], name="%s_in" % cc, dtype=tf.int32)
# embedding layers only work on CPU! No GPU implementation in tensorflow, yet!
cc_embed_var[cc] = zqtflearn.layers.embedding_ops.embedding(cc_input_var[cc], cc_size, 8, name="deep_%s_embed" % cc)
if self.verbose:
print (" %s_embed = %s" % (cc, cc_embed_var[cc]))
flat_vars.append(tf.squeeze(cc_embed_var[cc], squeeze_dims=[1], name="%s_squeeze" % cc))
network = tf.concat([wide_inputs] + flat_vars, 1, name="deep_concat")
for k in range(len(n_nodes)):
network = zqtflearn.fully_connected(network, n_nodes[k], activation="relu", name="deep_fc%d" % (k + 1))
if use_dropout:
network = zqtflearn.dropout(network, 0.5, name="deep_dropout%d" % (k + 1))
if self.verbose:
print ("Deep model network before output %s" % network)
network = zqtflearn.fully_connected(network, 1, activation="linear", name="deep_fc_output", bias=False)
network = tf.reshape(network, [-1, 1]) # so that accuracy is binary_accuracy
if self.verbose:
print ("Deep model network %s" % network)
return network
def deep_model(self, wide_inputs, n_inputs, n_nodes=[100, 50], use_dropout=False):
'''
Model - deep, i.e. two-layer fully connected network model
'''
cc_input_var = {}
cc_embed_var = {}
flat_vars = []
if self.verbose:
print ("--> deep model: %s categories, %d continuous" % (len(self.categorical_columns), n_inputs))
for cc, cc_size in self.categorical_columns.items():
cc_input_var[cc] = zqtflearn.input_data(shape=[None, 1], name="%s_in" % cc, dtype=tf.int32)#[？，1]
# embedding layers only work on CPU! No GPU implementation in tensorflow, yet!
cc_embed_var[cc] = zqtflearn.layers.embedding_ops.embedding(cc_input_var[cc], cc_size, 8, name="deep_%s_embed" % cc) #[?,1,embedding_size = 8]
if self.verbose:
print (" %s_embed = %s" % (cc, cc_embed_var[cc]))
flat_vars.append(tf.squeeze(cc_embed_var[cc], squeeze_dims=[1], name="%s_squeeze" % cc)) #[?,8]
network = tf.concat([wide_inputs] + flat_vars, axis = 1, name="deep_concat") #x=xigma(dim of each element in flat_vars) + wide_inputs.size(1) [?,x]
#这里是合并的步骤，合并采用的是前后拼接的方式。
#在这里是合并之后的逻辑，对于合并之后的输入共同处理。
for k in range(len(n_nodes)):#连续的两个全连接。
network = zqtflearn.fully_connected(network, n_nodes[k], activation="relu", name="deep_fc%d" % (k + 1)) #默认应该是用bais的。要不然下面为什么要写bias=False
if use_dropout:
network = zqtflearn.dropout(network, 0.5, name="deep_dropout%d" % (k + 1))
if self.verbose:
print ("Deep model network before output %s" % network)
network = zqtflearn.fully_connected(network, 1, activation="linear", name="deep_fc_output", bias=False) #[?,1]
network = tf.reshape(network, [-1, 1]) # so that accuracy is binary_accuracy added by zhengquan ,不reshape不也是[?,1]的吗?可能如果最后的输出维度是1的话，结果是[?]的尺寸
if self.verbose:
print ("Deep model network %s" % network)
return network
def test_sequencegenerator(self):
with tf.Graph().as_default():
text = "123456789101234567891012345678910123456789101234567891012345678910"
maxlen = 5
X, Y, char_idx = \
zqtflearn.data_utils.string_to_semi_redundant_sequences(text, seq_maxlen=maxlen, redun_step=3)
g = zqtflearn.input_data(shape=[None, maxlen, len(char_idx)])
g = zqtflearn.lstm(g, 32)
g = zqtflearn.dropout(g, 0.5)
g = zqtflearn.fully_connected(g, len(char_idx), activation='softmax')
g = zqtflearn.regression(g, optimizer='adam', loss='categorical_crossentropy',
learning_rate=0.1)
m = zqtflearn.SequenceGenerator(g, dictionary=char_idx,
seq_maxlen=maxlen,
clip_gradients=5.0)
m.fit(X, Y, validation_set=0.1, n_epoch=100, snapshot_epoch=False)
res = m.generate(10, temperature=.5, seq_seed="12345")
#self.assertEqual(res, "123456789101234", "SequenceGenerator test failed! Generated sequence: " + res + " expected '123456789101234'")
# Testing save method
m.save("test_seqgen.zqtflearn")
self.assertTrue(os.path.exists("test_seqgen.zqtflearn.index"))
# Testing load method
m.load("test_seqgen.zqtflearn")
res = m.generate(10, temperature=.5, seq_seed="12345")
def deep_model(self,
wide_inputs,
deep_inpus,
n_inputs,
n_nodes=[100, 50],
use_dropout=False):
'''
Model - deep, i.e. two-layer fully connected network model
'''
self.vocab, self.embedding = self.read_embedding(
embedding_path="I haven't deal with it")
self.embedding = np.array(self.embedding, dtype=np.float32)
net = zqtflearn.layers.embedding_ops.embedding(
deep_inpus,
len(self.vocab),
self.embedding.shape[1],
trainable=False,
name="deep_video_ids_embed" % cc)
network = tf.concat(
[wide_inputs, net], axis=1, name="deep_concat"
) #x=xigma(dim of each element in flat_vars) + wide_inputs.size(1) [?,x]
#这里是合并的步骤，合并采用的是前后拼接的方式。
#在这里是合并之后的逻辑，对于合并之后的输入共同处理。
for k in range(len(n_nodes)): #连续的两个全连接。
network = zqtflearn.fully_connected(
network,
n_nodes[k],
activation="relu",
name="deep_fc%d" % (k + 1)) #默认应该是用bais的。要不然下面为什么要写bias=False
if use_dropout:
network = zqtflearn.dropout(network,
0.5,
name="deep_dropout%d" % (k + 1))
if self.verbose:
print("Deep model network before output %s" % network)
network = zqtflearn.fully_connected(network,
1,
activation="linear",
name="deep_fc_output",
bias=False) #[?,1]
network = tf.reshape(
network, [-1, 1]
) # so that accuracy is binary_accuracy added by zhengquan ,不reshape不也是[?,1]的吗?可能如果最后的输出维度是1的话，结果是[?]的尺寸
if self.verbose:
print("Deep model network %s" % network)
return network
def test_core_layers(self):
X = [[0., 0.], [0., 1.], [1., 0.], [1., 1.]]
Y_nand = [[1.], [1.], [1.], [0.]]
Y_or = [[0.], [1.], [1.], [1.]]
# Graph definition
with tf.Graph().as_default():
# Building a network with 2 optimizers
g = zqtflearn.input_data(shape=[None, 2])
# Nand operator definition
g_nand = zqtflearn.fully_connected(g, 32, activation='linear')
g_nand = zqtflearn.fully_connected(g_nand, 32, activation='linear')
g_nand = zqtflearn.fully_connected(g_nand, 1, activation='sigmoid')
g_nand = zqtflearn.regression(g_nand, optimizer='sgd',
learning_rate=2.,
loss='binary_crossentropy')
# Or operator definition
g_or = zqtflearn.fully_connected(g, 32, activation='linear')
g_or = zqtflearn.fully_connected(g_or, 32, activation='linear')
g_or = zqtflearn.fully_connected(g_or, 1, activation='sigmoid')
g_or = zqtflearn.regression(g_or, optimizer='sgd',
learning_rate=2.,
loss='binary_crossentropy')
# XOR merging Nand and Or operators
g_xor = zqtflearn.merge([g_nand, g_or], mode='elemwise_mul')
# Training
m = zqtflearn.DNN(g_xor)
m.fit(X, [Y_nand, Y_or], n_epoch=400, snapshot_epoch=False)
# Testing
self.assertLess(m.predict([[0., 0.]])[0][0], 0.01)
self.assertGreater(m.predict([[0., 1.]])[0][0], 0.9)
self.assertGreater(m.predict([[1., 0.]])[0][0], 0.9)
self.assertLess(m.predict([[1., 1.]])[0][0], 0.01)
# Bulk Tests
with tf.Graph().as_default():
net = zqtflearn.input_data(shape=[None, 2])
net = zqtflearn.flatten(net)
net = zqtflearn.reshape(net, new_shape=[-1])
net = zqtflearn.activation(net, 'relu')
net = zqtflearn.dropout(net, 0.5)
net = zqtflearn.single_unit(net)
def test_sequencegenerator_words(self):
with tf.Graph().as_default():
text = ["hello","world"]*100
word_idx = {"hello": 0, "world": 1}
maxlen = 2
vec = [x for x in map(word_idx.get, text) if x is not None]
sequences = []
next_words = []
for i in range(0, len(vec) - maxlen, 3):
sequences.append(vec[i: i + maxlen])
next_words.append(vec[i + maxlen])
X = np.zeros((len(sequences), maxlen, len(word_idx)), dtype=np.bool)
Y = np.zeros((len(sequences), len(word_idx)), dtype=np.bool)
for i, seq in enumerate(sequences):
for t, idx in enumerate(seq):
X[i, t, idx] = True
Y[i, next_words[i]] = True
g = zqtflearn.input_data(shape=[None, maxlen, len(word_idx)])
g = zqtflearn.lstm(g, 32)
g = zqtflearn.dropout(g, 0.5)
g = zqtflearn.fully_connected(g, len(word_idx), activation='softmax')
g = zqtflearn.regression(g, optimizer='adam', loss='categorical_crossentropy',
learning_rate=0.1)
m = zqtflearn.SequenceGenerator(g, dictionary=word_idx,
seq_maxlen=maxlen,
clip_gradients=5.0)
m.fit(X, Y, validation_set=0.1, n_epoch=100, snapshot_epoch=False)
res = m.generate(4, temperature=.5, seq_seed=["hello","world"])
res_str = " ".join(res[-2:])
self.assertEqual(res_str, "hello world", "SequenceGenerator (word level) test failed! Generated sequence: " + res_str + " expected 'hello world'")
# Testing save method
m.save("test_seqgen_word.zqtflearn")
self.assertTrue(os.path.exists("test_seqgen_word.zqtflearn.index"))
# Testing load method
m.load("test_seqgen_word.zqtflearn")
res = m.generate(4, temperature=.5, seq_seed=["hello","world"])
res_str = " ".join(res[-2:])
self.assertEqual(res_str, "hello world", "Reloaded SequenceGenerator (word level) test failed! Generated sequence: " + res_str + " expected 'hello world'")
def deep_model(self,
wide_inputs,
deep_inputs,
n_inputs,
n_nodes=[100, 50],
use_dropout=False):
'''
Model - deep, i.e. two-layer fully connected network model
'''
vocab = pickle.load(open("video_ids.pkl", "rb"))
vocab = list(vocab)
self.vocab = {}
for idx, wd in enumerate(vocab):
self.vocab[wd] = idx
self.vocab["[UNK]"] = len(self.vocab)
if os.path.exists('self_embedding.pkl'):
self.embedding = pickle.load(open("self_embedding.pkl", "rb"))
print("making new self.embedding!!!")
else:
self.embedding = self.read_embedding(
embedding_path='videoID_vector.pkl', vocab=self.vocab)
pickle.dump(self.embedding, open("self_embedding.pkl", "wb"))
net = zqtflearn.layers.embedding_ops.embedding(
deep_inputs,
len(self.vocab),
self.embedding.shape[1],
trainable=False,
name="deep_video_ids_embed")
net = tf.squeeze(net, squeeze_dims=[1], name="video_ids_squeeze")
# net = zqtflearn.fully_connected(net, 108 , activation="relu",name="deep_fc_108" )
# net = zqtflearn.fully_connected(net, 54, activation="relu", name="deep_fc_54")
network = tf.concat(
[wide_inputs, net], axis=1, name="deep_concat"
) # x=xigma(dim of each element in flat_vars) + wide_inputs.size(1) [?,x]
print("n_nodes = ", n_nodes)
for k in range(len(n_nodes)):
network = zqtflearn.fully_connected(
network,
n_nodes[k],
activation="relu",
name="deep_fc%d" % (k + 1)) # 默认应该是用bais的。要不然下面为什么要写bias=False
if use_dropout:
network = zqtflearn.dropout(network,
0.5,
name="deep_dropout%d" % (k + 1))
if self.verbose:
print("Deep model network before output %s" % network)
network = zqtflearn.fully_connected(network,
1,
activation="linear",
name="deep_fc_output",
bias=False) # [?,1]
network = tf.reshape(
network, [-1, 1]
) # so that accuracy is binary_accuracy added by zhengquan ,不reshape不也是[?,1]的吗?可能如果最后的输出维度是1的话，结果是[?]的尺寸
if self.verbose:
print("Deep model network %s" % network)
return network
import zqtflearn
# Data loading and preprocessing
import zqtflearn.datasets.mnist as mnist
X, Y, testX, testY = mnist.load_data(one_hot=True)
# Building deep neural network
input_layer = zqtflearn.input_data(shape=[None, 784])
dense1 = zqtflearn.fully_connected(input_layer,
64,
activation='tanh',
regularizer='L2',
weight_decay=0.001)
dropout1 = zqtflearn.dropout(dense1, 0.8)
dense2 = zqtflearn.fully_connected(dropout1,
64,
activation='tanh',
regularizer='L2',
weight_decay=0.001)
dropout2 = zqtflearn.dropout(dense2, 0.8)
softmax = zqtflearn.fully_connected(dropout2, 10, activation='softmax')
# Regression using SGD with learning rate decay and Top-3 accuracy
sgd = zqtflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=1000)
top_k = zqtflearn.metrics.Top_k(3)
net = zqtflearn.regression(softmax,
optimizer=sgd,
metric=top_k,
loss='categorical_crossentropy')
mnist_data = mnist.read_data_sets(one_hot=True)
# User defined placeholders
with tf.Graph().as_default():
# Placeholders for data and labels
X = tf.placeholder(shape=(None, 784), dtype=tf.float32)
Y = tf.placeholder(shape=(None, 10), dtype=tf.float32)
net = tf.reshape(X, [-1, 28, 28, 1])
# Using TFLearn wrappers for network building
net = zqtflearn.conv_2d(net, 32, 3, activation='relu')
net = zqtflearn.max_pool_2d(net, 2)
net = zqtflearn.local_response_normalization(net)
net = zqtflearn.dropout(net, 0.8)
net = zqtflearn.conv_2d(net, 64, 3, activation='relu')
net = zqtflearn.max_pool_2d(net, 2)
net = zqtflearn.local_response_normalization(net)
net = zqtflearn.dropout(net, 0.8)
net = zqtflearn.fully_connected(net, 128, activation='tanh')
net = zqtflearn.dropout(net, 0.8)
net = zqtflearn.fully_connected(net, 256, activation='tanh')
net = zqtflearn.dropout(net, 0.8)
net = zqtflearn.fully_connected(net, 10, activation='linear')
# Defining other ops using Tensorflow
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=net, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01).minimize(loss)
from zqtflearn.data_utils import *
path = "US_Cities.txt"
if not os.path.isfile(path):
context = ssl._create_unverified_context()
moves.urllib.request.urlretrieve("https://raw.githubusercontent.com/tflearn/tflearn.github.io/master/resources/US_Cities.txt", path, context=context)
maxlen = 20
string_utf8 = open(path, "r").read().decode('utf-8')
X, Y, char_idx = \
string_to_semi_redundant_sequences(string_utf8, seq_maxlen=maxlen, redun_step=3)
g = zqtflearn.input_data(shape=[None, maxlen, len(char_idx)])
g = zqtflearn.lstm(g, 512, return_seq=True)
g = zqtflearn.dropout(g, 0.5)
g = zqtflearn.lstm(g, 512)
g = zqtflearn.dropout(g, 0.5)
g = zqtflearn.fully_connected(g, len(char_idx), activation='softmax')
g = zqtflearn.regression(g, optimizer='adam', loss='categorical_crossentropy',
learning_rate=0.001)
m = zqtflearn.SequenceGenerator(g, dictionary=char_idx,
seq_maxlen=maxlen,
clip_gradients=5.0,
checkpoint_path='model_us_cities')
for i in range(40):
seed = random_sequence_from_string(string_utf8, maxlen)
m.fit(X, Y, validation_set=0.1, batch_size=128,
n_epoch=1, run_id='us_cities')