def test_dnn(self): with tf.Graph().as_default(): X = [3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,7.042,10.791,5.313,7.997,5.654,9.27,3.1] Y = [1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,2.827,3.465,1.65,2.904,2.42,2.94,1.3] input = zqtflearn.input_data(shape=[None]) linear = zqtflearn.single_unit(input) regression = zqtflearn.regression(linear, optimizer='sgd', loss='mean_square', metric='R2', learning_rate=0.01) m = zqtflearn.DNN(regression) # Testing fit and predict m.fit(X, Y, n_epoch=1000, show_metric=True, snapshot_epoch=False) res = m.predict([3.2])[0] self.assertGreater(res, 1.3, "DNN test (linear regression) failed! with score: " + str(res) + " expected > 1.3") self.assertLess(res, 1.8, "DNN test (linear regression) failed! with score: " + str(res) + " expected < 1.8") # Testing save method m.save("test_dnn.zqtflearn") self.assertTrue(os.path.exists("test_dnn.zqtflearn.index")) with tf.Graph().as_default(): input = zqtflearn.input_data(shape=[None]) linear = zqtflearn.single_unit(input) regression = zqtflearn.regression(linear, optimizer='sgd', loss='mean_square', metric='R2', learning_rate=0.01) m = zqtflearn.DNN(regression) # Testing load method m.load("test_dnn.zqtflearn") res = m.predict([3.2])[0] self.assertGreater(res, 1.3, "DNN test (linear regression) failed after loading model! score: " + str(res) + " expected > 1.3") self.assertLess(res, 1.8, "DNN test (linear regression) failed after loading model! score: " + str(res) + " expected < 1.8")
def test_regression_placeholder(self): ''' Check that regression does not duplicate placeholders ''' with tf.Graph().as_default(): g = zqtflearn.input_data(shape=[None, 2]) g_nand = zqtflearn.fully_connected(g, 1, activation='linear') with tf.name_scope("Y"): Y_in = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="Y") zqtflearn.regression(g_nand, optimizer='sgd', placeholder=Y_in, learning_rate=2., loss='binary_crossentropy', op_name="regression1", name="Y") # for this test, just use the same default trainable_vars # in practice, this should be different for the two regressions zqtflearn.regression(g_nand, optimizer='adam', placeholder=Y_in, learning_rate=2., loss='binary_crossentropy', op_name="regression2", name="Y") self.assertEqual(len(tf.get_collection(tf.GraphKeys.TARGETS)), 1)
def test_sequencegenerator(self): with tf.Graph().as_default(): text = "123456789101234567891012345678910123456789101234567891012345678910" maxlen = 5 X, Y, char_idx = \ zqtflearn.data_utils.string_to_semi_redundant_sequences(text, seq_maxlen=maxlen, redun_step=3) g = zqtflearn.input_data(shape=[None, maxlen, len(char_idx)]) g = zqtflearn.lstm(g, 32) g = zqtflearn.dropout(g, 0.5) g = zqtflearn.fully_connected(g, len(char_idx), activation='softmax') g = zqtflearn.regression(g, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.1) m = zqtflearn.SequenceGenerator(g, dictionary=char_idx, seq_maxlen=maxlen, clip_gradients=5.0) m.fit(X, Y, validation_set=0.1, n_epoch=100, snapshot_epoch=False) res = m.generate(10, temperature=.5, seq_seed="12345") #self.assertEqual(res, "123456789101234", "SequenceGenerator test failed! Generated sequence: " + res + " expected '123456789101234'") # Testing save method m.save("test_seqgen.zqtflearn") self.assertTrue(os.path.exists("test_seqgen.zqtflearn.index")) # Testing load method m.load("test_seqgen.zqtflearn") res = m.generate(10, temperature=.5, seq_seed="12345")
def test_conv_layers(self): X = [[0., 0., 0., 0.], [1., 1., 1., 1.], [0., 0., 1., 0.], [1., 1., 1., 0.]] Y = [[1., 0.], [0., 1.], [1., 0.], [0., 1.]] with tf.Graph().as_default(): g = zqtflearn.input_data(shape=[None, 4]) g = zqtflearn.reshape(g, new_shape=[-1, 2, 2, 1]) g = zqtflearn.conv_2d(g, 4, 2, activation='relu') g = zqtflearn.max_pool_2d(g, 2) g = zqtflearn.fully_connected(g, 2, activation='softmax') g = zqtflearn.regression(g, optimizer='sgd', learning_rate=1.) m = zqtflearn.DNN(g) m.fit(X, Y, n_epoch=100, snapshot_epoch=False) # TODO: Fix test #self.assertGreater(m.predict([[1., 0., 0., 0.]])[0][0], 0.5) # Bulk Tests with tf.Graph().as_default(): g = zqtflearn.input_data(shape=[None, 4]) g = zqtflearn.reshape(g, new_shape=[-1, 2, 2, 1]) g = zqtflearn.conv_2d(g, 4, 2) g = zqtflearn.conv_2d(g, 4, 1) g = zqtflearn.conv_2d_transpose(g, 4, 2, [2, 2]) g = zqtflearn.max_pool_2d(g, 2)
def test_core_layers(self): X = [[0., 0.], [0., 1.], [1., 0.], [1., 1.]] Y_nand = [[1.], [1.], [1.], [0.]] Y_or = [[0.], [1.], [1.], [1.]] # Graph definition with tf.Graph().as_default(): # Building a network with 2 optimizers g = zqtflearn.input_data(shape=[None, 2]) # Nand operator definition g_nand = zqtflearn.fully_connected(g, 32, activation='linear') g_nand = zqtflearn.fully_connected(g_nand, 32, activation='linear') g_nand = zqtflearn.fully_connected(g_nand, 1, activation='sigmoid') g_nand = zqtflearn.regression(g_nand, optimizer='sgd', learning_rate=2., loss='binary_crossentropy') # Or operator definition g_or = zqtflearn.fully_connected(g, 32, activation='linear') g_or = zqtflearn.fully_connected(g_or, 32, activation='linear') g_or = zqtflearn.fully_connected(g_or, 1, activation='sigmoid') g_or = zqtflearn.regression(g_or, optimizer='sgd', learning_rate=2., loss='binary_crossentropy') # XOR merging Nand and Or operators g_xor = zqtflearn.merge([g_nand, g_or], mode='elemwise_mul') # Training m = zqtflearn.DNN(g_xor) m.fit(X, [Y_nand, Y_or], n_epoch=400, snapshot_epoch=False) # Testing self.assertLess(m.predict([[0., 0.]])[0][0], 0.01) self.assertGreater(m.predict([[0., 1.]])[0][0], 0.9) self.assertGreater(m.predict([[1., 0.]])[0][0], 0.9) self.assertLess(m.predict([[1., 1.]])[0][0], 0.01) # Bulk Tests with tf.Graph().as_default(): net = zqtflearn.input_data(shape=[None, 2]) net = zqtflearn.flatten(net) net = zqtflearn.reshape(net, new_shape=[-1]) net = zqtflearn.activation(net, 'relu') net = zqtflearn.dropout(net, 0.5) net = zqtflearn.single_unit(net)
def __init__(self): # Building deep neural network network = zqtflearn.input_data(shape=[None, 784], name="input") network = self.make_core_network(network) # Regression using SGD with learning rate decay and Top-3 accuracy sgd = zqtflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=1000) top_k = zqtflearn.metrics.Top_k(3) network = zqtflearn.regression(network, optimizer=sgd, metric=top_k, loss='categorical_crossentropy', name="target") model = zqtflearn.DNN(network, tensorboard_verbose=0) self.model = model
def test_recurrent_layers(self): X = [[1, 3, 5, 7], [2, 4, 8, 10], [1, 5, 9, 11], [2, 6, 8, 0]] Y = [[0., 1.], [1., 0.], [0., 1.], [1., 0.]] with tf.Graph().as_default(): g = zqtflearn.input_data(shape=[None, 4]) g = zqtflearn.embedding(g, input_dim=12, output_dim=4) g = zqtflearn.lstm(g, 6) g = zqtflearn.fully_connected(g, 2, activation='softmax') g = zqtflearn.regression(g, optimizer='sgd', learning_rate=1.) m = zqtflearn.DNN(g) m.fit(X, Y, n_epoch=300, snapshot_epoch=False) self.assertGreater(m.predict([[5, 9, 11, 1]])[0][1], 0.9)
def test_sequencegenerator_words(self): with tf.Graph().as_default(): text = ["hello","world"]*100 word_idx = {"hello": 0, "world": 1} maxlen = 2 vec = [x for x in map(word_idx.get, text) if x is not None] sequences = [] next_words = [] for i in range(0, len(vec) - maxlen, 3): sequences.append(vec[i: i + maxlen]) next_words.append(vec[i + maxlen]) X = np.zeros((len(sequences), maxlen, len(word_idx)), dtype=np.bool) Y = np.zeros((len(sequences), len(word_idx)), dtype=np.bool) for i, seq in enumerate(sequences): for t, idx in enumerate(seq): X[i, t, idx] = True Y[i, next_words[i]] = True g = zqtflearn.input_data(shape=[None, maxlen, len(word_idx)]) g = zqtflearn.lstm(g, 32) g = zqtflearn.dropout(g, 0.5) g = zqtflearn.fully_connected(g, len(word_idx), activation='softmax') g = zqtflearn.regression(g, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.1) m = zqtflearn.SequenceGenerator(g, dictionary=word_idx, seq_maxlen=maxlen, clip_gradients=5.0) m.fit(X, Y, validation_set=0.1, n_epoch=100, snapshot_epoch=False) res = m.generate(4, temperature=.5, seq_seed=["hello","world"]) res_str = " ".join(res[-2:]) self.assertEqual(res_str, "hello world", "SequenceGenerator (word level) test failed! Generated sequence: " + res_str + " expected 'hello world'") # Testing save method m.save("test_seqgen_word.zqtflearn") self.assertTrue(os.path.exists("test_seqgen_word.zqtflearn.index")) # Testing load method m.load("test_seqgen_word.zqtflearn") res = m.generate(4, temperature=.5, seq_seed=["hello","world"]) res_str = " ".join(res[-2:]) self.assertEqual(res_str, "hello world", "Reloaded SequenceGenerator (word level) test failed! Generated sequence: " + res_str + " expected 'hello world'")
def test_feed_dict_no_None(self): X = [[0., 0., 0., 0.], [1., 1., 1., 1.], [0., 0., 1., 0.], [1., 1., 1., 0.]] Y = [[1., 0.], [0., 1.], [1., 0.], [0., 1.]] with tf.Graph().as_default(): g = zqtflearn.input_data(shape=[None, 4], name="X_in") g = zqtflearn.reshape(g, new_shape=[-1, 2, 2, 1]) g = zqtflearn.conv_2d(g, 4, 2) g = zqtflearn.conv_2d(g, 4, 1) g = zqtflearn.max_pool_2d(g, 2) g = zqtflearn.fully_connected(g, 2, activation='softmax') g = zqtflearn.regression(g, optimizer='sgd', learning_rate=1.) m = zqtflearn.DNN(g) def do_fit(): m.fit({"X_in": X, 'non_existent': X}, Y, n_epoch=30, snapshot_epoch=False) self.assertRaisesRegexp(Exception, "Feed dict asks for variable named 'non_existent' but no such variable is known to exist", do_fit)
def build_simple_model(self): """Build a simple model for test Returns: DNN, [ (input layer name, input placeholder, input data) ], Target data """ inputPlaceholder1, inputPlaceholder2 = \ tf.placeholder(tf.float32, (1, 1), name = "input1"), tf.placeholder(tf.float32, (1, 1), name = "input2") input1 = zqtflearn.input_data(placeholder=inputPlaceholder1) input2 = zqtflearn.input_data(placeholder=inputPlaceholder2) network = zqtflearn.merge([input1, input2], "sum") network = zqtflearn.reshape(network, (1, 1)) network = zqtflearn.fully_connected(network, 1) network = zqtflearn.regression(network) return ( zqtflearn.DNN(network), [("input1:0", inputPlaceholder1, self.INPUT_DATA_1), ("input2:0", inputPlaceholder2, self.INPUT_DATA_2)], self.TARGET, )
def build_model(self, learning_rate=[0.001, 0.01]): ''' Model - wide and deep - built using zqtflearn ''' n_cc = len(self.continuous_columns) n_cc = 108 input_shape = [None, n_cc] if self.verbose: print("=" * 77 + " Model %s (type=%s)" % (self.name, self.model_type)) print(" Input placeholder shape=%s" % str(input_shape)) wide_inputs = zqtflearn.input_data(shape=input_shape, name="wide_X") deep_inputs = zqtflearn.input_data(shape=[None, 1], name="deep_X") if not isinstance(learning_rate, list): learning_rate = [learning_rate, learning_rate] # wide, deep if self.verbose: print(" Learning rates (wide, deep)=%s" % learning_rate) with tf.name_scope( "Y"): # placeholder for target variable (i.e. trainY input) Y_in = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="Y") with tf.variable_op_scope([wide_inputs], None, "cb_unit", reuse=False) as scope: central_bias = zqtflearn.variables.variable( 'central_bias', shape=[1], initializer=tf.constant_initializer(np.random.randn()), trainable=True, restore=True) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/cb_unit', central_bias) if 'wide' in self.model_type: wide_network = self.wide_model(wide_inputs, n_cc) network = wide_network wide_network_with_bias = tf.add(wide_network, central_bias, name="wide_with_bias") if 'deep' in self.model_type: deep_network = self.deep_model( wide_inputs, deep_inputs, n_cc ) # 这里面是wide inputs,在这个函数内部wide_inputs,会和deep_model制造的输入相合并。 deep_network_with_bias = tf.add(deep_network, central_bias, name="deep_with_bias") if 'wide' in self.model_type: network = tf.add(wide_network, deep_network) if self.verbose: print("Wide + deep model network %s" % network) else: network = deep_network network = tf.add(network, central_bias, name="add_central_bias") # add validation monitor summaries giving confusion matrix entries with tf.name_scope('Monitors'): predictions = tf.cast(tf.greater(network, 0), tf.int64) print("predictions=%s" % predictions) Ybool = tf.cast(Y_in, tf.bool) print("Ybool=%s" % Ybool) pos = tf.boolean_mask(predictions, Ybool) neg = tf.boolean_mask(predictions, ~Ybool) psize = tf.cast(tf.shape(pos)[0], tf.int64) nsize = tf.cast(tf.shape(neg)[0], tf.int64) true_positive = tf.reduce_sum(pos, name="true_positive") false_negative = tf.subtract(psize, true_positive, name="false_negative") false_positive = tf.reduce_sum(neg, name="false_positive") true_negative = tf.subtract(nsize, false_positive, name="true_negative") overall_accuracy = tf.truediv(tf.add(true_positive, true_negative), tf.add(nsize, psize), name="overall_accuracy") vmset = [ true_positive, true_negative, false_positive, false_negative, overall_accuracy ] trainable_vars = tf.trainable_variables() tv_deep = [v for v in trainable_vars if v.name.startswith('deep_')] tv_wide = [v for v in trainable_vars if v.name.startswith('wide_')] if self.verbose: print("DEEP trainable_vars") for v in tv_deep: print(" Variable %s: %s" % (v.name, v)) print("WIDE trainable_vars") for v in tv_wide: print(" Variable %s: %s" % (v.name, v)) # if 'wide' in self.model_type: # if not 'deep' in self.model_type: # tv_wide.append(central_bias) # zqtflearn.regression(wide_network_with_bias, # placeholder=Y_in, # optimizer='sgd', # loss='roc_auc_score', # #loss='binary_crossentropy', # metric="accuracy", # learning_rate=learning_rate[0], # validation_monitors=vmset, # trainable_vars=tv_wide, # op_name="wide_regression", # name="Y") # # if 'deep' in self.model_type: # if not 'wide' in self.model_type: # tv_wide.append(central_bias) # zqtflearn.regression(deep_network_with_bias, # placeholder=Y_in, # optimizer='adam', # loss='roc_auc_score', # #loss='binary_crossentropy', # metric="accuracy", # learning_rate=learning_rate[1], # validation_monitors=vmset if not 'wide' in self.model_type else None, # trainable_vars=tv_deep, # op_name="deep_regression", # name="Y") if self.model_type == 'wide+deep': # learn central bias separately for wide+deep zqtflearn.regression( network, placeholder=Y_in, optimizer='adam', #loss="roc_auc_score", loss='binary_crossentropy', metric="accuracy", validation_monitors=vmset, learning_rate=learning_rate[0], # use wide learning rate #trainable_vars=[central_bias], #[tv_deep,tv_wide,central_bias] # None op_name="central_bias_regression", name="Y") self.model = zqtflearn.DNN( network, tensorboard_verbose=self.tensorboard_verbose, max_checkpoints=self.max_checkpoints, checkpoint_path="%s/%s.tfl" % (self.checkpoints_dir, self.name), tensorboard_dir=self.tensorboard_dir) # tensorboard_dir="/tmp/tflearn_logs/" zqtflearn.DNN 我把他改为当前目录下的了,这样也比较好规范 if 'deep' in self.model_type: embeddingWeights = zqtflearn.get_layer_variables_by_name( 'deep_video_ids_embed')[0] # CUSTOM_WEIGHT = pickle.load("Haven't deal") # emb = np.array(CUSTOM_WEIGHT, dtype=np.float32) # emb = self.embedding new_emb_t = tf.convert_to_tensor(self.embedding) self.model.set_weights(embeddingWeights, new_emb_t) if self.verbose: print("Target variables:") for v in tf.get_collection(tf.GraphKeys.TARGETS): print(" variable %s: %s" % (v.name, v)) print("=" * 77) print("model build finish")
scope='decoder_h') decoder = zqtflearn.fully_connected(decoder, original_dim, activation='sigmoid', scope='decoder_out') # Define VAE Loss def vae_loss(x_reconstructed, x_true): # Reconstruction loss encode_decode_loss = x_true * tf.log(1e-10 + x_reconstructed) \ + (1 - x_true) * tf.log(1e-10 + 1 - x_reconstructed) encode_decode_loss = -tf.reduce_sum(encode_decode_loss, 1) # KL Divergence loss kl_div_loss = 1 + z_std - tf.square(z_mean) - tf.exp(z_std) kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1) return tf.reduce_mean(encode_decode_loss + kl_div_loss) net = zqtflearn.regression(decoder, optimizer='rmsprop', learning_rate=0.001, loss=vae_loss, metric=None, name='target_images') # We will need 2 models, one for training that will learn the latent # representation, and one that can take random normal noise as input and # use the decoder part of the network to generate an image # Train the VAE training_model = zqtflearn.DNN(net, tensorboard_verbose=0) training_model.fit({'input_images': X}, {'target_images': X}, n_epoch=100, validation_set=(testX, testX), batch_size=256, run_id="vae") # Build an image generator (re-using the decoding layers) # Input data is a normal (gaussian) random distribution (with dim = latent_dim) input_noise = zqtflearn.input_data(shape=[None, latent_dim], name='input_noise') decoder = zqtflearn.fully_connected(input_noise, hidden_dim, activation='relu', scope='decoder_h', reuse=True)
def build_model(self, learning_rate=[0.001, 0.01]): ''' Model - wide and deep - built using zqtflearn ''' n_cc = len(self.continuous_columns) n_categories = 1 # two categories: is_idv and is_not_idv input_shape = [None, n_cc] if self.verbose: print ("="*77 + " Model %s (type=%s)" % (self.name, self.model_type)) print (" Input placeholder shape=%s" % str(input_shape)) wide_inputs = zqtflearn.input_data(shape=input_shape, name="wide_X") if not isinstance(learning_rate, list): learning_rate = [learning_rate, learning_rate] # wide, deep if self.verbose: print (" Learning rates (wide, deep)=%s" % learning_rate) with tf.name_scope("Y"): # placeholder for target variable (i.e. trainY input) Y_in = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="Y") with tf.variable_scope(None, "cb_unit", [wide_inputs]) as scope: central_bias = zqtflearn.variables.variable('central_bias', shape=[1], initializer=tf.constant_initializer(np.random.randn()), trainable=True, restore=True) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/cb_unit', central_bias) if 'wide' in self.model_type: wide_network = self.wide_model(wide_inputs, n_cc) network = wide_network wide_network_with_bias = tf.add(wide_network, central_bias, name="wide_with_bias") if 'deep' in self.model_type: deep_network = self.deep_model(wide_inputs, n_cc) deep_network_with_bias = tf.add(deep_network, central_bias, name="deep_with_bias") if 'wide' in self.model_type: network = tf.add(wide_network, deep_network) if self.verbose: print ("Wide + deep model network %s" % network) else: network = deep_network network = tf.add(network, central_bias, name="add_central_bias") # add validation monitor summaries giving confusion matrix entries with tf.name_scope('Monitors'): predictions = tf.cast(tf.greater(network, 0), tf.int64) print ("predictions=%s" % predictions) Ybool = tf.cast(Y_in, tf.bool) print ("Ybool=%s" % Ybool) pos = tf.boolean_mask(predictions, Ybool) neg = tf.boolean_mask(predictions, ~Ybool) psize = tf.cast(tf.shape(pos)[0], tf.int64) nsize = tf.cast(tf.shape(neg)[0], tf.int64) true_positive = tf.reduce_sum(pos, name="true_positive") false_negative = tf.subtract(psize, true_positive, name="false_negative") false_positive = tf.reduce_sum(neg, name="false_positive") true_negative = tf.subtract(nsize, false_positive, name="true_negative") overall_accuracy = tf.truediv(tf.add(true_positive, true_negative), tf.add(nsize, psize), name="overall_accuracy") vmset = [true_positive, true_negative, false_positive, false_negative, overall_accuracy] trainable_vars = tf.trainable_variables() tv_deep = [v for v in trainable_vars if v.name.startswith('deep_')] tv_wide = [v for v in trainable_vars if v.name.startswith('wide_')] if self.verbose: print ("DEEP trainable_vars") for v in tv_deep: print (" Variable %s: %s" % (v.name, v)) print ("WIDE trainable_vars") for v in tv_wide: print (" Variable %s: %s" % (v.name, v)) if 'wide' in self.model_type: if not 'deep' in self.model_type: tv_wide.append(central_bias) zqtflearn.regression(wide_network_with_bias, placeholder=Y_in, optimizer='sgd', #loss='roc_auc_score', loss='binary_crossentropy', metric="accuracy", learning_rate=learning_rate[0], validation_monitors=vmset, trainable_vars=tv_wide, op_name="wide_regression", name="Y") if 'deep' in self.model_type: if not 'wide' in self.model_type: tv_wide.append(central_bias) zqtflearn.regression(deep_network_with_bias, placeholder=Y_in, optimizer='adam', #loss='roc_auc_score', loss='binary_crossentropy', metric="accuracy", learning_rate=learning_rate[1], validation_monitors=vmset if not 'wide' in self.model_type else None, trainable_vars=tv_deep, op_name="deep_regression", name="Y") if self.model_type=='wide+deep': # learn central bias separately for wide+deep zqtflearn.regression(network, placeholder=Y_in, optimizer='adam', loss='binary_crossentropy', metric="accuracy", learning_rate=learning_rate[0], # use wide learning rate trainable_vars=[central_bias], op_name="central_bias_regression", name="Y") self.model = zqtflearn.DNN(network, tensorboard_verbose=self.tensorboard_verbose, max_checkpoints=5, checkpoint_path="%s/%s.tfl" % (self.checkpoints_dir, self.name), ) if self.verbose: print ("Target variables:") for v in tf.get_collection(tf.GraphKeys.TARGETS): print (" variable %s: %s" % (v.name, v)) print ("="*77)
def test_dnn_loading_scope(self): with tf.Graph().as_default(): X = [ 3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167, 7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1 ] Y = [ 1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221, 2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3 ] input = zqtflearn.input_data(shape=[None]) linear = zqtflearn.single_unit(input) regression = zqtflearn.regression(linear, optimizer='sgd', loss='mean_square', metric='R2', learning_rate=0.01) m = zqtflearn.DNN(regression) # Testing fit and predict m.fit(X, Y, n_epoch=1000, show_metric=True, snapshot_epoch=False) res = m.predict([3.2])[0] self.assertGreater( res, 1.3, "DNN test (linear regression) failed! with score: " + str(res) + " expected > 1.3") self.assertLess( res, 1.8, "DNN test (linear regression) failed! with score: " + str(res) + " expected < 1.8") # Testing save method m.save("test_dnn.zqtflearn") self.assertTrue(os.path.exists("test_dnn.zqtflearn.index")) # Testing loading, with change of variable scope (saved with no scope, now loading into scopeA) with tf.Graph().as_default(): # start with clear graph with tf.variable_scope("scopeA") as scope: input = zqtflearn.input_data(shape=[None]) linear = zqtflearn.single_unit(input) regression = zqtflearn.regression(linear, optimizer='sgd', loss='mean_square', metric='R2', learning_rate=0.01) m = zqtflearn.DNN(regression) def try_load(): m.load("test_dnn.zqtflearn") self.assertRaises( tf.errors.NotFoundError, try_load) # fails, since names in file don't have "scopeA" m.load("test_dnn.zqtflearn", variable_name_map=( "scopeA/", "")) # succeeds, because variable names are rewritten res = m.predict([3.2])[0] self.assertGreater( res, 1.3, "DNN test (linear regression) failed after loading model! score: " + str(res) + " expected > 1.3") self.assertLess( res, 1.8, "DNN test (linear regression) failed after loading model! score: " + str(res) + " expected < 1.8")
import zqtflearn # Regression data X = [ 3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167, 7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1 ] Y = [ 1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221, 2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3 ] # Linear Regression graph input_ = zqtflearn.input_data(shape=[None]) linear = zqtflearn.single_unit(input_) regression = zqtflearn.regression(linear, optimizer='sgd', loss='mean_square', metric='R2', learning_rate=0.01) m = zqtflearn.DNN(regression) m.fit(X, Y, n_epoch=1000, show_metric=True, snapshot_epoch=False) print("\nRegression result:") print("Y = " + str(m.get_weights(linear.W)) + "*X + " + str(m.get_weights(linear.b))) print("\nTest prediction for x = 3.2, 3.3, 3.4:") print(m.predict([3.2, 3.3, 3.4])) # should output (close, not exact) y = [1.5315033197402954, 1.5585315227508545, 1.5855598449707031]
# Building Residual Network net = zqtflearn.input_data(shape=[None, 28, 28, 1]) net = zqtflearn.conv_2d(net, 64, 3, activation='relu', bias=False) # Residual blocks net = zqtflearn.residual_bottleneck(net, 3, 16, 64) net = zqtflearn.residual_bottleneck(net, 1, 32, 128, downsample=True) net = zqtflearn.residual_bottleneck(net, 2, 32, 128) net = zqtflearn.residual_bottleneck(net, 1, 64, 256, downsample=True) net = zqtflearn.residual_bottleneck(net, 2, 64, 256) net = zqtflearn.batch_normalization(net) net = zqtflearn.activation(net, 'relu') net = zqtflearn.global_avg_pool(net) # Regression net = zqtflearn.fully_connected(net, 10, activation='softmax') net = zqtflearn.regression(net, optimizer='momentum', loss='categorical_crossentropy', learning_rate=0.1) # Training model = zqtflearn.DNN(net, checkpoint_path='model_resnet_mnist', max_checkpoints=10, tensorboard_verbose=0) model.fit(X, Y, n_epoch=100, validation_set=(testX, testY), show_metric=True, batch_size=256, run_id='resnet_mnist')
# -*- coding: utf-8 -*- """ MNIST Classification using RNN over images pixels. A picture is representated as a sequence of pixels, coresponding to an image's width (timestep) and height (number of sequences). """ from __future__ import division, print_function, absolute_import import numpy as np import zqtflearn import zqtflearn.datasets.mnist as mnist X, Y, testX, testY = mnist.load_data(one_hot=True) X = np.reshape(X, (-1, 28, 28)) testX = np.reshape(testX, (-1, 28, 28)) net = zqtflearn.input_data(shape=[None, 28, 28]) net = zqtflearn.lstm(net, 128, return_seq=True) net = zqtflearn.lstm(net, 128) net = zqtflearn.fully_connected(net, 10, activation='softmax') net = zqtflearn.regression(net, optimizer='adam', loss='categorical_crossentropy', name="output1") model = zqtflearn.DNN(net, tensorboard_verbose=2) model.fit(X, Y, n_epoch=1, validation_set=0.1, show_metric=True, snapshot_step=100)
img_aug.add_random_flip_leftright() img_aug.add_random_crop([32, 32], padding=4) # Building Residual Network net = zqtflearn.input_data(shape=[None, 32, 32, 3], data_preprocessing=img_prep, data_augmentation=img_aug) net = zqtflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001) net = zqtflearn.residual_block(net, n, 16) net = zqtflearn.residual_block(net, 1, 32, downsample=True) net = zqtflearn.residual_block(net, n - 1, 32) net = zqtflearn.residual_block(net, 1, 64, downsample=True) net = zqtflearn.residual_block(net, n - 1, 64) net = zqtflearn.batch_normalization(net) net = zqtflearn.activation(net, 'relu') net = zqtflearn.global_avg_pool(net) # Regression net = zqtflearn.fully_connected(net, 10, activation='softmax') mom = zqtflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True) net = zqtflearn.regression(net, optimizer=mom, loss='categorical_crossentropy') # Training model = zqtflearn.DNN(net, checkpoint_path='model_resnet_cifar10', max_checkpoints=10, tensorboard_verbose=0, clip_gradients=0.) model.fit(X, Y, n_epoch=200, validation_set=(testX, testY), snapshot_epoch=False, snapshot_step=500, show_metric=True, batch_size=128, shuffle=True, run_id='resnet_cifar10')
def build_model(self, learning_rate=[0.001, 0.01]): ''' Model - wide and deep - built using tflearn ''' n_cc = len(self.continuous_columns) n_cc = 108 input_shape = [None, n_cc] if self.verbose: print("=" * 77 + " Model %s (type=%s)" % (self.name, self.model_type)) print(" Input placeholder shape=%s" % str(input_shape)) wide_inputs = zqtflearn.input_data(shape=input_shape, name="wide_X") if not isinstance(learning_rate, list): learning_rate = [learning_rate, learning_rate] # wide, deep if self.verbose: print(" Learning rates (wide, deep)=%s" % learning_rate) with tf.name_scope( "Y"): # placeholder for target variable (i.e. trainY input) Y_in = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="Y") with tf.variable_op_scope([wide_inputs], None, "cb_unit", reuse=False) as scope: central_bias = zqtflearn.variables.variable( 'central_bias', shape=[1], initializer=tf.constant_initializer(np.random.randn()), trainable=True, restore=True) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/cb_unit', central_bias) wide_network = self.wide_model(wide_inputs, n_cc) network = wide_network network = tf.add(network, central_bias, name="add_central_bias") # add validation monitor summaries giving confusion matrix entries with tf.name_scope('Monitors'): predictions = tf.cast(tf.greater(network, 0), tf.int64) print("predictions=%s" % predictions) Ybool = tf.cast(Y_in, tf.bool) print("Ybool=%s" % Ybool) pos = tf.boolean_mask(predictions, Ybool) neg = tf.boolean_mask(predictions, ~Ybool) psize = tf.cast(tf.shape(pos)[0], tf.int64) nsize = tf.cast(tf.shape(neg)[0], tf.int64) true_positive = tf.reduce_sum(pos, name="true_positive") false_negative = tf.subtract(psize, true_positive, name="false_negative") false_positive = tf.reduce_sum(neg, name="false_positive") true_negative = tf.subtract(nsize, false_positive, name="true_negative") overall_accuracy = tf.truediv(tf.add(true_positive, true_negative), tf.add(nsize, psize), name="overall_accuracy") vmset = [ true_positive, true_negative, false_positive, false_negative, overall_accuracy ] zqtflearn.regression( network, placeholder=Y_in, optimizer='adam', #loss="roc_auc_score", loss='binary_crossentropy', metric="accuracy", learning_rate=learning_rate[0], # use wide learning rate # trainable_vars=[central_bias], validation_monitors=vmset, op_name="central_bias_regression", name="Y") self.model = zqtflearn.DNN( network, tensorboard_verbose=self.tensorboard_verbose, max_checkpoints=self.max_checkpoints, checkpoint_path="%s/%s.tfl" % (self.checkpoints_dir, self.name), tensorboard_dir=self.tensorboard_dir) if self.verbose: print("Target variables:") for v in tf.get_collection(tf.GraphKeys.TARGETS): print(" variable %s: %s" % (v.name, v)) print("=" * 77)
# X, Y = image_preloader(data_dir, image_shape=(224, 224), mode='folder', # categorical_labels=True, normalize=True, # files_extension=['.jpg', '.png'], filter_channel=True) num_classes = 10 # num of your dataset # VGG preprocessing img_prep = ImagePreprocessing() img_prep.add_featurewise_zero_center(mean=[123.68, 116.779, 103.939], per_channel=True) # VGG Network x = zqtflearn.input_data(shape=[None, 224, 224, 3], name='input', data_preprocessing=img_prep) softmax = vgg16(x, num_classes) regression = zqtflearn.regression(softmax, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.001, restore=False) model = zqtflearn.DNN(regression, checkpoint_path='vgg-finetuning', max_checkpoints=3, tensorboard_verbose=2, tensorboard_dir="./logs") model_file = os.path.join(model_path, "vgg16.zqtflearn") model.load(model_file, weights_only=True) # Start finetuning model.fit(X, Y, n_epoch=10, validation_set=0.1, shuffle=True, show_metric=True, batch_size=64, snapshot_epoch=False, snapshot_step=200, run_id='vgg-finetuning') model.save('your-task-model-retrained-by-vgg')
import tensorflow as tf import zqtflearn # Logical NOT operator X = [[0.], [1.]] Y = [[1.], [0.]] # Graph definition with tf.Graph().as_default(): g = zqtflearn.input_data(shape=[None, 1]) g = zqtflearn.fully_connected(g, 128, activation='linear') g = zqtflearn.fully_connected(g, 128, activation='linear') g = zqtflearn.fully_connected(g, 1, activation='sigmoid') g = zqtflearn.regression(g, optimizer='sgd', learning_rate=2., loss='mean_square') # Model training m = zqtflearn.DNN(g) m.fit(X, Y, n_epoch=100, snapshot_epoch=False) # Test model print("Testing NOT operator") print("NOT 0:", m.predict([[0.]])) print("NOT 1:", m.predict([[1.]])) # Logical OR operator X = [[0., 0.], [0., 1.], [1., 0.], [1., 1.]] Y = [[0.], [1.], [1.], [1.]]
# Converting 'sex' field to float (id is 1 after removing labels column) passengers[i][1] = 1. if passengers[i][1] == 'female' else 0. return np.array(passengers, dtype=np.float32) # Ignore 'name' and 'ticket' columns (id 1 & 6 of data array) to_ignore=[1, 6] # Preprocess data data = preprocess(data, to_ignore) # Build neural network net = zqtflearn.input_data(shape=[None, 6]) net = zqtflearn.fully_connected(net, 32) net = zqtflearn.fully_connected(net, 32) net = zqtflearn.fully_connected(net, 2, activation='softmax') net = zqtflearn.regression(net) # Define model model = zqtflearn.DNN(net) # Start training (apply gradient descent algorithm) model.fit(data, labels, n_epoch=10, batch_size=16, show_metric=True) # Let's create some data for DiCaprio and Winslet dicaprio = [3, 'Jack Dawson', 'male', 19, 0, 0, 'N/A', 5.0000] winslet = [1, 'Rose DeWitt Bukater', 'female', 17, 1, 2, 'N/A', 100.0000] # Preprocess data dicaprio, winslet = preprocess([dicaprio, winslet], to_ignore) # Predict surviving chances (class 1 results) pred = model.predict([dicaprio, winslet]) print("DiCaprio Surviving Rate:", pred[0][1]) print("Winslet Surviving Rate:", pred[1][1])
disc_net = tf.concat([disc_fake, disc_real], axis=0) # Build Stacked Generator/Discriminator gen_net = generator(gen_input, reuse=True) stacked_gan_net = discriminator(gen_net, reuse=True) # Build Training Ops for both Generator and Discriminator. # Each network optimization should only update its own variable, thus we need # to retrieve each network variables (with get_layer_variables_by_scope). disc_vars = zqtflearn.get_layer_variables_by_scope('Discriminator') # We need 2 target placeholders, for both the real and fake image target. disc_target = zqtflearn.multi_target_data( ['target_disc_fake', 'target_disc_real'], shape=[None, 2]) disc_model = zqtflearn.regression(disc_net, optimizer='adam', placeholder=disc_target, loss='categorical_crossentropy', trainable_vars=disc_vars, batch_size=64, name='target_disc', op_name='DISC') gen_vars = zqtflearn.get_layer_variables_by_scope('Generator') gan_model = zqtflearn.regression(stacked_gan_net, optimizer='adam', loss='categorical_crossentropy', trainable_vars=gen_vars, batch_size=64, name='target_gen', op_name='GEN') # Define GAN model, that output the generated images. gan = zqtflearn.DNN(gan_model)
gen_sample = generator(gen_input) disc_real = discriminator(disc_input) disc_fake = discriminator(gen_sample, reuse=True) # Define Loss disc_loss = -tf.reduce_mean(tf.log(disc_real) + tf.log(1. - disc_fake)) gen_loss = -tf.reduce_mean(tf.log(disc_fake)) # Build Training Ops for both Generator and Discriminator. # Each network optimization should only update its own variable, thus we need # to retrieve each network variables (with get_layer_variables_by_scope) and set # 'placeholder=None' because we do not need to feed any target. gen_vars = zqtflearn.get_layer_variables_by_scope('Generator') gen_model = zqtflearn.regression(gen_sample, placeholder=None, optimizer='adam', loss=gen_loss, trainable_vars=gen_vars, batch_size=64, name='target_gen', op_name='GEN') disc_vars = zqtflearn.get_layer_variables_by_scope('Discriminator') disc_model = zqtflearn.regression(disc_real, placeholder=None, optimizer='adam', loss=disc_loss, trainable_vars=disc_vars, batch_size=64, name='target_disc', op_name='DISC') # Define GAN model, that output the generated images. gan = zqtflearn.DNN(gen_model) # Training # Generate noise to feed to the generator z = np.random.uniform(-1., 1., size=[total_samples, z_dim]) # Start training, feed both noise and real images. gan.fit(X_inputs={gen_input: z, disc_input: X}, Y_targets=None, n_epoch=100)
1536, 1, bias=False, activation=None, name='Conv2d_7b_1x1'))) net = avg_pool_2d(net, net.get_shape().as_list()[1:3], strides=2, padding='VALID', name='AvgPool_1a_8x8') net = flatten(net) net = dropout(net, dropout_keep_prob) loss = fully_connected(net, num_classes, activation='softmax') network = zqtflearn.regression(loss, optimizer='RMSprop', loss='categorical_crossentropy', learning_rate=0.0001) model = zqtflearn.DNN(network, checkpoint_path='inception_resnet_v2', max_checkpoints=1, tensorboard_verbose=2, tensorboard_dir="./tflearn_logs/") model.fit(X, Y, n_epoch=1000, validation_set=0.1, shuffle=True, show_metric=True, batch_size=32, snapshot_step=2000, snapshot_epoch=False,
n_words=10000, valid_portion=0.1) trainX, trainY = train testX, testY = test # Data preprocessing # Sequence padding trainX = pad_sequences(trainX, maxlen=100, value=0.) testX = pad_sequences(testX, maxlen=100, value=0.) # Converting labels to binary vectors trainY = to_categorical(trainY) testY = to_categorical(testY) # Network building net = zqtflearn.input_data([None, 100]) net = zqtflearn.embedding(net, input_dim=10000, output_dim=128) net = zqtflearn.lstm(net, 128, dropout=0.8) net = zqtflearn.fully_connected(net, 2, activation='softmax') net = zqtflearn.regression(net, optimizer='adam', learning_rate=0.001, loss='categorical_crossentropy') # Training model = zqtflearn.DNN(net, tensorboard_verbose=0) model.fit(trainX, trainY, validation_set=(testX, testY), show_metric=True, batch_size=32)
import zqtflearn.datasets.mnist as mnist X, Y, testX, testY = mnist.load_data(one_hot=True) # Building the encoder encoder = zqtflearn.input_data(shape=[None, 784]) encoder = zqtflearn.fully_connected(encoder, 256) encoder = zqtflearn.fully_connected(encoder, 64) # Building the decoder decoder = zqtflearn.fully_connected(encoder, 256) decoder = zqtflearn.fully_connected(decoder, 784, activation='sigmoid') # Regression, with mean square error net = zqtflearn.regression(decoder, optimizer='adam', learning_rate=0.001, loss='mean_square', metric=None) # Training the auto encoder model = zqtflearn.DNN(net, tensorboard_verbose=0) model.fit(X, X, n_epoch=20, validation_set=(testX, testX), run_id="auto_encoder", batch_size=256) # Encoding X[0] for test print("\nTest encoding of X[0]:") # New model, re-using the same session, for weights sharing
dense1 = zqtflearn.fully_connected(input_layer, 64, activation='tanh', regularizer='L2', weight_decay=0.001) dropout1 = zqtflearn.dropout(dense1, 0.8) dense2 = zqtflearn.fully_connected(dropout1, 64, activation='tanh', regularizer='L2', weight_decay=0.001) dropout2 = zqtflearn.dropout(dense2, 0.8) softmax = zqtflearn.fully_connected(dropout2, 10, activation='softmax') # Regression using SGD with learning rate decay and Top-3 accuracy sgd = zqtflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=1000) top_k = zqtflearn.metrics.Top_k(3) net = zqtflearn.regression(softmax, optimizer=sgd, metric=top_k, loss='categorical_crossentropy') # Training model = zqtflearn.DNN(net, tensorboard_verbose=0) model.fit(X, Y, n_epoch=20, validation_set=(testX, testY), show_metric=True, run_id="dense_model")
def model(self, mode="train", num_layers=1, cell_size=32, cell_type="BasicLSTMCell", embedding_size=20, learning_rate=0.0001, tensorboard_verbose=0, checkpoint_path=None): ''' Build tensor specifying graph of operations for the seq2seq neural network model. mode = string, either "train" or "predict" cell_type = attribute of rnn_cell specifying which RNN cell type to use cell_size = size for the hidden layer in the RNN cell num_layers = number of RNN cell layers to use Return TFLearn model instance. Use DNN model for this. ''' assert mode in ["train", "predict"] checkpoint_path = checkpoint_path or ( "%s%ss2s_checkpoint.tfl" % (self.data_dir or "", "/" if self.data_dir else "")) GO_VALUE = self.out_max_int + 1 # unique integer value used to trigger decoder outputs in the seq2seq RNN network = zqtflearn.input_data( shape=[None, self.in_seq_len + self.out_seq_len], dtype=tf.int32, name="XY") encoder_inputs = tf.slice(network, [0, 0], [-1, self.in_seq_len], name="enc_in") # get encoder inputs encoder_inputs = tf.unstack( encoder_inputs, axis=1 ) # transform into list of self.in_seq_len elements, each [-1] decoder_inputs = tf.slice(network, [0, self.in_seq_len], [-1, self.out_seq_len], name="dec_in") # get decoder inputs decoder_inputs = tf.unstack( decoder_inputs, axis=1 ) # transform into list of self.out_seq_len elements, each [-1] go_input = tf.multiply( tf.ones_like(decoder_inputs[0], dtype=tf.int32), GO_VALUE ) # insert "GO" symbol as the first decoder input; drop the last decoder input decoder_inputs = [ go_input ] + decoder_inputs[:self.out_seq_len - 1] # insert GO as first; drop last decoder input feed_previous = not (mode == "train") if self.verbose > 3: print("feed_previous = %s" % str(feed_previous)) print("encoder inputs: %s" % str(encoder_inputs)) print("decoder inputs: %s" % str(decoder_inputs)) print("len decoder inputs: %s" % len(decoder_inputs)) self.n_input_symbols = self.in_max_int + 1 # default is integers from 0 to 9 self.n_output_symbols = self.out_max_int + 2 # extra "GO" symbol for decoder inputs single_cell = getattr(rnn_cell, cell_type)(cell_size, state_is_tuple=True) if num_layers == 1: cell = single_cell else: cell = rnn_cell.MultiRNNCell([single_cell] * num_layers) if self.seq2seq_model == "embedding_rnn": model_outputs, states = seq2seq.embedding_rnn_seq2seq( encoder_inputs, # encoder_inputs: A list of 2D Tensors [batch_size, input_size]. decoder_inputs, cell, num_encoder_symbols=self.n_input_symbols, num_decoder_symbols=self.n_output_symbols, embedding_size=embedding_size, feed_previous=feed_previous) elif self.seq2seq_model == "embedding_attention": model_outputs, states = seq2seq.embedding_attention_seq2seq( encoder_inputs, # encoder_inputs: A list of 2D Tensors [batch_size, input_size]. decoder_inputs, cell, num_encoder_symbols=self.n_input_symbols, num_decoder_symbols=self.n_output_symbols, embedding_size=embedding_size, num_heads=1, initial_state_attention=False, feed_previous=feed_previous) else: raise Exception('[TFLearnSeq2Seq] Unknown seq2seq model %s' % self.seq2seq_model) tf.add_to_collection( tf.GraphKeys.LAYER_VARIABLES + '/' + "seq2seq_model", model_outputs) # for TFLearn to know what to save and restore # model_outputs: list of the same length as decoder_inputs of 2D Tensors with shape [batch_size x output_size] containing the generated outputs. if self.verbose > 2: print("model outputs: %s" % model_outputs) network = tf.stack( model_outputs, axis=1 ) # shape [-1, n_decoder_inputs (= self.out_seq_len), num_decoder_symbols] if self.verbose > 2: print("packed model outputs: %s" % network) if self.verbose > 3: all_vars = tf.get_collection(tf.GraphKeys.VARIABLES) print("all_vars = %s" % all_vars) with tf.name_scope( "TargetsData" ): # placeholder for target variable (i.e. trainY input) targetY = tf.placeholder(shape=[None, self.out_seq_len], dtype=tf.int32, name="Y") network = zqtflearn.regression(network, placeholder=targetY, optimizer='adam', learning_rate=learning_rate, loss=self.sequence_loss, metric=self.accuracy, name="Y") model = zqtflearn.DNN(network, tensorboard_verbose=tensorboard_verbose, checkpoint_path=checkpoint_path) return model