def main(): dataset = MnistDataSet(validation_sample_count=10000) lr_list = [0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15] lr_periods = [500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500] threshold_decay_periods = [500, 1000] threshold_decays = [0.5, 0.75] list_of_lists = [ lr_list, lr_periods, threshold_decay_periods, threshold_decays ] for idx in itertools.product(*list_of_lists): tf.reset_default_graph() lr = idx[0] lr_period = idx[1] threshold_decay_period = idx[2] threshold_decay = idx[3] train_program_path = UtilityFuncs.get_absolute_path( script_file=__file__, relative_path="train_program.json") train_program = TrainProgram(program_file=train_program_path) train_program.set_train_program_element(element_name="lr_initial", keywords=",", skipwords="", value=lr) train_program.set_train_program_element( element_name="lr_update_interval", keywords=",", skipwords="", value=lr_period) train_program.set_train_program_element( element_name="BranchingProbThreshold", keywords={"decay"}, skipwords={}, value=threshold_decay) train_program.set_train_program_element( element_name="BranchingProbThreshold", keywords={"decayPeriod"}, skipwords={}, value=threshold_decay_period) cnn_lenet = TreeNetwork( dataset=dataset, parameter_file=None, tree_degree=2, tree_type=TreeType.hard, problem_type=ProblemType.classification, train_program=train_program, explanation= "100 Epochs, {0} lr decay period, {1} initial lr params runId17 " "threshold decay period: {2} threshold decay: {3}".format( lr_period, lr, threshold_decay_period, threshold_decay), list_of_node_builder_functions=[root_func, l1_func, leaf_func]) optimizer = SgdOptimizer(network=cnn_lenet, use_biased_gradient_estimates=True) cnn_lenet.set_optimizer(optimizer=optimizer) cnn_lenet.build_network() cnn_lenet.check_grads() cnn_lenet.init_session() cnn_lenet.train()
def main(): dataset = MnistDataSet(validation_sample_count=5000) dataset.load_dataset() train_program_path = UtilityFuncs.get_absolute_path( script_file=__file__, relative_path="train_program.json") train_program = TrainProgram(program_file=train_program_path) cnn_lenet = TreeNetwork(run_id=0, dataset=dataset, parameter_file=None, tree_degree=2, tree_type=TreeType.hard, problem_type=ProblemType.classification, train_program=train_program, list_of_node_builder_functions=[baseline_network]) optimizer = SgdOptimizer(network=cnn_lenet, use_biased_gradient_estimates=True) cnn_lenet.set_optimizer(optimizer=optimizer) cnn_lenet.build_network() cnn_lenet.init_session() cnn_lenet.train()
variance_epsilon=1e-5) return normed_x def batch_norm_eval(x, decay, b_mean, b_var, g, b): normed_x = tf.nn.batch_normalization(x=x, mean=b_mean, variance=b_var, offset=b, scale=g, variance_epsilon=1e-5) return normed_x decay = 0.5 dataset = MnistDataSet(validation_sample_count=10000, load_validation_from="validation_indices") # Network flat_data = tf.contrib.layers.flatten(GlobalConstants.TRAIN_DATA_TENSOR) iteration = tf.placeholder(name="iteration", dtype=tf.int64) is_decision_phase = tf.placeholder(name="is_decision_phase", dtype=tf.int64) is_training_phase = tf.placeholder(name="is_training_phase", dtype=tf.int64) if GlobalConstants.USE_TRAINABLE_PARAMS_WITH_BATCH_NORM: gamma = tf.Variable(name="gamma", initial_value=tf.ones([flat_data.get_shape()[-1]])) beta = tf.Variable(name="beta", initial_value=tf.zeros([flat_data.get_shape()[-1]])) else: gamma = None beta = None
x_1 = branch_array[j, :] dif = x_0 - x_1 nz = np.flatnonzero(dif) if len(nz) != 0: raise Exception("!!!ERROR!!!") print("Correct Result.") k = 3 D = MnistDataSet.MNIST_SIZE * MnistDataSet.MNIST_SIZE threshold = 0.3 feature_count = 32 epsilon = 0.000001 batch_size = 100 dataset = MnistDataSet(validation_sample_count=5000) dataset.load_dataset() samples, labels, indices_list = dataset.get_next_batch() index_list = np.arange(0, batch_size) initializer = tf.contrib.layers.xavier_initializer() x = tf.placeholder(tf.float32, name="x") indices = tf.placeholder(tf.int64, name="indices") # Convolution x_image = tf.reshape(x, [-1, MnistDataSet.MNIST_SIZE, MnistDataSet.MNIST_SIZE, 1]) C = tf.get_variable(name="C", shape=[5, 5, 1, feature_count], initializer=initializer, dtype=tf.float32) b_c = tf.get_variable(name="b_c", shape=(feature_count,), initializer=initializer, dtype=tf.float32) conv_without_bias = tf.nn.conv2d(x_image, C, strides=[1, 1, 1, 1], padding="SAME") conv = conv_without_bias + b_c # Branching
def main(_): # Import data # mnist = input_data.read_data_sets(FLAGS.data_dir, validation_size=10000, one_hot=True) # Create the model x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1)) # Define loss and optimizer y_ = tf.placeholder(tf.float32, [None, 10]) # Build the graph for the deep net y_conv = deepnn(x) with tf.name_scope('loss'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv) cross_entropy = tf.reduce_mean(cross_entropy) l2_loss_list = [] for v in tf.trainable_variables(): loss_tensor = tf.nn.l2_loss(v) if "bias" in v.name: l2_loss_list.append(0.0 * loss_tensor) else: l2_loss_list.append(0.0005 * loss_tensor) l2_loss = tf.add_n(l2_loss_list) final_loss = cross_entropy + l2_loss with tf.name_scope('momentum_optimizer'): # train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) train_step = tf.train.MomentumOptimizer(learningRate, 0.9).minimize( final_loss, global_step=globalCounter) with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) correct_prediction = tf.cast(correct_prediction, tf.float32) accuracy = tf.reduce_mean(correct_prediction) graph_location = tempfile.mkdtemp() print('Saving graph to: %s' % graph_location) train_writer = tf.summary.FileWriter(graph_location) train_writer.add_graph(tf.get_default_graph()) dataset = MnistDataSet(validation_sample_count=10000, load_validation_from=None) dataset.set_current_data_set_type(dataset_type=DatasetTypes.training) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for i in range(120000): samples, labels, indices_list, one_hot_labels = dataset.get_next_batch( batch_size=125) samples = np.expand_dims(samples, axis=3) train_step.run(feed_dict={x: samples, y_: one_hot_labels}) lr = sess.run([learningRate]) if dataset.isNewEpoch: print("i={0} lr={1}".format(i, lr)) dataset.set_current_data_set_type( dataset_type=DatasetTypes.test) test_samples, test_labels, test_indices_list, test_one_hot_labels = dataset.get_next_batch( batch_size=10000) test_samples = np.expand_dims(test_samples, axis=3) print('test accuracy %g' % accuracy.eval(feed_dict={ x: test_samples, y_: test_one_hot_labels })) dataset.set_current_data_set_type( dataset_type=DatasetTypes.training)