def main():
dataset = MnistDataSet(validation_sample_count=10000)
lr_list = [0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15]
lr_periods = [500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500]
threshold_decay_periods = [500, 1000]
threshold_decays = [0.5, 0.75]
list_of_lists = [
lr_list, lr_periods, threshold_decay_periods, threshold_decays
]
for idx in itertools.product(*list_of_lists):
tf.reset_default_graph()
lr = idx[0]
lr_period = idx[1]
threshold_decay_period = idx[2]
threshold_decay = idx[3]
train_program_path = UtilityFuncs.get_absolute_path(
script_file=__file__, relative_path="train_program.json")
train_program = TrainProgram(program_file=train_program_path)
train_program.set_train_program_element(element_name="lr_initial",
keywords=",",
skipwords="",
value=lr)
train_program.set_train_program_element(
element_name="lr_update_interval",
keywords=",",
skipwords="",
value=lr_period)
train_program.set_train_program_element(
element_name="BranchingProbThreshold",
keywords={"decay"},
skipwords={},
value=threshold_decay)
train_program.set_train_program_element(
element_name="BranchingProbThreshold",
keywords={"decayPeriod"},
skipwords={},
value=threshold_decay_period)
cnn_lenet = TreeNetwork(
dataset=dataset,
parameter_file=None,
tree_degree=2,
tree_type=TreeType.hard,
problem_type=ProblemType.classification,
train_program=train_program,
explanation=
"100 Epochs, {0} lr decay period, {1} initial lr params runId17 "
"threshold decay period: {2} threshold decay: {3}".format(
lr_period, lr, threshold_decay_period, threshold_decay),
list_of_node_builder_functions=[root_func, l1_func, leaf_func])
optimizer = SgdOptimizer(network=cnn_lenet,
use_biased_gradient_estimates=True)
cnn_lenet.set_optimizer(optimizer=optimizer)
cnn_lenet.build_network()
cnn_lenet.check_grads()
cnn_lenet.init_session()
cnn_lenet.train()
def main():
dataset = MnistDataSet(validation_sample_count=5000)
dataset.load_dataset()
train_program_path = UtilityFuncs.get_absolute_path(
script_file=__file__, relative_path="train_program.json")
train_program = TrainProgram(program_file=train_program_path)
cnn_lenet = TreeNetwork(run_id=0,
dataset=dataset,
parameter_file=None,
tree_degree=2,
tree_type=TreeType.hard,
problem_type=ProblemType.classification,
train_program=train_program,
list_of_node_builder_functions=[baseline_network])
optimizer = SgdOptimizer(network=cnn_lenet,
use_biased_gradient_estimates=True)
cnn_lenet.set_optimizer(optimizer=optimizer)
cnn_lenet.build_network()
cnn_lenet.init_session()
cnn_lenet.train()
variance_epsilon=1e-5)
return normed_x
def batch_norm_eval(x, decay, b_mean, b_var, g, b):
normed_x = tf.nn.batch_normalization(x=x,
mean=b_mean,
variance=b_var,
offset=b,
scale=g,
variance_epsilon=1e-5)
return normed_x
decay = 0.5
dataset = MnistDataSet(validation_sample_count=10000,
load_validation_from="validation_indices")
# Network
flat_data = tf.contrib.layers.flatten(GlobalConstants.TRAIN_DATA_TENSOR)
iteration = tf.placeholder(name="iteration", dtype=tf.int64)
is_decision_phase = tf.placeholder(name="is_decision_phase", dtype=tf.int64)
is_training_phase = tf.placeholder(name="is_training_phase", dtype=tf.int64)
if GlobalConstants.USE_TRAINABLE_PARAMS_WITH_BATCH_NORM:
gamma = tf.Variable(name="gamma",
initial_value=tf.ones([flat_data.get_shape()[-1]]))
beta = tf.Variable(name="beta",
initial_value=tf.zeros([flat_data.get_shape()[-1]]))
else:
gamma = None
beta = None
x_1 = branch_array[j, :]
dif = x_0 - x_1
nz = np.flatnonzero(dif)
if len(nz) != 0:
raise Exception("!!!ERROR!!!")
print("Correct Result.")
k = 3
D = MnistDataSet.MNIST_SIZE * MnistDataSet.MNIST_SIZE
threshold = 0.3
feature_count = 32
epsilon = 0.000001
batch_size = 100
dataset = MnistDataSet(validation_sample_count=5000)
dataset.load_dataset()
samples, labels, indices_list = dataset.get_next_batch()
index_list = np.arange(0, batch_size)
initializer = tf.contrib.layers.xavier_initializer()
x = tf.placeholder(tf.float32, name="x")
indices = tf.placeholder(tf.int64, name="indices")
# Convolution
x_image = tf.reshape(x, [-1, MnistDataSet.MNIST_SIZE, MnistDataSet.MNIST_SIZE, 1])
C = tf.get_variable(name="C", shape=[5, 5, 1, feature_count], initializer=initializer,
dtype=tf.float32)
b_c = tf.get_variable(name="b_c", shape=(feature_count,), initializer=initializer, dtype=tf.float32)
conv_without_bias = tf.nn.conv2d(x_image, C, strides=[1, 1, 1, 1], padding="SAME")
conv = conv_without_bias + b_c
# Branching
def main(_):
# Import data
# mnist = input_data.read_data_sets(FLAGS.data_dir, validation_size=10000, one_hot=True)
# Create the model
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, 10])
# Build the graph for the deep net
y_conv = deepnn(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
l2_loss_list = []
for v in tf.trainable_variables():
loss_tensor = tf.nn.l2_loss(v)
if "bias" in v.name:
l2_loss_list.append(0.0 * loss_tensor)
else:
l2_loss_list.append(0.0005 * loss_tensor)
l2_loss = tf.add_n(l2_loss_list)
final_loss = cross_entropy + l2_loss
with tf.name_scope('momentum_optimizer'):
# train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
train_step = tf.train.MomentumOptimizer(learningRate, 0.9).minimize(
final_loss, global_step=globalCounter)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
dataset = MnistDataSet(validation_sample_count=10000,
load_validation_from=None)
dataset.set_current_data_set_type(dataset_type=DatasetTypes.training)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(120000):
samples, labels, indices_list, one_hot_labels = dataset.get_next_batch(
batch_size=125)
samples = np.expand_dims(samples, axis=3)
train_step.run(feed_dict={x: samples, y_: one_hot_labels})
lr = sess.run([learningRate])
if dataset.isNewEpoch:
print("i={0} lr={1}".format(i, lr))
dataset.set_current_data_set_type(
dataset_type=DatasetTypes.test)
test_samples, test_labels, test_indices_list, test_one_hot_labels = dataset.get_next_batch(
batch_size=10000)
test_samples = np.expand_dims(test_samples, axis=3)
print('test accuracy %g' %
accuracy.eval(feed_dict={
x: test_samples,
y_: test_one_hot_labels
}))
dataset.set_current_data_set_type(
dataset_type=DatasetTypes.training)