def main(cli_args):
parser = argparse.ArgumentParser(
description="CSCE 496 HW 2, Classify Cifar data")
parser.add_argument('--input_dir',
type=str,
default='/work/cse496dl/shared/homework/02',
help='Numpy datafile input')
parser.add_argument(
'--model_dir',
type=str,
default='./homework_2/',
help='directory where model graph and weights are saved')
parser.add_argument('--epoch',
type=int,
default=100,
help="Epoch : number of iterations for the model")
parser.add_argument('--batch_size',
type=int,
default=32,
help="Batch Size")
parser.add_argument('--model',
type=int,
help=" '1' for basic model, '2' for best model")
parser.add_argument(
'--stopCount',
type=int,
default=100,
help="Number of times for dropping accuracy before early stopping")
args_input = parser.parse_args(cli_args)
if args_input.input_dir:
input_dir = args_input.input_dir
else:
raise ValueError("Provide a valid input data path")
if args_input.model_dir:
model_dir = args_input.model_dir
else:
raise ValueError("Provide a valid model data path")
if args_input.epoch:
epochs = args_input.epoch
else:
raise ValueError("Epoch value cannot be null and has to be an integer")
if args_input.batch_size:
batch_size = args_input.batch_size
else:
raise ValueError(
"Batch Size value cannot be null and has to be an integer")
if args_input.model:
model = args_input.model
else:
raise ValueError("Model selection must not be empty")
if args_input.stopCount:
stop_counter = args_input.stopCount
else:
raise ValueError("StopCount have to be an int")
input_dir = '/work/cse496dl/shared/homework/02'
#Make output model dir
if os.path.exists(model_dir) == False:
os.mkdir(model_dir)
#Load Data
x = tf.placeholder(tf.float32, [None, 32, 32, 3], name='input_placeholder')
y = tf.placeholder(tf.float32, [None, 100], name='labels')
#Specify Model
if (str(model) == '1'):
train_images, train_labels, test_images, test_labels, val_images, val_labels = util.load_data(
"")
_, outputLayer = initiate_basic_model(x)
#Run Training with early stopping and save output
counter = stop_counter
prev_winner = 0
curr_winner = 0
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)
cross_entropy = util.cross_entropy_op(y, outputLayer)
global_step_tensor = util.global_step_tensor('global_step_tensor')
train_op = util.train_op_basic(cross_entropy, global_step_tensor,
optimizer)
conf_matrix = util.confusion_matrix_op(y, outputLayer, 100)
saver = tf.train.Saver()
with tf.Session() as session:
session.run(tf.global_variables_initializer())
counter = stop_counter
for epoch in range(epochs):
if counter > 0:
print("Epoch : " + str(epoch))
util.training(batch_size, x, y, train_images, train_labels,
session, train_op, conf_matrix, 100)
accuracy = util.validation(batch_size, x, y, val_images,
val_labels, session,
cross_entropy, conf_matrix, 100)
if epoch == 0:
prev_winner = accuracy
print("Saving.......")
saver.save(session,
os.path.join("./homework_2/", "homework_2"))
else:
curr_winner = accuracy
if (curr_winner > prev_winner) and (counter > 0):
prev_winner = curr_winner
print("Saving.......")
saver.save(
session,
os.path.join("./homework_2/", "homework_2"))
else:
counter -= 1
test_accuracy = util.test(batch_size, x, y, test_images,
test_labels, session,
cross_entropy, conf_matrix, 100)
#Calculate the confidence interval
value1, value2 = util.confidence_interval(
test_accuracy, 1.96, test_images.shape[0])
print("Confidence Interval : " + str(value1) + " , " +
str(value2))
else:
break
elif (str(model) == '2'):
sparsity_weight = 5e-3
#Load the data and reshape it
train_data = np.load(
os.path.join(os.path.join(input_dir, 'imagenet_images.npy')))
train_images, train_labels, test_images, test_labels, val_images, val_labels = util.load_data(
"")
#train_data = np.reshape(train_data, [-1,32,32,1])
#Add noise to the data
noise_level = 0.2
x_noise = x + noise_level * tf.random_normal(tf.shape(x))
code, outputs = initiate_autoencoder(x_noise, 100)
#Optimizer for Autoencoder
sparsity_loss = tf.norm(code, ord=1, axis=1)
reconstruction_loss = tf.reduce_mean(tf.square(outputs -
x)) # Mean Square Error
total_loss = reconstruction_loss + sparsity_weight * sparsity_loss
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)
train_op = optimizer.minimize(total_loss)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
util.autoencoder_training(x, code, epochs, batch_size, train_data,
sess, train_op)
saver.save(sess, os.path.join("./homework_2/", "homework_2"))
print("Done : " + str(code))
_, outputLayer = initiate_dense_model(code)
#Run Training with early stopping and save output
counter = stop_counter
prev_winner = 0
curr_winner = 0
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)
cross_entropy = util.cross_entropy_op(y, outputLayer)
global_step_tensor = util.global_step_tensor('global_step_tensor')
#train_op = util.train_op_encoder(cross_entropy, global_step_tensor, optimizer, var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, "code_layer"))
train_op = util.train_op_basic(cross_entropy, global_step_tensor,
optimizer)
conf_matrix = util.confusion_matrix_op(y, outputLayer, 100)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
if os.path.isfile(os.path.join("./homework_2/", "homework_2")):
saver = tf.train.import_meta_graph("homework_2.meta")
saver.restore(session, "./homework_2/homework_2")
code_encode = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
"code_layer")
session.run(
tf.variables_initializer(code_encode,
name="init_encoded_layer"))
tf.stop_gradient(
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
"init_encoded_layer"))
counter = stop_counter
for epoch in range(epochs):
if counter > 0:
print("Epoch : " + str(epoch))
util.training(batch_size, x, y, train_images, train_labels,
session, train_op, conf_matrix, 100)
accuracy = util.validation(batch_size, x, y, val_images,
val_labels, session,
cross_entropy, conf_matrix, 100)
if epoch == 0:
prev_winner = accuracy
print("Saving.......")
saver.save(session,
os.path.join("./homework_2/", "homework_2"))
else:
curr_winner = accuracy
if (curr_winner > prev_winner) and (counter > 0):
prev_winner = curr_winner
print("Saving.......")
saver.save(
session,
os.path.join("./homework_2/", "homework_2"))
else:
print("Validation Loss : " +
str(curr_winner - prev_winner))
counter -= 1
test_accuracy = util.test(batch_size, x, y, test_images,
test_labels, session,
cross_entropy, conf_matrix, 100)
#Calculate the confidence interval
value1, value2 = util.confidence_interval(
test_accuracy, 1.96, test_images.shape[0])
print("Confidence Interval : " + str(value1) + " , " +
str(value2))
else:
break
def initiate_vgg_model(features,
labels,
filename,
num_classes,
weight_decay,
learning_rate,
handle="training",
reuse_model=None):
#VGG Architecture
with tf.variable_scope("vgg_arch_model", reuse=reuse_model) as scope:
#Declare feature input for the model
x_placeholder = tf.reshape(features / 255, [-1, 224, 224, 3])
#Hidden Layer 1
with tf.name_scope('first_layer'):
conv_1 = tf.layers.conv2d(
x_placeholder,
64,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_1')
conv_2 = tf.layers.conv2d(
conv_1,
64,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_2')
max_pool_1 = tf.layers.max_pooling2d(conv_2, 2, 2, padding='same')
with tf.name_scope('second_layer'):
conv_3 = tf.layers.conv2d(
max_pool_1,
128,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_3')
conv_4 = tf.layers.conv2d(
conv_3,
128,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_4')
max_pool_2 = tf.layers.max_pooling2d(conv_4, 2, 2, padding='same')
with tf.name_scope('third_layer'):
conv_5 = tf.layers.conv2d(
max_pool_2,
256,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_5')
conv_6 = tf.layers.conv2d(
conv_5,
256,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_6')
max_pool_3 = tf.layers.max_pooling2d(conv_6, 2, 2, padding='same')
with tf.name_scope('fourth_layer'):
conv_7 = tf.layers.conv2d(
max_pool_3,
512,
3,
padding='same',
activation=tf.nn.relu,
kernel_regularizer=tf.contrib.layers.l2_regularizer(
scale=0.01),
bias_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),
name='conv_7')
max_pool_4 = tf.layers.max_pooling2d(conv_7, 2, 2, padding='same')
flat_layer = tf.contrib.layers.flatten(max_pool_4)
#inclusion of batch normalization vs dropout based on vgg.py model?
fc_layer_1 = tf.layers.dense(flat_layer,
4096,
activation=tf.nn.relu,
name='fc_layer_1')
#Batch Normalization
batch_norm_layer_1 = tf.layers.batch_normalization(fc_layer_1,
training=True)
#Dropout
dropout_layer_1 = tf.layers.dropout(batch_norm_layer_1, 0.5)
fc_layer_2 = tf.layers.dense(dropout_layer_1,
4096,
activation=tf.nn.relu,
name='fc_layer_2')
#Batch Normalization
batch_norm_layer_2 = tf.layers.batch_normalization(fc_layer_2,
training=True)
#Dropout
dropout_layer_2 = tf.layers.dropout(batch_norm_layer_2, 0.5)
#num_output is not yet defined
logits = tf.layers.dense(dropout_layer_2, num_classes, name='output')
tf.identity(logits, name='output')
#Declare Loss and Optimizer functions
label_placeholder = tf.one_hot(labels, num_classes)
optimizer = tf.contrib.opt.MomentumWOptimizer(
weight_decay=weight_decay,
learning_rate=learning_rate,
momentum=0.9,
name="MomentumWeightDecay")
global_step_tensor = util.global_step_tensor('global_step_tensor')
#with tf.name_scope("images"):
# tf.summary.image(str(handle+"_image"), features[0])
with tf.name_scope("cost"):
cross_entropy = util.cross_entropy_op(label_placeholder, logits,
"cross_entropy")
train_op = util.train_op(cross_entropy, global_step_tensor,
optimizer)
tf.summary.scalar(str(handle + "_loss"), cross_entropy)
with tf.name_scope("confusion_matrix"):
conf_matrix_op = util.confusion_matrix_op(label_placeholder,
logits, num_classes)
conf_mtx = tf.reshape(tf.cast(conf_matrix_op, tf.float32),
[1, num_classes, num_classes, 1])
tf.summary.image(str(handle + "_confusion_matrix"), conf_mtx)
with tf.name_scope("accuracy"):
accuracy = tf.equal(tf.argmax(logits, 1),
tf.argmax(label_placeholder, 1))
accuracy = tf.reduce_mean(tf.cast(accuracy, tf.float32))
tf.summary.scalar(str(handle + "_accuracy"), accuracy)
return scope, train_op, cross_entropy, conf_matrix_op, accuracy
def main(cli_args):
parser = argparse.ArgumentParser(
description="CSCE 496 HW 1, Classify Fashion MNIST data")
parser.add_argument('--input_dir',
type=str,
default='/work/cse496dl/shared/homework/01',
help='Numpy datafile input')
parser.add_argument(
'--model_dir',
type=str,
default='./homework_1/',
help='directory where model graph and weights are saved')
parser.add_argument('--epoch',
type=int,
default=100,
help="Epoch : number of iterations for the model")
parser.add_argument('--batch_size',
type=int,
default=32,
help="Batch Size")
parser.add_argument('--model',
type=int,
help=" '1' for basic model, '2' for best model")
parser.add_argument(
'--stopCount',
type=int,
default=100,
help="Number of times for dropping accuracy before early stopping")
args_input = parser.parse_args(cli_args)
if args_input.input_dir:
input_dir = args_input.input_dir
else:
raise ValueError("Provide a valid input data path")
if args_input.model_dir:
model_dir = args_input.model_dir
else:
raise ValueError("Provide a valid model data path")
if args_input.epoch:
epochs = args_input.epoch
else:
raise ValueError("Epoch value cannot be null and has to be an integer")
if args_input.batch_size:
batch_size = args_input.batch_size
else:
raise ValueError(
"Batch Size value cannot be null and has to be an integer")
if args_input.model:
model = args_input.model
else:
raise ValueError("Model selection must not be empty")
if args_input.stopCount:
stop_counter = args_input.stopCount
else:
raise ValueError("StopCount have to be an int")
input_dir = '/work/cse496dl/shared/homework/01'
#Make output model dir
if os.path.exists(model_dir) == False:
os.mkdir(model_dir)
#Load Data
train_images, train_labels, test_images, test_labels, val_images, val_labels = util.load_data(
input_dir)
x = tf.placeholder(tf.float32, [None, 784], name='input_placeholder')
y = tf.placeholder(tf.float32, [None, 10], name='labels')
#Specify Model
if (str(model) == '1'):
_, outputLayer = initiate_basic_model(x)
elif (str(model) == '2'):
_, outputLayer = initiate_better_model(x)
#Run Training with early stopping and save output
counter = stop_counter
prev_winner = 0
curr_winner = 0
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)
cross_entropy = util.cross_entropy_op(y, outputLayer)
global_step_tensor = util.global_step_tensor('global_step_tensor')
train_op = util.train_op(cross_entropy, global_step_tensor, optimizer)
conf_matrix = util.confusion_matrix_op(y, outputLayer, 10)
saver = tf.train.Saver()
with tf.Session() as session:
session.run(tf.global_variables_initializer())
for i in range(10):
print("KFold : " + str(i))
counter = stop_counter
for epoch in range(epochs):
if counter > 0:
print("Epoch : " + str(epoch))
util.training(batch_size, x, y, train_images[i],
train_labels[i], session, train_op,
conf_matrix, 10)
accuracy = util.validation(batch_size, x, y, val_images[i],
val_labels[i], session,
cross_entropy, conf_matrix, 10)
if epoch == 0:
prev_winner = accuracy
else:
curr_winner = accuracy
if (curr_winner > prev_winner) and (counter > 0):
prev_winner = curr_winner
else:
counter -= 1
test_accuracy = util.test(batch_size, x, y, test_images[i],
test_labels[i], session,
cross_entropy, conf_matrix, 10)
#Calculate the confidence interval
value1, value2 = util.confidence_interval(
test_accuracy, 1.96, test_images[i].shape[0])
print("Confidence Interval : " + str(value1) + " , " +
str(value2))
else:
break
print("Saving.......")
saver.save(session, os.path.join("./homework_1/", "homework_1"))