def test(cifar10_test_images):
# Always use tf.reset_default_graph() to avoid error
# TODO: Write your testing code here
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# (Above 2 steps should be the same as in train function)
# - Create label prediction tensor
# - Run testing
# DO NOT RUN TRAINING HERE!
# LOAD THE MODEL AND RUN TEST ON THE TEST SET
tf.reset_default_graph()
#tf.reset_default_graph()
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None, 10))
is_training = False
dropout_keep_probablity = 1
EPOCHS = 1
cifar10_test = Cifar10(batch_size=BATCH_SIZE,
one_hot=True,
test=True,
shuffle=False)
cifar10_test_images, cifar10_test_labels = cifar10_test._images, cifar10_test._labels # Get all images and labels of the test set.
cifar10_test_images = tf.cast(cifar10_test_images, tf.float32)
#print(cifar10_train_labels.shape);
#print(batch_y.shape);
logits = net(x, is_training, dropout_keep_probablity)
training_op = tf.argmax(logits, 1)
predicted_y_test = []
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('ckpt'))
num_examples = cifar10_test_images.shape[0]
#sess.run(tf.global_variables_initializer())
print("Testing...")
global_step = 0
for i in range(EPOCHS):
#X_train, y_train = shuffle(X_train, y_train)
for offset in range(0, num_examples, BATCH_SIZE):
batch_x, batch_y = cifar10_test.get_next_batch()
yp = sess.run([training_op],
feed_dict={
x: batch_x,
y: batch_y
})
global_step += 1
if (len(predicted_y_test) == 0):
predicted_y_test = yp
else:
predicted_y_test = np.hstack((predicted_y_test, yp))
sess.close()
return predicted_y_test
def test(cifar10_test_images, batch_size=128):
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# (Above 2 steps should be the same as in train function)
# - Create label prediction tensor
# - Run testing
# input Tensors
input_x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
input_y = tf.placeholder(tf.float32, shape=[None, 10], name='y')
dropout_kept_prob = tf.placeholder(tf.float32, name="dropout_kept_prob")
is_training = tf.placeholder(tf.bool)
global_step = tf.Variable(0, name="global_step", trainable=False)
# ConvNet, optimizer, accuracy
logits = Net(input_x, dropout_kept_prob, is_training)
prediction = tf.argmax(logits, 1)
checkpoint_dir = os.path.join(os.path.curdir, 'model')
saver = tf.train.Saver(max_to_keep=5)
predicted_y_test = []
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
saver.restore(sess, tf.train.latest_checkpoint(checkpoint_dir))
print("Model restored!")
cifar10_test = Cifar10(batch_size=batch_size,
one_hot=False,
test=True,
shuffle=False)
cifar10_test_images, cifar10_test_labels = cifar10_test._images, cifar10_test._labels
num_test_batches = math.ceil(len(cifar10_test_images) / batch_size)
for iteration in range(num_test_batches):
if iteration != (num_test_batches - 1):
x_test_batch = cifar10_test_images[:batch_size]
cifar10_test_images = cifar10_test_images[batch_size:]
else:
x_test_batch = cifar10_test_images[:len(cifar10_test_images)]
cifar10_test_images = cifar10_test_images[
len(cifar10_test_images):]
pred = sess.run([prediction],
feed_dict={
input_x: x_test_batch,
is_training: False,
dropout_kept_prob: 1.0
})[0]
predicted_y_test += pred.tolist()
return np.array(predicted_y_test)
def test(cifar10_test_images):
print("In TEST function")
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
# Get test set
cifar10_train = Cifar10(test=True, shuffle=False, one_hot=True)
x_test, y_test = cifar10_train._images, cifar10_train._labels
cifar10_train_images = tf.cast(x_test, tf.float32)
number_of_images = cifar10_train_images.shape[0]
# - Create placeholder for inputs, training boolean, dropout keep probablity
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None, 10))
training = False
dropoutprob = tf.placeholder(tf.float32)
EPOCHS = 10
BATCH_SIZE = 128
logits = net(x, training, dropoutprob)
training_yp = tf.argmax(logits, 1)
Y = []
saver = tf.train.Saver()
save_path = 'ci10model'
if not os.path.isabs(save_path):
save_path = os.path.abspath(os.path.join(os.getcwd(), save_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, save_path)
prediction_labels = []
predictions = sess.run(logits,
feed_dict={
x: cifar10_test_images,
dropoutprob: 1.0
})
print("Prediction: ", predictions[0])
for array in predictions:
predicted_class = 0
max_score = array[0]
for i, j in enumerate(array):
if j > max_score:
max_score = j
predicted_class = i
prediction_labels.append([predicted_class])
print(prediction_labels)
return np.array(prediction_labels)
def train():
tf.reset_default_graph()
learning_rate = 0.001
X = tf.placeholder(tf.float32, shape=(None, 32, 32, 3), name="X")
Y = tf.placeholder(tf.float32, shape=(None, 10), name="y")
logits = net(X, True, None)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=Y)
loss_operation = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
grads_and_vars = optimizer.compute_gradients(
loss_operation, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
training_operation = optimizer.apply_gradients(grads_and_vars)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/histogram", var)
add_gradient_summaries(grads_and_vars)
tf.summary.scalar('loss_operation', loss_operation)
merged_summary_op = tf.summary.merge_all()
saver = tf.train.Saver()
#accuracy = 100.0 * tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, axis=1), tf.argmax(Y, axis=1)),dtype=tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
cifar10_train = Cifar10(batch_size=100,
one_hot=True,
test=False,
shuffle=True)
n_epochs = 100
n_batches = 100
for epoch in range(n_epochs):
# compute model
for iteration in range(n_batches):
batch_x, batch_y = cifar10_train.get_next_batch()
sess.run([training_operation, merged_summary_op],
feed_dict={
X: batch_x,
Y: batch_y
})
saver.save(sess, 'ckpt/', global_step=n_epochs)
def test(cifar10_test_images):
tf.reset_default_graph()
# Gettting the testing set
cifar10_train = Cifar10(test=True, shuffle=False, one_hot=True)
x_test, y_test = cifar10_train._images, cifar10_train._labels
cifar10_train_images = tf.cast(x_test, tf.float32)
number_of_images = cifar10_train_images.shape[0]
# - Create placeholder for inputs, training boolean, dropout keep probablity
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None, 10))
is_training = False
dropout_keep_probability = 1
n_epochs = 1 # if n_epochs = 10, takes 10 minutes to train
batch_size = 250
# - Construct your model
logits = net(x, is_training, dropout_keep_probability)
training_yp = tf.argmax(logits, 1)
# - Create label prediction tensor
Y = []
saver = tf.train.Saver()
# - Run testing
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('ckpt'))
for epoch in range(n_epochs):
n_batches = number_of_images // batch_size
start = 0
end = batch_size
for iteration in range(n_batches):
# print("iteration: ", iteration, " start: ", start, " end: ", end)
batch_x, batch_y = x_test[start:end], y_test[start:end]
predicted_y = sess.run([training_yp],
feed_dict={
x: batch_x,
y: batch_y
})
start = end
end += batch_size
if len(Y):
Y = np.hstack((Y, predicted_y))
else:
Y = predicted_y
return Y
def main(_):
if tf.gfile.Exists(FLAGS.log_dir):
tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.checkpoint)
with tf.Graph().as_default():
run_config = tf.ConfigProto(allow_soft_placement=True)
run_config.gpu_options.allow_growth = True
with tf.Session(config=run_config) as sess:
model = Cifar10(sess=sess, data_dir=FLAGS.data_dir, batch_size=128,
training=FLAGS.train, checkpoint=FLAGS.checkpoint, log_dir=FLAGS.log_dir,
mode=FLAGS.mode, weight_decay=1e-4)
if FLAGS.train:
model.train(steps=FLAGS.steps)
else:
model.eval('eval')
def main(argv=None):
assert FLAGS.model_dir is not None and FLAGS.cifar10_dir is not None
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
random.seed(43)
np.random.seed(43)
cifar10 = Cifar10()
rf = RandForest()
for i in xrange(100):
randSelect = np.random.randint(cifar10.trainData.shape[0],
size=(FLAGS.tree_input_size, ))
img_inputs = cifar10.trainData[randSelect]
img_labels = cifar10.trainLabelRaw[randSelect]
tree = DT('tree_' + str(i), img_inputs, img_labels)
print 'DT', i, score(cifar10.validLabelRaw,
tree.eval(cifar10.validData))
rf.add_tree(tree)
print 'RF', i, score(cifar10.validLabelRaw, rf.eval(cifar10.validData))
def train():
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
#Gettting the training set
cifar10_train = Cifar10(test=False, shuffle=False, one_hot=True)
x_train, y_train = cifar10_train._images, cifar10_train._labels
cifar10_train_images = tf.cast(x_train, tf.float32)
number_of_images = cifar10_train_images.shape[0]
# - Create placeholder for inputs, training boolean, dropout keep probablity
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None, 10))
# one_hot_y = tf.one_hot(y, 10)
is_training = True
dropout_keep_probability = 0.2
rate = tf.placeholder(tf.float32)
# - Construct your model
logits = net(x, is_training, dropout_keep_probability)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y)
# - Create loss and training op
loss_operation = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=rate)
grads_and_vars = optimizer.compute_gradients(
loss_operation, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
training_operation = optimizer.apply_gradients(grads_and_vars)
lr = 0.001
n_epochs = 15 # if n_epochs = 10, takes 10 minutes to train
batch_size = 250
saver = tf.train.Saver()
# - Run training
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(n_epochs):
# print("epoch: ", epoch)
n_batches = number_of_images // batch_size
start = 0
end = batch_size
if epoch % 2 == 0 and epoch != 0:
lr = lr / 2
for iteration in range(n_batches):
if iteration % 10 == 0:
print("iteration: ", iteration, " start: ", start,
" end: ", end)
batch_x, batch_y = x_train[start:end], y_train[start:end]
sess.run([training_operation],
feed_dict={
x: batch_x,
y: batch_y,
rate: lr
})
start = end
end += batch_size
saver.save(sess, 'ckpt/netModel')
def train():
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
# TODO: Write your training code here
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# - Create loss and training op
# - Run training
# AS IT WILL TAKE VERY LONG ON CIFAR10 DATASET TO TRAIN
# YOU SHOULD USE tf.train.Saver() TO SAVE YOUR MODEL AFTER TRAINING
# AT TEST TIME, LOAD THE MODEL AND RUN TEST ON THE TEST SET
learning_rate = 0.001
dropout_kept_prob = [0.6, 0.5, 0.6, 0.6, 0.5]
X = tf.placeholder(tf.float32, shape=(None, 32, 32, 3), name="X")
Y = tf.placeholder(tf.float32, shape=(None, 10), name="y")
logits = net(X, True, dropout_kept_prob)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=Y)
loss_operation = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
grads_and_vars = optimizer.compute_gradients(
loss_operation, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
training_operation = optimizer.apply_gradients(grads_and_vars)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/histogram", var)
add_gradient_summaries(grads_and_vars)
tf.summary.scalar('loss_operation', loss_operation)
merged_summary_op = tf.summary.merge_all()
saver = tf.train.Saver()
accuracy = 100.0 * tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logits, axis=1), tf.argmax(Y, axis=1)),
dtype=tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
cifar10_train = Cifar10(batch_size=100,
one_hot=True,
test=False,
shuffle=True)
n_epochs = 100
n_batches = 100
for epoch in range(n_epochs):
# compute model
for iteration in range(n_batches):
batch_x, batch_y = cifar10_train.get_next_batch()
sess.run([training_operation, merged_summary_op],
feed_dict={
X: batch_x,
Y: batch_y
})
acc = accuracy.eval(feed_dict={X: batch_x, Y: batch_y})
loss = loss_operation.eval(feed_dict={X: batch_x, Y: batch_y})
print(epoch, "Training batch accuracy:", acc, "loss is: ", loss)
saver.save(sess, 'ckpt/', global_step=n_epochs)
def train(current_host, hosts, num_cpus, num_gpus, channel_input_dirs,
model_dir, hyperparameters, **kwargs):
# retrieve the hyperparameters we set in notebook (with some defaults)
batch_size = hyperparameters.get('batch_size', 128)
epochs = hyperparameters.get('epochs', 100)
momentum = hyperparameters.get('momentum', 0.9)
wd = hyperparameters.get('wd', 0.0005)
lr_multiplier = hyperparameters.get('lr_multiplier', 1.)
lr_schedule = {
0: 0.01 * lr_multiplier,
5: 0.1 * lr_multiplier,
95: 0.01 * lr_multiplier,
110: 0.001 * lr_multiplier
}
#Inform properly the algorithm about hardware choice
if len(hosts) == 1:
kvstore = 'device' if num_gpus > 0 else 'local'
else:
kvstore = 'dist_device_sync'
part_index = 0
for i, host in enumerate(hosts):
if host == current_host:
part_index = i
break
#Set-up the right context
ctx = [mx.gpu(i) for i in range(num_gpus)] if num_gpus > 0 else [mx.cpu()]
#Set-up the right batch size
batch_size *= max(1, len(ctx))
# Prepare the data
given_path = channel_input_dirs['training']
train_data, valid_data = Cifar10(batch_size=batch_size,
data_shape=(3, 32, 32),
padding=4,
padding_value=0,
normalization_type="channel",
given_path=given_path,
num_parts=len(hosts),
part_index=part_index,
num_cpus=num_cpus).return_dataloaders()
# Create the model
model = WideResNet(num_classes=10, depth=40,
k=2) # We will use WideResNet40-2
model.initialize(mx.init.Xavier(rnd_type='gaussian',
factor_type='out',
magnitude=2),
ctx=ctx)
optimizer = mx.optimizer.SGD(learning_rate=lr_schedule[0],
rescale_grad=1.0 / batch_size,
momentum=momentum,
wd=wd)
trainer = mx.gluon.Trainer(params=model.collect_params(),
optimizer=optimizer,
kvstore=kvstore)
train_metric = mx.metric.Accuracy()
criterion = mx.gluon.loss.SoftmaxCrossEntropyLoss()
early_stopping_criteria = lambda e: e >= 0.94
start_time = time.time()
for epoch in range(epochs):
epoch_tick = time.time()
train_metric.reset()
# update learning rate according to the schedule
if epoch in lr_schedule.keys():
trainer.set_learning_rate(lr_schedule[epoch])
logging.info("Epoch {}, Changed learning rate.".format(epoch))
logging.info('Epoch {}, Learning rate={}'.format(
epoch, trainer.learning_rate))
for batch_idx, (data, label) in enumerate(train_data):
batch_tick = time.time()
batch_size = data.shape[0]
data = mx.gluon.utils.split_and_load(data,
ctx_list=ctx,
batch_axis=0)
label = mx.gluon.utils.split_and_load(label,
ctx_list=ctx,
batch_axis=0)
y_pred = []
losses = []
with mx.autograd.record():
for x_part, y_true_part in zip(data, label):
y_pred_part = model(x_part)
loss = criterion(y_pred_part, y_true_part)
# Calculate loss on each partition of data.
# Store the losses and do backward after we have done forward on all GPUs,
# for better performance on multiple GPUs.
losses.append(loss)
y_pred.append(y_pred_part)
for loss in losses:
loss.backward()
trainer.step(batch_size)
train_metric.update(label, y_pred)
# log training accuracy
_, trn_acc = train_metric.get()
logging.info('Epoch {}, Training accuracy={}'.format(epoch, trn_acc))
# log validation accuracy
_, val_acc = test(ctx, model, valid_data)
logging.info('Epoch {}, Validation accuracy={}'.format(epoch, val_acc))
logging.info('Epoch {}, Duration={}'.format(epoch,
time.time() - epoch_tick))
if early_stopping_criteria:
if early_stopping_criteria(val_acc):
logging.info(
"Epoch {}, Reached early stopping target, stopping training."
.format(epoch))
break
return model
num_epochs = 50
learning_rate = 0.001
k = 76 # Give an even number
sequence_length = k + 1
w = 0.5
window_size = 1
test_size = 1000
train_size = 4000
from_checkpoint = True
if __name__ == '__main__':
compose = transforms.Compose([ToTensor(), Normalize()])
print("Extracting Dataset")
transformed_dataset = Cifar10(
cropped=root + 'dataset/cifar-10/cropped_400.npy',
ground_truth=root + 'dataset/cifar-10/original_400.npy',
size=train_size + test_size,
transform=compose)
print("Done")
train_set, test_set = torch.utils.data.random_split(
transformed_dataset, [train_size, test_size])
train_loader, test_loader = DataLoader(train_set, shuffle=False), \
DataLoader(test_set, shuffle=False)
if from_checkpoint:
model = RNN(input_size, hidden_size, num_layers,
output_size).to(device)
model.load_state_dict(
torch.load('models/model30_0.005.pt', map_location=device))
else:
model = RNN(input_size, hidden_size, num_layers,
output_size).to(device)
def train():
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
# TODO: Write your training code here
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# - Create loss and training op
# - Run training
# AS IT WILL TAKE VERY LONG ON CIFAR10 DATASET TO TRAIN
# YOU SHOULD USE tf.train.Saver() TO SAVE YOUR MODEL AFTER TRAINING
# AT TEST TIME, LOAD THE MODEL AND RUN TEST ON THE TEST SET
cifar_train = Cifar10(test=False, shuffle=False, one_hot=True)
cifar10_test = Cifar10(batch_size=100,
one_hot=False,
test=True,
shuffle=False)
X_train = cifar_train._images
y_train = cifar_train._labels
EPOCHS = 15
BATCH_SIZE = 128
dropoutprob = tf.placeholder(tf.float32)
training = True
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
y = tf.placeholder(tf.int32, (None, 10))
#one_hot_y = tf.one_hot(y, 10)
rate = 0.0005
#dropoutprob = 0
logits = net(x, training, dropoutprob)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y)
loss_operation = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=rate)
grads_and_vars = optimizer.compute_gradients(
loss_operation, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
training_operation = optimizer.apply_gradients(grads_and_vars)
add_gradient_summaries(grads_and_vars)
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))
accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction,
tf.float32))
def evaluate(X_data, y_data):
num_examples = len(X_data)
total_accuracy = 0
sess = tf.get_default_session()
for offset in range(0, num_examples, BATCH_SIZE):
batch_x, batch_y = X_data[offset:offset +
BATCH_SIZE], y_data[offset:offset +
BATCH_SIZE]
accuracy = sess.run(accuracy_operation,
feed_dict={
x: batch_x,
y: batch_y,
dropoutprob: 1
})
total_accuracy += (accuracy * len(batch_x))
return total_accuracy / num_examples
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
num_examples = len(X_train)
print("Training...")
print()
for i in range(EPOCHS):
X_train, y_train = shuffle(X_train, y_train)
for offset in range(0, num_examples, BATCH_SIZE):
end = offset + BATCH_SIZE
batch_x, batch_y = X_train[offset:end], y_train[offset:end]
#print("Print batch_x...")
#print(batch_x)
#print("Print batch_y...")
#print(batch_y)
sess.run(training_operation,
feed_dict={
x: batch_x,
y: batch_y,
dropoutprob: 0.5
})
validation_accuracy = evaluate(X_train, y_train)
print("EPOCH {} ...".format(i + 1))
print("Validation Accuracy = {:.3f}".format(validation_accuracy))
try:
saver
except NameError:
saver = tf.train.Saver()
save_path = 'ci10model'
if not os.path.isabs(save_path):
save_path = os.path.abspath(os.path.join(os.getcwd(), save_path))
saver.save(sess, save_path)
print("Model saved")
train_op = model.optimizer.apply_gradients(grads_and_vars)
zipped_val = sess.run(grads_and_vars, feed_dict=feed_dict)
for rsl, tensor in zip(zipped_val, grads_and_vars):
print('-----------------------------------------')
print(
'name', tensor[0].name.replace('/tuple/control_dependency_1:0',
'').replace('gradients/', ''))
print('gradient', rsl[0])
print('value', rsl[1])
if __name__ == "__main__":
model = AlexNet(LEARNING_RATE)
data = Cifar10()
modifier = ImageAugmentation()
with tf.Session(graph=model.graph) as sess:
model.init.run()
for epoch in range(NUM_EPOCHS):
data.reset_pos()
avg_loss = 0
model.Training = True
data.shuffle_data()
while data.pos < data.N:
batch_x, batch_y = data.get_batch(BATCH_SIZE)
batch_x = modifier.fit_batch(batch_x, model.get_size())
batch_x = modifier.augment_batch(batch_x)
def train(batch_size=128, num_epochs=100):
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# - Create loss and training op
# - Run training
# input Tensors
input_x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
input_y = tf.placeholder(tf.float32, shape=[None, 10], name='y')
dropout_kept_prob = tf.placeholder(tf.float32, name="dropout_kept_prob")
is_training = tf.placeholder(tf.bool)
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
cifar10_train = Cifar10(batch_size=batch_size,
one_hot=True,
test=False,
shuffle=True)
num_train = cifar10_train.num_samples
num_batches = math.ceil(num_train / batch_size)
# ConvNet
sess = tf.Session()
logits = Net(input_x, dropout_kept_prob, is_training)
# logits = ResNet(input_x, is_training, dropout_kept_prob, resnet_size=32)
# Loss Func + Regularization Loss
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=input_y)
loss_operation = tf.reduce_mean(cross_entropy)
# Exponential Learning Rate Decay
initial_learning_rate = 0.01
learning_rate = tf.train.exponential_decay(initial_learning_rate,
global_step,
decay_steps=num_batches *
num_epochs,
decay_rate=0.95,
staircase=True)
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss_operation, global_step=global_step)
# Prediction, Saver
correct_predictions = tf.equal(tf.argmax(logits, 1), tf.argmax(input_y, 1))
prediction = tf.argmax(logits, 1)
accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"),
name="accuracy")
saver = tf.train.Saver(max_to_keep=5)
checkpoint_dir = os.path.curdir
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
# Training Loop
init = tf.global_variables_initializer()
sess.run(init)
for epoch in range(num_epochs):
# num_train = cifar10_train.num_samples
# num_batches = math.ceil(num_train / batch_size)
for iteration in range(num_batches):
batch_x, batch_y = cifar10_train.get_next_batch()
_, step, acc, l = sess.run(
[train_op, global_step, accuracy, loss_operation],
feed_dict={
input_x: batch_x,
input_y: batch_y,
is_training: True,
dropout_kept_prob: 0.8
})
print("step:", step, "epoch:", epoch + 1, "acc:", acc, "loss", l)
if step % 100 == 0:
predicted_cifar10_test_labels = []
cifar10_test = Cifar10(batch_size=batch_size,
one_hot=False,
test=True,
shuffle=False)
cifar10_test_images, cifar10_test_labels = cifar10_test._images, cifar10_test._labels
num_test_batches = math.ceil(
len(cifar10_test_images) / batch_size)
for iteration in range(num_test_batches):
if iteration != (num_test_batches - 1):
x_test_batch = cifar10_test_images[:batch_size]
cifar10_test_images = cifar10_test_images[batch_size:]
else:
x_test_batch = cifar10_test_images[:len(
cifar10_test_images)]
cifar10_test_images = cifar10_test_images[
len(cifar10_test_images):]
pred = sess.run(
[prediction],
feed_dict={
input_x: x_test_batch,
is_training: False,
dropout_kept_prob: 1.0
})[0]
predicted_cifar10_test_labels += pred.tolist()
correct_predict = (cifar10_test_labels.flatten() == np.array(
predicted_cifar10_test_labels).flatten()).astype(
np.int32).sum()
incorrect_predict = len(cifar10_test_labels) - correct_predict
acc_test = float(correct_predict) / len(cifar10_test_labels)
print("\nTesting Result:", "step:", step, "epoch:", epoch + 1,
"acc:", acc_test, "\n")
max_acc = 0
if acc_test > 0.8:
if acc_test > max_acc:
max_acc = acc_test
path = saver.save(sess,
checkpoint_prefix,
global_step=global_step)
print("Saved model checkpoint to {}\n".format(path))
return
('epochs', args.epochs),
('batch_size', args.batch_size),
('base_lr', args.base_lr),
('max_lr', args.max_lr),
('cycle_epoch', args.cycle_epoch),
('cycle_ratio', args.cycle_ratio),
('num_classes', args.num_classes)])
return config
config = parse_args()
### call data ###
cifar10 = Cifar10()
n_samples = cifar10.num_examples
n_test_samples = cifar10.num_test_examples
### call models ###
model = VGG16(config['num_classes'])
### make folder ###
mother_folder = config['model_name']
try:
os.mkdir(mother_folder)
except OSError:
pass
# TODO: Write your training code here # - Create placeholder for inputs, training boolean, dropout keep probablity # - Construct your model # - Create loss and training op # - Run training # AS IT WILL TAKE VERY LONG ON CIFAR10 DATASET TO TRAIN # YOU SHOULD USE tf.train.Saver() TO SAVE YOUR MODEL AFTER TRAINING # AT TEST TIME, LOAD THE MODEL AND RUN TEST ON THE TEST SET <<<<<<< HEAD x = tf.placeholder(tf.float32, (None, 32, 32, 3)) y = tf.placeholder(tf.int32, (None)) #one_hot_y = tf.one_hot(y,10) '''data preparation''' cifar10_train = Cifar10(batch_size = 100, one_hot = True, test = False, shuffle = True) cifar10_test = Cifar10(batch_size = 100, one_hot = False, test = True, shuffle = False) test_images, test_labels = cifar10_test.images, cifar10_test.labels lr = 0.0001 logits = net(x, True, 0.7) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = y) loss = tf.reduce_mean(cross_entropy) optimizer = tf.train.AdamOptimizer(learning_rate = lr) grads_and_vars = optimizer.compute_gradients(loss, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)) training_operation = optimizer.apply_gradients(grads_and_vars) #training_operation = optimizer.minimize(grads_and_vars) #train = tf.train.AdamOptimizer.minimize(loss) '''create summary'''
def train():
# Always use tf.reset_default_graph() to avoid error
tf.reset_default_graph()
# TODO: Write your training code here
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# - Create loss and training op
# - Run training
# AS IT WILL TAKE VERY LONG ON CIFAR10 DATASET TO TRAIN
# YOU SHOULD USE tf.train.Saver() TO SAVE YOUR MODEL AFTER TRAINING
# AT TEST TIME, LOAD THE MODEL AND RUN TEST ON THE TEST SET
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None))
#one_hot_y = tf.one_hot(y,10)
'''data preparation'''
cifar10_train = Cifar10(batch_size=100,
one_hot=True,
test=False,
shuffle=True)
cifar10_test = Cifar10(batch_size=100,
one_hot=False,
test=True,
shuffle=False)
test_images, test_labels = cifar10_test.images, cifar10_test.labels
lr = 0.003
logits = net(x, True, 0.3)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y)
loss = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
grads_and_vars = optimizer.compute_gradients(
loss, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
training_operation = optimizer.apply_gradients(grads_and_vars)
#training_operation = optimizer.minimize(grads_and_vars)
#train = tf.train.AdamOptimizer.minimize(loss)
'''create summary'''
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/histogram", var)
add_gradient_summaries(grads_and_vars)
tf.summary.scalar('loss_operation', loss)
merged_summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter('logs/')
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))
accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction,
tf.float32))
saver = tf.train.Saver(max_to_keep=5)
sess = tf.Session()
try:
print(
"================Trying to restore last checkpoint ...================"
)
saver.restore(sess, tf.train.latest_checkpoint('ckpt'))
print("================Checkpoint restored .================")
except:
print(
"================Failed to find last check point================")
sess.run(tf.global_variables_initializer())
'''training'''
start_time = time.time()
with sess:
print("Training")
global_step = 0
for i in range(20):
for offset in range(0, 500):
batch_x, batch_y = cifar10_train.get_next_batch()
_, summaries = sess.run(
[training_operation, merged_summary_op],
feed_dict={
x: batch_x,
y: batch_y
})
if global_step % 100 == 1:
_loss = sess.run(loss, feed_dict={x: batch_x, y: batch_y})
print(
"steps: ", global_step, "|time consume: %.2f" %
((time.time() - start_time) / 3600),
"hours. || Current Loss: ", _loss)
summary_writer.add_summary(summaries,
global_step=global_step)
global_step += 1
'''
#validation_accuracy = evaluate(test_images, test_labels, accuracy_operation)
total_accuracy = 0
test_x, test_y = cifar10_test.get_next_batch()
accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y})
total_accuracy += (accuracy * len(batch_x))
'''
training_accuracy = evaluate(batch_x, batch_y, accuracy_operation,
x, y)
print("sets: ", i)
print("accuracy: ", training_accuracy * 100, " % ")
print()
saver.save(sess, 'ckpt/netCheckpoint', global_step=i)
def compute_score(acc, min_thres, max_thres):
if acc <= min_thres:
base_score = 0.0
elif acc >= max_thres:
base_score = 100.0
else:
base_score = float(acc - min_thres) / (max_thres - min_thres) \
* 100
return base_score
if __name__ == '__main__':
TRAIN = True
if TRAIN:
train()
cifar10_test = Cifar10(test=True, shuffle=False, one_hot=False)
cifar10_test_images, cifar10_test_labels = cifar10_test._images, cifar10_test._labels
start = timeit.default_timer()
np.random.seed(0)
predicted_cifar10_test_labels = test(cifar10_test_images)
np.random.seed()
stop = timeit.default_timer()
run_time = stop - start
correct_predict = (cifar10_test_labels.flatten() ==
predicted_cifar10_test_labels.flatten()).astype(
np.int32).sum()
incorrect_predict = len(cifar10_test_labels) - correct_predict
accuracy = float(correct_predict) / len(cifar10_test_labels)
print('Acc: {}. Testing took {}s.'.format(accuracy, stop - start))
def train():
# Always use tf.reset_default_graph() to avoid error
# TODO: Write your training code here
# - Create placeholder for inputs, training boolean, dropout keep probablity
# - Construct your model
# - Create loss and training op
# - Run training
# AS IT WILL TAKE VERY LONG ON CIFAR10 DATASET TO TRAIN
# YOU SHOULD USE tf.train.Saver() TO SAVE YOUR MODEL AFTER TRAINING
# AT TEST TIME, LOAD THE MODEL AND RUN TEST ON THE TEST SET
tf.reset_default_graph()
#tf.reset_default_graph()
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None, 10))
is_training = True
dropout_keep_probablity = 0.95
rate = 0.0007
EPOCHS = 13
cifar10_train = Cifar10(batch_size=BATCH_SIZE,
one_hot=True,
test=False,
shuffle=False)
cifar10_train_images, cifar10_train_labels = cifar10_train._images, cifar10_train._labels # Get all images and labels of the test set.
cifar10_train_images = tf.cast(cifar10_train_images, tf.float32)
#print(batch_y.shape);
logits = net(x, is_training, dropout_keep_probablity)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y)
loss_operation = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=rate)
grads_and_vars = optimizer.compute_gradients(
loss_operation, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
training_operation = optimizer.apply_gradients(grads_and_vars)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/histogram", var)
add_gradient_summaries(grads_and_vars)
tf.summary.scalar('loss_operation', loss_operation)
merged_summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter('logs/')
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
#sess.run(conv1_W.initializer)
num_examples = cifar10_train_images.get_shape().as_list()[0]
global_total = EPOCHS * (num_examples / BATCH_SIZE)
#training_operation, merged_summary_op = train_lenet()
print("Training...")
global_step = 0
for i in range(EPOCHS):
#cifar10_train_images, cifar10_train_labels = shuffle(cifar10_train_images, cifar10_train_labels)
for offset in range(0, num_examples, BATCH_SIZE):
print(
str(global_step / global_total * 100) + "%" +
" done training")
#print(offset)
batch_x, batch_y = cifar10_train.get_next_batch()
_, summaries = sess.run(
[training_operation, merged_summary_op],
feed_dict={
x: batch_x,
y: batch_y
})
if global_step % 100 == 1:
summary_writer.add_summary(summaries,
global_step=global_step)
global_step += 1
saver.save(sess, 'ckpt/spencenet', global_step=i)
rate /= 2 # anneal learning rate
print("Learning rate: " + str(rate))
print("Model saved")
sess.close()