def _load_data(self):
self.config_data['batch_size'] = self.batch_size
self.config_data['patch_type_x'] = self.config_net['patch_type_x']
self.config_data['patch_type_y'] = self.config_net['patch_type_y']
self.config_data['patch_shape_x'] = self.config_net['patch_shape_x']
self.config_data['patch_shape_y'] = self.config_net['patch_shape_y']
if(self.mode == 'train'):
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
self.data_agent = ImageLoader(self.config_data)
# create an reinitializable iterator given the dataset structure
train_dataset = self.data_agent.get_dataset('train')
valid_dataset = self.data_agent.get_dataset('valid')
train_iterator = Iterator.from_structure(train_dataset.output_types,
train_dataset.output_shapes)
valid_iterator = Iterator.from_structure(valid_dataset.output_types,
valid_dataset.output_shapes)
self.next_train_batch = train_iterator.get_next()
self.next_valid_batch = valid_iterator.get_next()
# Ops for initializing the two different iterators
self.train_init_op = train_iterator.make_initializer(train_dataset)
self.valid_init_op = valid_iterator.make_initializer(valid_dataset)
else:
self.data_agent = ImageLoader(self.config_data)
self.test_dataset = self.data_agent.get_dataset('test')
def generate_input_pipline(tfrecords_path, filenames, batch_size,
test_batch_size, capacity, _parse_function_train,
_parse_function_test):
'''
:param sess:
:param tfrecords_path:
:param filenames:
:param epochs:
:param batch_size:
:param capacity:
:param _parse_function: tf dataset api parse function, you should provide this function by yourself
:return:
'''
file_paths = [os.path.join(tfrecords_path, name) for name in filenames]
train_dataset = tf.contrib.data.TFRecordDataset(file_paths[0])
train_dataset = train_dataset.map(_parse_function_train)
train_dataset = train_dataset.repeat(1)
train_dataset = train_dataset.batch(batch_size)
train_dataset = train_dataset.shuffle(buffer_size=capacity)
test_dataset = tf.contrib.data.TFRecordDataset(file_paths[1])
test_dataset = test_dataset.map(_parse_function_test)
test_dataset = test_dataset.repeat(1)
test_dataset = test_dataset.batch(test_batch_size)
iterator = Iterator.from_structure(train_dataset.output_types,
train_dataset.output_shapes)
next_element = iterator.get_next()
training_init_op = iterator.make_initializer(train_dataset)
test_init_op = iterator.make_initializer(test_dataset)
return next_element, training_init_op, test_init_op
def get_dataset_ops(data_train,
data_val,
batch_size,
train_size,
val_size,
shuffle=True):
"""
"""
# shuffle the dataset and create batches
if shuffle:
data_train = data_train.shuffle(train_size)
data_val = data_val.shuffle(val_size)
data_train = data_train.batch(batch_size)
data_val = data_val.batch(batch_size)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(data_train.output_types,
data_train.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
init_op_train = iterator.make_initializer(data_train)
init_op_val = iterator.make_initializer(data_val)
return init_op_train, init_op_val, next_batch
def tf_create_iterator(dataset, batch_size):
""" """
dataset_prefix = os.path.join(SNPX_DATASET_ROOT, dataset, dataset)
train_rec_file = dataset_prefix + "_train.tfrecords"
val_rec_file = dataset_prefix + "_val.tfrecords"
# Create the training dataset object
train_set = TFRecordDataset(train_rec_file)
train_set = train_set.map(tf_parse_record,
num_threads=4,
output_buffer_size=1000)
train_set = train_set.shuffle(buffer_size=50000)
train_set = train_set.batch(batch_size)
# Create the validation dataset object
val_set = TFRecordDataset(val_rec_file)
val_set = val_set.map(tf_parse_record)
val_set = val_set.batch(batch_size)
# Create a reinitializable iterator from both datasets
iterator = Iterator.from_structure(train_set.output_types,
train_set.output_shapes)
train_init_op = iterator.make_initializer(train_set)
val_init_op = iterator.make_initializer(val_set)
iter_op = iterator.get_next()
return train_init_op, val_init_op, iter_op
def validate(model_def):
"""
Validate my alexnet implementation
Args:
model_def: the model class/definition
"""
img_dir = os.path.join('.', 'images')
images = []
print "loading images ..."
files = fnmatch.filter(os.listdir(img_dir), '*.jpeg')
for f in files:
print "> " + f
img_file = tf.read_file(os.path.join(img_dir, f))
img_decoded = tf.image.decode_jpeg(img_file, channels=3)
img_processed = model_def.image_prep.preprocess_image(
image=img_decoded,
output_height=model_def.image_size,
output_width=model_def.image_size,
is_training=False)
images.append(img_processed)
# create TensorFlow Iterator object
images = Dataset.from_tensors(images)
iterator = Iterator.from_structure(images.output_types,
images.output_shapes)
next_element = iterator.get_next()
iterator_init_op = iterator.make_initializer(images)
# create the model and get scores (pipe to softmax)
model = model_def(next_element)
scores = model.get_final_op()
softmax = tf.nn.softmax(scores)
print 'start validation ...'
with tf.Session() as sess:
# Initialize all variables and the iterator
sess.run(tf.global_variables_initializer())
sess.run(iterator_init_op)
# Load the pretrained weights into the model
model.load_initial_weights(sess)
# run the graph
probs = sess.run(softmax)
if model_def is ResNetV2:
probs = prep_resnet_results(probs)
# sometime we have an offset
offset = len(class_names) - len(probs[0])
# print the results
for prob in probs:
best_index = np.argmax(prob)
print "> " + class_names[best_index +
offset] + " -> %.4f" % prob[best_index]
def init_tfdata(self, batch_size, main_dir, resize_shape, mode='train'):
self.data_session = tf.Session()
print("Creating the iterator for training data")
with tf.device('/cpu:0'):
segdl = SegDataLoader(main_dir, batch_size, (resize_shape[0], resize_shape[1]), resize_shape,
# * 2), resize_shape,
'data/cityscapes_tfdata/train.txt')
iterator = Iterator.from_structure(segdl.data_tr.output_types, segdl.data_tr.output_shapes)
next_batch = iterator.get_next()
self.init_op = iterator.make_initializer(segdl.data_tr)
self.data_session.run(self.init_op)
print("Loading Validation data in memoryfor faster training..")
self.val_data = {'X': np.load(self.args.data_dir + "X_val.npy"),
'Y': np.load(self.args.data_dir + "Y_val.npy")}
# self.crop()
# import cv2
# cv2.imshow('crop1', self.val_data['X'][0,:,:,:])
# cv2.imshow('crop2', self.val_data['X'][1,:,:,:])
# cv2.imshow('seg1', self.val_data['Y'][0,:,:])
# cv2.imshow('seg2', self.val_data['Y'][1,:,:])
# cv2.waitKey()
self.val_data_len = self.val_data['X'].shape[0] - self.val_data['X'].shape[0] % self.args.batch_size
# self.num_iterations_validation_per_epoch = (
# self.val_data_len + self.args.batch_size - 1) // self.args.batch_size
self.num_iterations_validation_per_epoch = self.val_data_len // self.args.batch_size
print("Val-shape-x -- " + str(self.val_data['X'].shape) + " " + str(self.val_data_len))
print("Val-shape-y -- " + str(self.val_data['Y'].shape))
print("Num of iterations on validation data in one epoch -- " + str(self.num_iterations_validation_per_epoch))
print("Validation data is loaded")
return next_batch, segdl.data_len
def train(self):
tf.global_variables_initializer().run()
could_load = 0
# load ckpt
#could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
tr_data = ImageDataGenerator(
FLAGS.train_file,
img_size=[self.DATA_HEIGHT, self.DATA_WIDTH],
label_size=[self.LABEL_HEIGHT, self.LABEL_WIDTH],
batch_size=self.BATCH_SIZE,
shuffle=True)
print('[Model] Data done ...')
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
batch_idxs = int(np.floor(tr_data.data_size / self.BATCH_SIZE))
print(tr_data.data_size)
for epoch in range(self.num_epochs): #
print('[Ves9] Begin epoch ', epoch, '...')
for iteration in range(batch_idxs):
start_time = time.time()
batch_images, batch_gts = self.sess.run(next_batch)
print(batch_images[0].shape, batch_images[0])
print(batch_gts[0].shape, batch_gts[0])
'''
_cost, result, acc_, _ = self.sess.run(
[self.cost, self.output, self.accuracy, self.train_op],
feed_dict={self.images: batch_images,
self.ground_truth: batch_gts})
'''
tm = time.time() - start_time
'''
if np.mod(iteration, 5000) == 0 and iteration != 0:
print('[INFO] Save checkpoint...')
self.save(self.checkpoint_dir, iteration)
if (iteration % 50 == 0):
print(iteration, ': ', '%.2f' % (tm), _cost)
'''
print('[INFO] Save the epoch checkpoint ... ')
def eval_data_generator(file_path, batch_size, sub_img_folder):
temp = imageDataGenerator(file_path, batch_size, sub_img_folder)
iterator = Iterator.from_structure(temp.data.output_types,
temp.data.output_shapes)
next_batch = iterator.get_next()
dataset_init_op = iterator.make_initializer(temp.data)
with tf.Session() as sess:
sess.run(dataset_init_op)
if sub_img_folder == '/test/':
img, label = sess.run(next_batch)
img, label = sess.run(next_batch)
else:
img, label = sess.run(next_batch)
# print(img.shape, label)
# imgplot = plt.imshow(img[0])
# plt.show()
return img, label
def confusion_matrix(val_file, num_classes, checkpoint_path):
with tf.device('/cpu:0'):
val_data = ImageDataGenerator(val_file, mode='inference', shuffle=True)
iterator = Iterator.from_structure(val_data.data.output_types,
val_data.data.output_shapes)
next_batch = iterator.get_next()
val_init_op = iterator.make_initializer(val_data.data)
x = tf.placeholder(tf.float32,
shape=[1, None, None, 3],
name="input_image")
keep_prob = tf.placeholder(tf.float32)
model = VGG16_FCN(x, num_classes, keep_prob)
cm = np.zeros([num_classes, num_classes], dtype=np.uint32)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(val_init_op)
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
for step in range(val_data.data_size):
image, label = sess.run(next_batch)
result_label = sess.run(model.pred_up,
feed_dict={
x: image,
keep_prob: 1.0
})
cml = confusion_matrix_label(result_label[0], label[0],
num_classes)
cm += cml
return cm
def input_pipeline(data_dir):
BATCH_SIZE = 16
filenames = [
join(data_dir, filename) for filename in os.listdir(data_dir)
if isfile(join(data_dir, filename))
]
np.random.shuffle(filenames)
train_images = tf.constant(filenames)
tr_data = Dataset.from_tensor_slices(train_images)
tr_data = tr_data.map(input_parser).map(distort_input)
tr_data = tr_data.batch(16)
# create TensorFlow Iterator object
iterator = Iterator.from_structure(tr_data.output_types,
tr_data.output_shapes)
training_init_op = iterator.make_initializer(tr_data)
next_element = iterator.get_next()
with tf.Session() as sess:
# initialize the iterator on the training data
sess.run(training_init_op)
# get each element of the training dataset until the end is reached
while True:
try:
elem = sess.run(next_element)
print(elem)
except tf.errors.OutOfRangeError:
print("End of training dataset.")
break
def train(model, saver, sess, exp_string, train_file_list, test_file, resume_itr=0):
SUMMARY_INTERVAL = 100
SAVE_INTERVAL = 10000
PRINT_INTERVAL = 100
TEST_PRINT_INTERVAL = 100
dropout_rate = 0.5
if FLAGS.log:
train_writer = tf.summary.FileWriter(FLAGS.logdir + '/' + exp_string, sess.graph)
print('Done initializing, starting training.')
prelosses, postlosses = [], []
num_classes = FLAGS.num_classes # for classification, 1 otherwise
# Defining data loaders
with tf.device('/cpu:0'):
tr_data_list = []
train_iterator_list = []
train_next_list = []
for i in range(len(train_file_list)):
tr_data = ImageDataGenerator(train_file_list[i],
dataroot=FLAGS.dataroot,
mode='training',
batch_size=FLAGS.meta_batch_size,
num_classes=num_classes,
shuffle=True)
tr_data_list.append(tr_data)
train_iterator_list.append(Iterator.from_structure(tr_data_list[i].data.output_types,
tr_data_list[i].data.output_shapes))
train_next_list.append(train_iterator_list[i].get_next())
test_data = ImageDataGenerator(test_file,
dataroot=FLAGS.dataroot,
mode='inference',
batch_size=1,
num_classes=num_classes,
shuffle=False)
test_iterator = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
test_next_batch = test_iterator.get_next()
# create an reinitializable iterator given the dataset structure
# Ops for initializing different iterators
training_init_op = []
train_batches_per_epoch = []
for i in range(len(train_file_list)):
training_init_op.append(train_iterator_list[i].make_initializer(tr_data_list[i].data))
train_batches_per_epoch.append(int(np.floor(tr_data_list[i].data_size/FLAGS.meta_batch_size)))
test_init_op = test_iterator.make_initializer(test_data.data)
test_batches_per_epoch = int(np.floor(test_data.data_size / 1))
# Training begins
for itr in range(resume_itr, FLAGS.pretrain_iterations + FLAGS.metatrain_iterations):
feed_dict = {}
# Sampling training and test tasks
num_training_tasks = len(train_file_list)
num_meta_train = num_training_tasks-1
num_meta_test = num_training_tasks-num_meta_train
# Randomly choosing meta train and meta test domains
task_list = np.random.permutation(num_training_tasks)
meta_train_index_list = task_list[:num_meta_train]
meta_test_index_list = task_list[num_meta_train:]
for i in range(len(train_file_list)):
if itr%train_batches_per_epoch[i] == 0:
sess.run(training_init_op[i])
# Populating input tensors
# Sampling meta train data
for i in range(num_meta_train):
task_ind = meta_train_index_list[i]
if i == 0:
inputa, labela = sess.run(train_next_list[task_ind])
else:
inp_tmp, lab_tmp = sess.run(train_next_list[task_ind])
inputa = np.concatenate((inputa, inp_tmp), axis=0)
labela = np.concatenate((labela, lab_tmp), axis=0)
inputs_all = list(zip(inputa, labela))
shuffle(inputs_all)
inputa, labela = zip(*inputs_all)
# Sampling meta test data
for i in range(num_meta_test):
task_ind = meta_test_index_list[i]
if i == 0:
inputb, labelb = sess.run(train_next_list[task_ind])
else:
inp_tmp, lab_tmp = sess.run(train_next_list[task_ind])
inputb = np.concatenate((inputb, inp_tmp), axis=0)
labelb = np.concatenate((labelb, lab_tmp), axis=0)
feed_dict = {model.inputa: inputa, model.inputb: inputb, model.labela: labela, model.labelb: labelb, model.KEEP_PROB: dropout_rate}
if itr<FLAGS.pretrain_iterations:
input_tensors = [model.pretrain_op]
else:
input_tensors = [model.metatrain_op]
if (itr % SUMMARY_INTERVAL == 0 or itr % PRINT_INTERVAL == 0):
input_tensors.extend([model.summ_op, model.total_loss1, model.total_losses2[FLAGS.num_updates-1]])
input_tensors.extend([model.total_accuracy1, model.total_accuracies2[FLAGS.num_updates-1]])
result = sess.run(input_tensors, feed_dict)
if itr % SUMMARY_INTERVAL == 0:
prelosses.append(result[-2])
if FLAGS.log:
train_writer.add_summary(result[1], itr)
postlosses.append(result[-1])
if (itr!=0) and itr % PRINT_INTERVAL == 0:
print_str = 'Iteration ' + str(itr - FLAGS.pretrain_iterations)
print_str += ': ' + str(np.mean(prelosses)) + ', ' + str(np.mean(postlosses))
print(print_str)
prelosses, postlosses = [], []
if (itr!=0) and itr % SAVE_INTERVAL == 0:
saver.save(sess, FLAGS.logdir + '/' + exp_string + '/model' + str(itr))
# Testing periodically
if itr % TEST_PRINT_INTERVAL == 0:
test_acc = 0.
test_loss = 0.
test_count = 0
sess.run(test_init_op)
for it in range(test_batches_per_epoch):
test_input, test_label = sess.run(test_next_batch)
feed_dict = {model.test_input: test_input, model.test_label: test_label, model.KEEP_PROB: 1.}
input_tensors = [model.test_loss, model.test_acc]
result = sess.run(input_tensors, feed_dict)
test_loss += result[0]
test_acc += result[1]
test_count += 1
print('Validation results: Iteration %d, Loss: %f, Accuracy: %f' %(itr, test_loss/test_count, test_acc/test_count))
saver.save(sess, FLAGS.logdir + '/' + exp_string + '/model' + str(itr))
batch_size = FLAGS.bs
# Network params
num_classes = FLAGS.num_class
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
test_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator_test = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
images_batch_test, labels_batch_test = iterator_test.get_next()
# Ops for initializing the two different iterators
test_init_op = iterator_test.make_initializer(test_data.data)
images_batch = tf.concat([images_batch_test, images_batch_test], axis=0)
y = tf.concat([labels_batch_test, labels_batch_test], axis=0)
with slim.arg_scope(resnet.resnet_arg_scope()):
Gap5, CONV5, net, _ = resnet.resnet_v2_50(images_batch, is_training=False)
net = tf.nn.dropout(net, 1.0)
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
num_classes=num_classes)
val_text_k = TextualDataGenerator(valid_text_k,valid_k_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
tes_text_i = TextualDataGenerator(test_text_i,test_i_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
tes_text_j = TextualDataGenerator(test_text_j,test_j_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
tes_text_k = TextualDataGenerator(test_text_k,test_k_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
i_text_ite = Iterator.from_structure(tra_text_i.data.output_types,
tra_text_i.data.output_shapes)
j_text_ite = Iterator.from_structure(tra_text_j.data.output_types,
tra_text_j.data.output_shapes)
k_text_ite = Iterator.from_structure(tra_text_k.data.output_types,
tra_text_k.data.output_shapes)
next_batch = iterator.get_next()
next_batchi = i_text_ite.get_next()
next_batchj = j_text_ite.get_next()
next_batchk = k_text_ite.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
def alex_tr_pred(setname='Agora',
threshold='10',
step_='pseudo_pairing',
root_dir='/media/intel/m2/train_test_data/',
tmp_file='pretrained_models/Alexnet_tmpfile'):
# Loading class names
tr_test_path = os.path.join(root_dir, step_, setname, str(threshold),
'labels')
with open(os.path.join(tr_test_path, 'class_name.json')) as fp:
class_name = loadjson(fp)
# Path to the textfiles for the training and testing set
train_file = os.path.join(tr_test_path, 'train.txt')
test_file = os.path.join(tr_test_path, 'test.txt')
# Learning params
learning_rate = 0.01
num_epochs = 30
batch_size = 128
# Network params
dropout_rate = 0.5
num_classes = len(class_name)
train_layers = ['fc8', 'fc7', 'fc6']
# How often we want to write the tf.summary data to disk
display_step = 20
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = os.path.join(tmp_file, "tensorboard")
checkpoint_path = os.path.join(tmp_file, "checkpoints")
try:
os.makedirs(filewriter_path)
except:
pass
try:
os.makedirs(checkpoint_path)
except:
pass
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
test_data = ImageDataGenerator(test_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
test_init_op = iterator.make_initializer(test_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
score = model.fc8
softmax = tf.nn.softmax(score)
# List of trainable variables of the layers we want to train
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size))
test_batches_per_epoch = int(np.ceil(test_data.data_size / batch_size))
# Start Session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("Epoch number: %d" % (epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
sess.run(train_op,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: dropout_rate
})
# Generate summary with the current batch of data and write to file
if step % display_step == 0:
s = sess.run(merged_summary,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
writer.add_summary(s, epoch * train_batches_per_epoch + step)
print("Saving checkpoint of model")
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("Checkpoint saved at {}".format(datetime.now(), checkpoint_name))
#
#
# Prediction
final_pred = {}
sess.run(test_init_op)
for j in range(test_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
if j == 0:
_temp_img_batch, _temp_label_batch = img_batch, label_batch
elif j == test_batches_per_epoch - 1:
img_batch = np.concatenate((img_batch, _temp_img_batch),
axis=0)[:batch_size]
probs = sess.run(softmax, feed_dict={
x: img_batch,
keep_prob: 1
})[:len(label_batch)]
for i in range(len(label_batch)):
img_class = list(label_batch[i]).index(1.)
if img_class not in final_pred:
final_pred[img_class] = []
final_pred[img_class].append(list(probs[i]))
final_pred_path = os.path.join(tr_test_path, '../final_pred', 'AlexNet')
try:
os.makedirs(final_pred_path)
except:
pass
with open(os.path.join(final_pred_path, 'prob.plk'), 'wb') as fp:
pickle.dump(final_pred, fp)
test_ct = len(final_pred.keys())
corr_ct = 0.
for k in final_pred.keys():
pred_class = np.argmax(np.array(final_pred[k]).mean(axis=0))
if k == pred_class:
corr_ct += 1
else:
print("%s <xxx> %s" %
(class_name[str(k)], class_name[str(pred_class)]))
print(test_ct, int(corr_ct), corr_ct * 100. / test_ct)
sess.close()
def main():
"""Run the demo."""
data = fetch_gpml_sarcos_data()
Xr = data.train.data.astype(np.float32)
Yr = data.train.targets.astype(np.float32)[:, np.newaxis]
Xs = data.test.data.astype(np.float32)
Ys = data.test.targets.astype(np.float32)[:, np.newaxis]
N, D = Xr.shape
print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE))))
# Scale and centre the data, as per the original experiment
ss = StandardScaler()
Xr = ss.fit_transform(Xr)
Xs = ss.transform(Xs)
ym = Yr.mean()
Yr -= ym
Ys -= ym
# Training batches
data_tr = Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \
.shuffle(buffer_size=1000) \
.batch(BATCH_SIZE)
# Testing iterators
data_ts = Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat()
with tf.name_scope("DataIterators"):
iterator = Iterator.from_structure(data_tr.output_types,
data_tr.output_shapes)
data = iterator.get_next()
training_init = iterator.make_initializer(data_tr)
testing_init = iterator.make_initializer(data_ts)
with tf.name_scope("Deepnet"):
phi, kl = net(X=data['X'])
std = tf.Variable(NOISE, name="noise")
lkhood = tf.distributions.Normal(phi, scale=ab.pos(std))
loss = ab.elbo(lkhood, data['Y'], N, kl)
tf.summary.scalar('loss', loss)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
with tf.name_scope("Test"):
r2 = rsquare(data['Y'], phi)
# Logging
log = tf.train.LoggingTensorHook(
{'step': global_step, 'loss': loss},
every_n_iter=1000
)
with tf.train.MonitoredTrainingSession(
config=CONFIG,
scaffold=tf.train.Scaffold(local_init_op=training_init),
checkpoint_dir="./sarcos/",
save_summaries_steps=None,
save_checkpoint_secs=20,
save_summaries_secs=20,
hooks=[log]
) as sess:
summary_writer = sess._hooks[1]._summary_writer
for i in range(NEPOCHS):
# Train for one epoch
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
pass
# Init testing and assess and log R-square score on test set
sess.run(testing_init)
r2_score = sess.run(r2)
score_sum = tf.Summary(value=[
tf.Summary.Value(tag='r-square', simple_value=r2_score)
])
summary_writer.add_summary(score_sum, sess.run(global_step))
# Re-init training
sess.run(training_init)
# Prediction
sess.run(testing_init)
Ey = ab.predict_samples(phi, feed_dict=None, n_groups=NPREDICTSAMPLES,
session=sess)
sigma = sess.run(std)
r2_score = sess.run(r2)
# Score mean standardised log likelihood
Eymean = Ey.mean(axis=0)
Eyvar = Ey.var(axis=0) + sigma**2 # add sigma2 for obervation noise
snlp = msll(Ys.flatten(), Eymean, Eyvar, Yr.flatten())
print("------------")
print("r-square: {:.4f}, smse: {:.4f}, msll: {:.4f}."
.format(r2_score, 1 - r2_score, snlp))
def main():
"""
Configuration Part.
"""
# Path to the textfiles for the trainings and validation set
train_file = './train.txt'
val_file = './val.txt'
# Learning params
learning_rate = 0.01
num_epochs = 100
batch_size = 10
# Network params
dropout_rate = 0.5
num_classes = 2
train_layers = ['fc8', 'fc7', 'fc6']
# How often we want to write the tf.summary data to disk
display_step = 20
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "./tmp/tensorboard"
checkpoint_path = "./tmp/checkpoints"
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
score = model.fc8
pred = tf.nn.softmax(score)
# List of trainable variables of the layers we want to train
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
# optimizer = tf.train.GradientDescentOptimizer(learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size))
# Start Tensorflow session
with tf.Session() as sess:
print("# isTrain : ", args.istrain)
if not args.istrain:
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
print("Reading model parameters from %s" %
ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("checkpoint path isn't correct")
return
file = tf.read_file(args.input_path)
decoded_img = tf.image.decode_jpeg(file, channels=3)
resized_img = tf.image.resize_images(decoded_img, [227, 227])
resized_img = tf.reshape(resized_img, [1, 227, 227, 3])
print("# decoded img : ", decoded_img.eval().shape)
pred_ = sess.run(pred,
feed_dict={
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("P(man|data) : ", pred_[0][0])
print("P(woman|data) : ", pred_[0][1])
img = decoded_img.eval()
plt.imshow(img)
plt.show()
if args.visualize:
w1, w2, w3, w4, w5 = sess.run(model.weight,
feed_dict={
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("W1 : ", w1.shape)
visualize(w1[:, :, 0, :25])
print("W2 : ", w2.shape)
visualize(w2[:, :, 0, :25])
print("W3 : ", w3.shape)
visualize(w3[:, :, 0, :25])
print("W4 : ", w4.shape)
visualize(w4[:, :, 0, :25])
print("W5 : ", w5.shape)
visualize(w5[:, :, 0, :25])
f1, f2, f3, f4, f5 = sess.run(model.fm, {
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("F1 : ", f1.shape)
visualize(f1[0][:, :, :25])
print("F2 : ", f2.shape)
visualize(f2[0][:, :, :25])
print("F3 : ", f3.shape)
visualize(f3[0][:, :, :25])
print("F4 : ", f4.shape)
visualize(f4[0][:, :, :25])
print("F5 : ", f5.shape)
visualize(f5[0][:, :, :25])
return
else:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
train_acc = 0.
train_count = 0.
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
acc, _ = sess.run([accuracy, train_op],
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: dropout_rate
})
train_acc += acc
train_count += 1
# Generate summary with the current batch of data and write to file
# if step % display_step == 0:
# s = sess.run(merged_summary, feed_dict={x: img_batch,
# y: label_batch,
# keep_prob: 1.})
#
# writer.add_summary(s, epoch*train_batches_per_epoch + step)
train_acc /= train_count
print("{} Train Accuracy = {:.4f}".format(datetime.now(),
train_acc))
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
for ind in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
acc = sess.run(accuracy,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
test_acc += acc
test_count += 1
if epoch is 2 and ind is 0:
fm = sess.run(model.fm,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
weight = sess.run(model.weight,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
# print("fm0 : ", np.array(fm[0]).shape)
# print("fm1 : ", np.array(fm[1]).shape)
# print("fm2 : ", np.array(fm[2]).shape)
# print("fm3 : ", np.array(fm[3]).shape)
# print("fm4 : ", np.array(fm[4]).shape)
#
# print("weight0 : ", np.array(weight[0]).shape)
# print("weight1 : ", np.array(weight[1]).shape)
# print("weight2 : ", np.array(weight[2]).shape)
# print("weight3 : ", np.array(weight[3]).shape)
# print("weight4 : ", np.array(weight[4]).shape)
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(
datetime.now(), test_acc))
print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
target_val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False,
mean='/home/camalab/caffe/models/vgg19/dog129sandt_mean.npy')
tst_data = ImageDataGenerator(
test_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False,
mean='/home/camalab/caffe/models/vgg19/dog129sandt_mean.npy')
# create an reinitializable iterator given the dataset structure
source_iterator = Iterator.from_structure(
source_tr_data.data.output_types, source_tr_data.data.output_shapes)
target_iterator = Iterator.from_structure(
target_tr_data.data.output_types, target_tr_data.data.output_shapes)
source_next_batch = source_iterator.get_next()
target_next_batch = target_iterator.get_next()
# Ops for initializing the two different iterators
source_training_init_op = source_iterator.make_initializer(source_tr_data.data)
target_training_init_op = target_iterator.make_initializer(target_tr_data.data)
source_validation_init_op = source_iterator.make_initializer(
source_val_data.data)
target_validation_init_op = target_iterator.make_initializer(
target_val_data.data)
testing_init_op = target_iterator.make_initializer(tst_data.data)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
val_data = ImageDataGenerator(val_file,
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="input_image")
if FLAGS.optimization == "cross_entropy":
annotation = tf.placeholder(tf.int32,
shape=[None, LOGITS_SIZE, LOGITS_SIZE, 1],
name="annotation") # For cross entropy
logits = u_net(x=image, keep_prob=0.75, channels=1, n_class=2)
label = tf.squeeze(annotation, squeeze_dims=[3])
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=label, name="entropy")) # For softmax
#loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=tf.squeeze(annotation, squeeze_dims=[3]),name="entropy")) # For softmax
elif FLAGS.optimization == "dice":
annotation = tf.placeholder(tf.int32,
shape=[None, LOGITS_SIZE, LOGITS_SIZE, 1],
name="annotation") # For DICE
logits = u_net(x=image, keep_prob=0.75, channels=1, n_class=2)
# pred_annotation (argmax) is not differentiable so it cannot be optimized. So in loss, we need to use logits instead of pred_annotation!
logits = tf.nn.softmax(logits) # axis = -1 default
logits2 = tf.slice(logits, [0, 0, 0, 1],
[-1, LOGITS_SIZE, LOGITS_SIZE, 1])
loss = 1 - tl.cost.dice_coe(logits2,
tf.cast(annotation, dtype=tf.float32))
total_var = tf.trainable_variables()
# ========================================
# To limit the training range
# scope_name = 'inference'
# trainable_var = [var for var in total_var if scope_name in var.name]
# ========================================
# Train all model
trainable_var = total_var
train_op = train(loss, trainable_var)
# All the variables defined HERE -------------------------------
dir_path = 'AP/Train'
batch_size = 3
opt_crop = False
crop_shape = (224, 224)
opt_resize = False
resize_shape = (224, 224)
rotation_status = True
rotation_angle = [-5, 5]
# --------------------------------------------------------------
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(dir_path=dir_path,
mode='training',
rotation_status=rotation_status,
rotation_angle=rotation_angle,
batch_size=batch_size)
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
training_init_op = iterator.make_initializer(tr_data.data)
sess = tf.Session()
#sess = tf.Session(config=tf.ConfigProto(device_count={'GPU': 0})) # CPU ONLY
print("Setting up Saver...")
saver = tf.train.Saver()
#summary_writer = tf.summary.FileWriter(FLAGS.logs_dir, sess.graph)
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
print("Start training")
start = time.time()
train_loss_list = []
x_train = []
validation_loss_list = []
x_validation = []
sess.run(training_init_op)
# for itr in xrange(MAX_ITERATION):
for itr in xrange(MAX_ITERATION): # about 12 hours of work / 2000
train_images, train_annotations = sess.run(next_batch)
# Reshape the annotation as the output (mask) has different dimension with the input
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.75
}
sess.run(train_op, feed_dict=feed_dict)
if (itr + 1) % 20 == 0:
#train_loss, summary_str = sess.run([loss, summary_op], feed_dict=feed_dict)
train_loss = sess.run(loss, feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_loss_list.append(train_loss)
x_train.append(itr + 1)
#summary_writer.add_summary(summary_str, itr)
end = time.time()
print("Iteration #", itr + 1, ",", np.int32(end - start), "s")
saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr + 1)
# Draw loss functions
plt.plot(x_train, train_loss_list, label='train')
plt.title("loss functions")
plt.xlabel("epoch")
plt.ylabel("loss")
plt.ylim(ymin=min(train_loss_list))
plt.ylim(ymax=max(train_loss_list) * 1.1)
plt.legend()
plt.savefig("loss_functions.png")
# Need to add another mode to draw the contour based on image only.
elif FLAGS.mode == "test":
print("Setting up test data...")
img_dir_name = '..\H&N_CTONLY'
test_batch_size = 10
test_index = 5
ind = 0
test_records = dicom_batchImage.read_DICOMbatchImage(
dir_name=img_dir_name,
opt_resize=opt_resize,
resize_shape=resize_shape,
opt_crop=opt_crop,
crop_shape=crop_shape)
test_annotations = np.zeros([test_batch_size, 224, 224,
1]) # fake input
for index in range(test_index):
print("Start creating data")
test_images = test_records.next_batch(batch_size=test_batch_size)
pred = sess.run(pred_annotation,
feed_dict={
image: test_images,
annotation: test_annotations,
keep_probability: 1.0
})
pred = np.squeeze(pred, axis=3)
print("Start saving data")
for itr in range(test_batch_size):
plt.subplot(121)
plt.imshow(test_images[itr, :, :, 0], cmap='gray')
plt.title("image")
plt.subplot(122)
plt.imshow(pred[itr], cmap='gray')
plt.title("pred mask")
plt.savefig(FLAGS.logs_dir + "/Prediction_test" + str(ind) +
".png")
print("Test iteration : ", ind)
ind += 1
elif FLAGS.mode == "visualize":
print("Setting up validation data...")
validation_records = dicom_batch.read_DICOM(
dir_name=dir_name + 'validation_set',
dir_image=dir_image,
dir_mask=dir_mask,
contour_name=contour_name,
opt_resize=opt_resize,
resize_shape=resize_shape,
opt_crop=opt_crop,
crop_shape=crop_shape,
rotation=False,
rotation_angle=rotation_angle,
bitsampling=False,
bitsampling_bit=bitsampling_bit)
dice_array = []
bins = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
# Save the image for display. Use matplotlib to draw this.
for itr in range(20):
valid_images, valid_annotations = validation_records.next_batch(
batch_size=1)
pred = sess.run(pred_annotation,
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0
})
pred = np.squeeze(pred, axis=3)
print(valid_images.shape, valid_annotations.shape, pred.shape)
valid_annotations = np.squeeze(valid_annotations, axis=3)
dice_coeff = dice(valid_annotations[0], pred[0])
dice_array.append(dice_coeff)
print("min max of prediction : ",
pred.flatten().min(),
pred.flatten().max())
print("min max of validation : ",
valid_annotations.flatten().min(),
valid_annotations.flatten().max())
print("DICE : ", dice_coeff)
print(valid_annotations.shape)
# Save images
plt.subplot(131)
plt.imshow(valid_images[0, :, :, 0], cmap='gray')
plt.title("image")
plt.subplot(132)
plt.imshow(valid_annotations[0, :, :], cmap='gray')
plt.title("mask original")
plt.subplot(133)
plt.imshow(pred[0], cmap='gray')
plt.title("mask predicted")
plt.suptitle("DICE : " + str(dice_coeff))
plt.savefig(FLAGS.logs_dir + "/Prediction_validation" + str(itr) +
".png")
# plt.show()
plt.figure()
plt.hist(dice_array, bins)
plt.xlabel('Dice')
plt.ylabel('frequency')
plt.title('Dice coefficient distribution of validation dataset')
plt.savefig(FLAGS.logs_dir + "/dice histogram" + ".png")
def main(_):
# Create training directories
now = datetime.datetime.now()
train_dir_name = now.strftime('alexnet_%Y%m%d_%H%M%S')
train_dir = os.path.join(FLAGS.tensorboard_root_dir, train_dir_name)
checkpoint_dir = os.path.join(train_dir, 'checkpoint')
tensorboard_dir = os.path.join(train_dir, 'tensorboard')
tensorboard_train_dir = os.path.join(tensorboard_dir, 'train')
tensorboard_val_dir = os.path.join(tensorboard_dir, 'val')
if not os.path.isdir(FLAGS.tensorboard_root_dir): os.mkdir(FLAGS.tensorboard_root_dir)
if not os.path.isdir(train_dir): os.mkdir(train_dir)
if not os.path.isdir(checkpoint_dir): os.mkdir(checkpoint_dir)
if not os.path.isdir(tensorboard_dir): os.mkdir(tensorboard_dir)
if not os.path.isdir(tensorboard_train_dir): os.mkdir(tensorboard_train_dir)
if not os.path.isdir(tensorboard_val_dir): os.mkdir(tensorboard_val_dir)
# Write flags to txt
flags_file_path = os.path.join(train_dir, 'flags.txt')
flags_file = open(flags_file_path, 'w')
flags_file.write('learning_rate={}\n'.format(FLAGS.learning_rate))
flags_file.write('dropout_keep_prob={}\n'.format(FLAGS.dropout_keep_prob))
flags_file.write('num_epochs={}\n'.format(FLAGS.num_epochs))
flags_file.write('batch_size={}\n'.format(FLAGS.batch_size))
flags_file.write('train_layers={}\n'.format(FLAGS.train_layers))
flags_file.write('multi_scale={}\n'.format(FLAGS.multi_scale))
flags_file.write('tensorboard_root_dir={}\n'.format(FLAGS.tensorboard_root_dir))
flags_file.write('log_step={}\n'.format(FLAGS.log_step))
flags_file.close()
# Placeholders
x = tf.placeholder(tf.float32, [FLAGS.batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.num_classes])
dropout_keep_prob = tf.placeholder(tf.float32)
is_training = tf.placeholder('bool', [])
# Model
train_layers = FLAGS.train_layers.split(',')
model = AlexNetModel(num_classes=FLAGS.num_classes, dropout_keep_prob=dropout_keep_prob)
loss = model.loss(x, y)
train_op = model.optimize(FLAGS.learning_rate, train_layers)
# Training accuracy of the model
correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Summaries
tf.summary.scalar('train_loss', loss)
tf.summary.scalar('train_accuracy', accuracy)
merged_summary = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(tensorboard_train_dir)
val_writer = tf.summary.FileWriter(tensorboard_val_dir)
saver = tf.train.Saver()
# Batch preprocessors
# multi_scale = FLAGS.multi_scale.split(',')
# if len(multi_scale) == 2:
# multi_scale = [int(multi_scale[0]), int(multi_scale[1])]
# else:
# multi_scale = None
# train_preprocessor = BatchPreprocessor(dataset_file_path=FLAGS.training_file, num_classes=FLAGS.num_classes,
# output_size=[227, 227], horizontal_flip=True, shuffle=True, multi_scale=multi_scale)
# val_preprocessor = BatchPreprocessor(dataset_file_path=FLAGS.val_file, num_classes=FLAGS.num_classes, output_size=[227, 227])
# Get the number of training/validation steps per epoch
# train_batches_per_epoch = np.floor(len(train_preprocessor.labels) / FLAGS.batch_size).astype(np.int16)
# val_batches_per_epoch = np.floor(len(val_preprocessor.labels) / FLAGS.batch_size).astype(np.int16)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(FLAGS.training_file,
mode='training',
batch_size=80,
num_classes=FLAGS.num_classes,
shuffle=True)
val_data = ImageDataGenerator(FLAGS.val_file,
mode='inference',
batch_size=80,
num_classes=FLAGS.num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
train_batches_per_epoch = int(np.floor(tr_data.data_size/FLAGS.batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size / FLAGS.batch_size))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
train_writer.add_graph(sess.graph)
# Initialize iterator with the training dataset
sess.run(training_init_op)
# Load the pretrained weights
# model.load_original_weights(sess, skip_layers=train_layers)
# Directly restore (your model should be exactly the same with checkpoint)
saver.restore(sess, "../training/resnet_20171202_143010/checkpoint/model_epoch8.ckpt")
print("{} Start training...".format(datetime.datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.datetime.now(), tensorboard_dir))
for epoch in range(FLAGS.num_epochs):
print("{} Epoch number: {}".format(datetime.datetime.now(), epoch+1))
step = 1
# Start training
while step < train_batches_per_epoch:
batch_xs, batch_ys = sess.run(next_batch)
# print("batch_xs")
# print(batch_xs.shape)
# print("batch_ys")
# print(batch_ys.shape)
sess.run(train_op, feed_dict={x: batch_xs, y: batch_ys, dropout_keep_prob: FLAGS.dropout_keep_prob, is_training: True})
# sess.run(training_init_op)
# Logging
if step % FLAGS.log_step == 0:
s = sess.run(merged_summary, feed_dict={x: batch_xs, y: batch_ys, dropout_keep_prob: 1.})
train_writer.add_summary(s, epoch * train_batches_per_epoch + step)
step += 1
# Epoch completed, start validation
print("{} Start validation".format(datetime.datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
for _ in range(val_batches_per_epoch):
batch_tx, batch_ty = sess.run(next_batch)
acc = sess.run(accuracy, feed_dict={x: batch_tx, y: batch_ty, dropout_keep_prob: 1., is_training: False})
test_acc += acc
test_count += 1
test_acc /= test_count
s = tf.Summary(value=[
tf.Summary.Value(tag="validation_accuracy", simple_value=test_acc)
])
val_writer.add_summary(s, epoch+1)
print("{} Validation Accuracy = {:.4f}".format(datetime.datetime.now(), test_acc))
# Reset the dataset pointers
# val_preprocessor.reset_pointer()
# train_preprocessor.reset_pointer()
print("{} Saving checkpoint of model...".format(datetime.datetime.now()))
#save checkpoint of the model
checkpoint_path = os.path.join(checkpoint_dir, 'model_epoch'+str(epoch+1)+'.ckpt')
save_path = saver.save(sess, checkpoint_path)
print("{} Model checkpoint saved at {}".format(datetime.datetime.now(), checkpoint_path))
def build_loss(patch, patch_est):
loss = tf.losses.mean_squared_error(labels=patch, predictions=patch_est)
return loss
# create tf dataset from input data
training_dataset = tf.contrib.data.Dataset.from_tensor_slices((training_measurements, training_patches))
validation_dataset = tf.contrib.data.Dataset.from_tensor_slices((validation_measurements, validation_patches))
nEpochs = 10
training_dataset = training_dataset.batch(1000)
validation_dataset = validation_dataset.batch(3000)
# initialize iterator
iterator = Iterator.from_structure(training_dataset.output_types, training_dataset.output_shapes)
next_measurement, next_patch = iterator.get_next()
training_init_op = iterator.make_initializer(training_dataset)
validation_init_op = iterator.make_initializer(validation_dataset)
patch_est = build_linear_mapping(next_measurement)
loss = build_loss(patch_est, next_patch)
# define optimization procedure
learning_rate = 0.01
learning_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)
with tf.Session() as sess:
def train(summary_dir, steps, samples):
time_steps = 128
num_classes = 16
feature_size = 13
learning_rate = 0.001
training_steps = steps
x_train, y_train, x_test, y_test = train_model.load_data(
"accent", samples, int(samples / 5))
_, counts = np.unique(y_train, return_counts=True)
y_train = tf.one_hot(y_train, num_classes)
y_test = tf.one_hot(y_test, num_classes)
counts_inverse = [samples - i for i in counts]
normed_weights = [(float(i) / sum(counts_inverse)) for i in counts_inverse]
print(normed_weights)
class_weights = tf.constant([normed_weights])
x, y, loss, accuracy, optimizer, summary_op = train_model.build_graph(
feature_size, time_steps, num_classes, class_weights, learning_rate)
train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(summary_dir, sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
for step in range(1, training_steps + 1):
total_loss = 0.0
total_accuracy = 0.0
total_steps = 0
iterator = Iterator.from_structure(train_data.output_types,
train_data.output_shapes)
next_element = iterator.get_next()
sess.run(iterator.make_initializer(train_data))
print("Step: ", step)
while True:
try:
total_steps += 1
x_element, y_element = sess.run(next_element)
x_element = x_element.reshape(
(1, time_steps, feature_size))
y_element = y_element.reshape((-1, num_classes))
feed_dict = {x: x_element, y: y_element}
loss_value, accuracy_value, _ = sess.run(
[loss, accuracy, optimizer], feed_dict=feed_dict)
total_loss += loss_value
total_accuracy += accuracy_value
except tf.errors.OutOfRangeError:
break
if step % 5 == 0:
feed_dict = {x: x_train, y: y_train.eval()}
_, _, summary = sess.run([loss, accuracy, summary_op],
feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
print("Loss: ", total_loss / total_steps)
print("Accuracy: ", total_accuracy / total_steps)
sys.stdout.flush()
saver.save(sess, './models/model-accent.ckpt')
print("\nAccent Train Accuracy: ",
sess.run(accuracy, feed_dict={
x: x_train,
y: y_train.eval()
}))
print("Accent Test Accuracy: ",
sess.run(accuracy, feed_dict={
x: x_test,
y: y_test.eval()
}))
def train(args, use_hex=True):
num_classes = 7
dataroot = '../data/PACS/'
cat = 'photo'
batch_size=args.batch_size
train_file, val_file, test_file=set_path(cat) # can change
tr_data = ImageDataGenerator(train_file,
dataroot=dataroot,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
dataroot=dataroot,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
test_data = ImageDataGenerator(test_file,
dataroot=dataroot,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
test_init_op = iterator.make_initializer(test_data.data)
train_batches_per_epoch = int(np.floor(tr_data.data_size/args.batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size / args.batch_size))
test_batches_per_epoch = int(np.floor(test_data.data_size / args.batch_size))
x_re = tf.placeholder(tf.float32, (None,28*28))
x_d = tf.placeholder(tf.float32, (None, 28*28))
x = tf.placeholder(tf.float32,(None,227,227,3))
y = tf.placeholder(tf.float32, (None, num_classes))
model = MNISTcnn(x, y, x_re, x_d, args, Hex_flag=use_hex)
# optimizer = tf.train.AdamOptimizer(1e-4).minimize(model.loss)
optimizer = tf.train.AdamOptimizer(1e-4) # default was 0.0005
first_train_op = optimizer.minimize(model.loss)
saver = tf.train.Saver(tf.trainable_variables())
with tf.Session() as sess:
print('Starting training')
print('load Alex net weights')
sess.run(tf.initialize_all_variables())
model.load_initial_weights(sess)
if args.load_params:
ckpt_file = os.path.join(args.ckpt_dir, 'mnist_model.ckpt')
print('Restoring parameters from', ckpt_file)
saver.restore(sess, ckpt_file)
validation = True
best_validate_accuracy = 0
score = 0
train_acc=[]
test_acc=[]
val_acc=[]
val_rep = None
val_re = None
val_d = None
val_y = None
for epoch in range(args.epochs):
begin = time.time()
sess.run(training_init_op)
#sess.run(validation_init_op)
#sess.run(test_init_op)
# train
######
train_accuracies = []
train_losses = []
train_rep = None
train_re = None
train_d = None
train_y = None
for i in range(train_batches_per_epoch):
batch_x, img_batch, batch_y = sess.run(next_batch)
batch_xd,batch_re=preparion(img_batch,args)
_, acc, loss, rep = sess.run([first_train_op, model.accuracy, model.loss, model.rep], feed_dict={x: batch_x,
x_re: batch_re,
x_d: batch_xd,
y: batch_y,
model.keep_prob: 0.5,
model.e: epoch,
model.batch: i})
train_accuracies.append(acc)
train_losses.append(loss)
if train_rep is None:
train_rep = rep
else:
train_rep = np.append(train_rep, rep, 0)
if train_re is None:
train_re = batch_re
else:
train_re = np.append(train_re, batch_re, 0)
if train_d is None:
train_d = batch_xd
else:
train_d = np.append(train_d, batch_xd, 0)
if train_y is None:
train_y = batch_y
else:
train_y = np.append(train_y, batch_y, 0)
train_acc_mean = np.mean(train_accuracies)
train_acc.append(train_acc_mean)
train_loss_mean = np.mean(train_losses)
# print ()
# compute loss over validation data
if validation:
sess.run(validation_init_op)
val_accuracies = []
val_rep = None
val_re = None
val_d = None
val_y = None
for i in range(val_batches_per_epoch):
batch_x, img_batch, batch_y = sess.run(next_batch)
batch_xd,batch_re=preparion(img_batch,args)
acc, rep = sess.run([model.accuracy, model.rep], feed_dict={x: batch_x, x_re:batch_re,
x_d: batch_xd, y: batch_y,
model.keep_prob: 1.0,
model.e: epoch,
model.batch: i})
val_accuracies.append(acc)
if val_rep is None:
val_rep = rep
else:
val_rep = np.append(val_rep, rep, 0)
if val_re is None:
val_re = batch_re
else:
val_re = np.append(val_re, batch_re, 0)
if val_d is None:
val_d = batch_xd
else:
val_d = np.append(val_d, batch_xd, 0)
if val_y is None:
val_y = batch_y
else:
val_y = np.append(val_y, batch_y, 0)
val_acc_mean = np.mean(val_accuracies)
val_acc.append(val_acc_mean)
# log progress to console
print("\nEpoch %d, time = %ds, train accuracy = %.4f, loss = %.4f, validation accuracy = %.4f" % (epoch, time.time()-begin, train_acc_mean, train_loss_mean, val_acc_mean))
else:
print("\nEpoch %d, time = %ds, train accuracy = %.4f" % (epoch, time.time()-begin, train_acc_mean))
sys.stdout.flush()
#test
if val_acc_mean > best_validate_accuracy:
best_validate_accuracy = val_acc_mean
test_accuracies = []
test_rep = None
test_re = None
test_d = None
test_y = None
sess.run(test_init_op)
for i in range(test_batches_per_epoch):
batch_x, img_batch, batch_y = sess.run(next_batch)
batch_xd, batch_re=preparion(img_batch,args)
acc, rep = sess.run([model.accuracy, model.rep], feed_dict={x: batch_x,
x_re: batch_re, x_d: batch_xd, y: batch_y,
model.keep_prob: 1.0,
model.e: epoch,
model.batch: i})
test_accuracies.append(acc)
if test_rep is None:
test_rep = rep
else:
test_rep = np.append(test_rep, rep, 0)
if test_re is None:
test_re = batch_re
else:
test_re = np.append(test_re, batch_re, 0)
if test_d is None:
test_d = batch_xd
else:
test_d = np.append(test_d, batch_xd, 0)
if test_y is None:
test_y = batch_y
else:
test_y = np.append(test_y, batch_y, 0)
score = np.mean(test_accuracies)
print("Best Validated Model Prediction Accuracy = %.4f " % (score))
np.save('representations/'+cat+'_train_rep', train_rep)
np.save('representations/'+cat+'_train_re', train_re)
np.save('representations/'+cat+'_train_d', train_d)
np.save('representations/'+cat+'_train_y', train_y)
np.save('representations/'+cat+'_val_rep', val_rep)
np.save('representations/'+cat+'_val_re', val_re)
np.save('representations/'+cat+'_val_d', val_d)
np.save('representations/'+cat+'_val_y', val_y)
np.save('representations/'+cat+'_test_rep', test_rep)
np.save('representations/'+cat+'_test_re', test_re)
np.save('representations/'+cat+'_test_d', test_d)
np.save('representations/'+cat+'_test_y', test_y)
# tvars = tf.trainable_variables()
# tvars_vals = sess.run(tvars)
#
# weights = {}
# for var, val in zip(tvars, tvars_vals):
# v = var.name.split('/')[0]
# if v not in weights:
# weights[v] = []
# weights[v].append(val)
# np.save('weights/pacs_sketch', weights)
test_acc.append(score)
if (epoch + 1) % 10 == 0:
ckpt_file = os.path.join(args.ckpt_dir, 'mnist_model.ckpt')
ckpt_file = os.path.join(args.ckpt_dir, 'mnist_model.ckpt')
saver.save(sess, ckpt_file)
""" reuse """
print("Best Validated Model Prediction Accuracy = %.4f " % (score))
return (train_acc,val_acc,test_acc)
# Mapping train_dataset
train_dataset = train_dataset.map(data_generator,
num_threads=4,
output_buffer_size=8 * batch_size)
train_dataset = train_dataset.shuffle(8 * batch_size)
train_dataset = train_dataset.batch(batch_size)
# Mapping valid_dataset
valid_dataset = valid_dataset.map(data_generator,
num_threads=4,
output_buffer_size=8 * batch_size)
valid_dataset = valid_dataset.shuffle(8 * batch_size)
valid_dataset = valid_dataset.batch(batch_size)
# create TensorFlow Iterator object
iterator = Iterator.from_structure(train_dataset.output_types,
train_dataset.output_shapes)
next_element = iterator.get_next()
# create two initialization ops to switch between the datasets
training_init_op = iterator.make_initializer(train_dataset)
validation_init_op = iterator.make_initializer(valid_dataset)
###### Simplified Object detection model #####
# Define single layers
# convolution
def conv2d(name, input_layer, kernel_size, filters, padding='same', relu=True):
if relu:
output = tf.layers.conv2d(inputs=input_layer,
filters=filters,
def main():
parser = argparse.ArgumentParser(description='Finetune Alexnet')
parser.add_argument('--kfold', type=int, default=1)
parser.add_argument('--images_dirname', type=str, default='images')
parser.add_argument('--from_checkpoint', action='store_true', default=False)
parser.add_argument('--no-from_checkpoint', action='store_false', dest='from_checkpoint')
parser.add_argument('--prototype_run', action='store_true', default=False)
parser.add_argument('--save_every', type=int, default=10)
parser.add_argument('--display_step', type=int, default=20, help='how often to write the tf.summary to disk')
parser.add_argument('--loss', type=str, default='features')
parser.add_argument('--learning_rate', type=float, default=0.01)
parser.add_argument('--dropout_rate', type=float, default=0.5)
parser.add_argument('--num_epochs', type=int, default=1000)
parser.add_argument('--batch_size', type=int, default=16) # TODO: make sure this still trains well (was 128)
parser.add_argument('--patience', type=int, default=10)
args = parser.parse_args()
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logger.info('Running with args {:s}'.format(', '.join(
['{}={}'.format(key, value) for (key, value) in vars(args).items()])))
"""
Configuration Part.
"""
# Path to the textfiles for the trainings and validation set
data_path = os.path.join(os.path.dirname(__file__),
args.images_dirname + ('' if not args.prototype_run else '.proto'))
train_file = os.path.join(data_path, 'train{}.txt'.format(args.kfold))
val_file = os.path.join(data_path, 'val{}.txt'.format(args.kfold))
test_file = os.path.join(data_path, 'test{}.txt'.format(args.kfold))
predictions_file = os.path.join(data_path, "predictions{}.txt".format(args.kfold))
# Network params
skip_layers = [] # train everything, not just ['fc8', 'fc7', 'fc6']
num_classes = 325 if args.loss == 'categorical' else None
logger.debug("Num classes: {}".format(num_classes))
# Path for tf.summary.FileWriter and to store model checkpoints
storage_path = os.path.join(os.path.dirname(__file__), "storage", "kfold{}".format(args.kfold))
filewriter_path = os.path.join(storage_path, "tensorboard")
checkpoint_path = os.path.join(storage_path, "checkpoints")
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.makedirs(checkpoint_path, exist_ok=True)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
logger.info('Loading data')
tr_data = ImageDataGenerator(train_file,
mode='training',
num_classes=num_classes,
batch_size=args.batch_size,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
num_classes=num_classes,
batch_size=args.batch_size,
shuffle=False)
test_data = ImageDataGenerator(test_file,
mode='inference',
num_classes=num_classes,
batch_size=args.batch_size,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
test_init_op = iterator.make_initializer(test_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [args.batch_size, 227, 227, 3], name='x')
y = tf.placeholder(tf.float32, [args.batch_size, 4096 if args.loss == 'features' else num_classes], name='y')
keep_prob = tf.placeholder(tf.float32)
# Initialize model
logger.info('Creating model')
model = AlexNet(x, keep_prob, skip_layer=skip_layers + ['fc8'] if args.loss == 'categorical' else [],
num_classes=num_classes)
# Link variable to model output
class_output = model.fc8
features_output = model.fc7
# List of trainable variables of the layers we want to train
var_list = [v for v in tf.trainable_variables() if not skip_layers or v.name.split('/')[0] in skip_layers]
# Op for calculating the loss
with tf.name_scope("loss"):
if args.loss == 'features':
loss = tf.losses.mean_squared_error(predictions=features_output, labels=y)
elif args.loss == 'categorical':
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=class_output, labels=y))
# loss = -tf.reduce_sum(y * tf.log(class_output + 1e-10))
else:
raise ValueError('Invalid value for loss provided: {}'.format(args.loss))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
gradients = [(g, v) for g, v in gradients if g is not None]
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(args.learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('loss', loss)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = max(int(np.floor(tr_data.data_size / args.batch_size)), 1)
val_batches_per_epoch = max(int(np.floor(val_data.data_size / args.batch_size)), 1)
test_batches_per_epoch = max(int(np.floor(test_data.data_size / args.batch_size)), 1)
# Start Tensorflow session
with tf.Session() as sess:
start_epoch = 0
if args.from_checkpoint:
checkpoints = [f[:-6] for f in
[os.path.join(checkpoint_path, f) for f in os.listdir(checkpoint_path)
if f.endswith(".ckpt.index")] if os.path.isfile(f)]
if len(checkpoints) == 0:
logger.warning("No checkpoints found - starting from scratch")
args.from_checkpoint = False
else:
last_checkpoint = sorted(checkpoints)[-1]
try:
saver.restore(sess, last_checkpoint)
start_epoch = int(
last_checkpoint[-6]) + 1 # TODO: fix hard-coded single-digit (doesn't work for 11)
except Exception:
logger.warning("Unable to recover checkpoint {} - starting from scratch".format(last_checkpoint))
args.from_checkpoint = False
if not args.from_checkpoint:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
logger.info("{} Start training...".format(datetime.now()))
logger.info("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path))
# Loop over number of epochs
best_val_epoch, best_val_loss = None, math.inf
for epoch in range(start_epoch, args.num_epochs):
logger.info("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
sess.run(train_op, feed_dict={x: img_batch,
y: label_batch,
keep_prob: args.dropout_rate})
# Generate summary with the current batch of data and write to file
if True or step % args.display_step == 0: # TODO: remove True
s = sess.run(merged_summary, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
writer.add_summary(s, epoch * train_batches_per_epoch + step)
# save checkpoint of the model
if epoch % args.save_every == 0:
logger.info("{} Saving checkpoint of model...".format(datetime.now()))
checkpoint_name = os.path.join(checkpoint_path, 'model_epoch{:d}.ckpt'.format(epoch + 1))
save_path = saver.save(sess, checkpoint_name)
logger.info("{} Model checkpoint saved at {}".format(datetime.now(), save_path))
# Validate the model on the entire validation set
logger.info("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
val_loss = 0.
val_count = 0
for _ in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
_loss = sess.run(loss, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
val_loss += _loss
val_count += 1
val_loss /= val_count
logger.info("{} Validation Loss = {:.4f}".format(datetime.now(), val_loss))
if val_loss < best_val_loss:
best_val_loss, best_val_epoch = val_loss, epoch
saver.save(sess, os.path.join(checkpoint_path, 'model_best.ckpt'))
# Test the model
logger.info("{} Start test".format(datetime.now()))
test_predictions = np.empty((test_data.data_size, num_classes or 4096))
sess.run(test_init_op)
for batch_num in range(test_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
preds = sess.run(features_output, feed_dict={x: img_batch, keep_prob: 1.})
test_predictions[batch_num * args.batch_size:(batch_num + 1) * args.batch_size, :] = preds
with open(predictions_file, 'w') as f:
for img_path, prediction in zip(test_data._img_paths, test_predictions):
f.write('{} {}\n'.format(img_path, ",".join([str(p) for p in prediction])))
logger.info("Wrote predictions to {}".format(predictions_file))
elif epoch - best_val_epoch > args.patience:
logger.info("Validation loss has not decreased for {:d} epochs - stop (best epoch: {:d})".format(
epoch + 1 - best_val_epoch, best_val_epoch))
break
def train(config_file):
# 1, load configuration parameters
config = parse_config(config_file)
config_data = config['data']
config_net = config['network']
config_train = config['training']
random.seed(config_train.get('random_seed', 1))
assert(config_data['with_ground_truth'])
net_type = config_net['net_type']
net_name = config_net['net_name']
class_num = config_net['class_num']
batch_size = config_data.get('batch_size', 5)
# 2, construct graph
full_data_shape = [batch_size] + config_data['data_shape']
full_label_shape = [batch_size] + config_data['label_shape']
x = tf.placeholder(tf.float32, shape = full_data_shape)
w = tf.placeholder(tf.float32, shape = full_label_shape)
y = tf.placeholder(tf.int64, shape = full_label_shape)
w_regularizer = regularizers.l2_regularizer(config_train.get('decay', 1e-7))
b_regularizer = regularizers.l2_regularizer(config_train.get('decay', 1e-7))
net_class = NetFactory.create(net_type)
net = net_class(num_classes = class_num,
w_regularizer = w_regularizer,
b_regularizer = b_regularizer,
name = net_name)
net.set_params(config_net)
predicty = net(x, is_training = True)
proby = tf.nn.softmax(predicty)
loss_func = LossFunction(n_class=class_num)
loss = loss_func(predicty, y, weight_map = w)
print('size of predicty:',predicty)
# 3, initialize session and saver
lr = config_train.get('learning_rate', 1e-3)
opt_step = tf.train.AdamOptimizer(lr).minimize(loss)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
loader = DataLoader(config_data)
train_data = loader.get_dataset('train', shuffle = True)
batch_per_epoch = loader.get_batch_per_epoch()
train_iterator = Iterator.from_structure(train_data.output_types,
train_data.output_shapes)
next_train_batch = train_iterator.get_next()
train_init_op = train_iterator.make_initializer(train_data)
# 4, start to train
loss_file = config_train['model_save_prefix'] + "_loss.txt"
start_it = config_train.get('start_iteration', 0)
if( start_it > 0):
saver.restore(sess, config_train['model_pre_trained'])
loss_list, temp_loss_list = [], []
for n in range(start_it, config_train['maximal_iteration']):
if((n-start_it)%batch_per_epoch == 0):
sess.run(train_init_op)
one_batch = sess.run(next_train_batch)
feed_dict = {x:one_batch['image'], w:one_batch['weight'], y:one_batch['label']}
opt_step.run(session = sess, feed_dict=feed_dict)
loss_value = loss.eval(feed_dict = feed_dict)
temp_loss_list.append(loss_value)
if((n+1)%config_train['loss_display_iteration'] == 0):
avg_loss = np.asarray(temp_loss_list, np.float32).mean()
t = time.strftime('%X %x %Z')
print(t, 'iter', n+1,'loss', avg_loss)
loss_list.append(avg_loss)
np.savetxt(loss_file, np.asarray(loss_list))
temp_loss_list = []
if((n+1)%config_train['snapshot_iteration'] == 0):
saver.save(sess, config_train['model_save_prefix']+"_{0:}.ckpt".format(n+1))
sess.close()
from alexnet import AlexNet
from dataprocess import ImageDataGenerator
from datetime import datetime
from tensorflow.contrib.data import Iterator
from config import *
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
with tf.device('/cpu:0'):
val_data = ImageDataGenerator(rest_file,
mode='inference',
batch_size=test_batch_size,
num_classes=num_classes,
shuffle=False)
iterator = Iterator.from_structure(val_data.data.output_types,
val_data.data.output_shapes)
next_batch = iterator.get_next()
validation_init_op = iterator.make_initializer(val_data.data)
x = tf.placeholder(tf.float32, [test_batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [test_batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
model = AlexNet(x, keep_prob, num_classes, train_layers)
score = model.fc8_softmax
with tf.name_scope("accuracy"):
index=tf.argmax(score, 1)
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
saver = tf.train.Saver()
print("loading data [train and test] \n")
filename_train = os.path.join(conf.DATA_DIR, "train.tfrecords")
filename_test = os.path.join(conf.DATA_DIR, "test.tfrecords")
#---------------read TFRecords data for training
data_train = tf.data.TFRecordDataset(filename_train)
data_train = data_train.map(data.parser_tfrecord)
data_train = data_train.batch(conf.BATCH_SIZE)
data_train = data_train.shuffle(conf.ESTIMATED_NUMBER_OF_BATCHES)
#---------------read TFRecords data for validation
data_test = tf.data.TFRecordDataset(filename_test)
data_test = data_test.map(data.parser_tfrecord)
data_test = data_test.batch(conf.BATCH_SIZE)
data_test = data_test.shuffle(conf.ESTIMATED_NUMBER_OF_BATCHES_TEST)
#defining saver to save snapshots
#defining a reinitializable iterator
iterator = Iterator.from_structure(data_train.output_types,
data_train.output_shapes)
iterator_test = Iterator.from_structure(data_test.output_types,
data_test.output_shapes)
next_batch = iterator.get_next()
next_batch_test = iterator_test.get_next()
#tensor that initialize the iterator:
training_init_op = iterator.make_initializer(data_train)
testing_init_op = iterator_test.make_initializer(data_test)
print("OK")
with tf.device(device_name):
net = mnet.net()
print("train")
#to save snapshots
saver = tf.train.Saver()
#load mean
noise2 = tf.random_normal([32, 1024, 128])
cloud = generate_cloud(tf.expand_dims(feature, axis=1), noise2)
return cloud
######################################### main #############################################
######################################### main #############################################
######################################### main #############################################
######################################### main #############################################
######################################### main #############################################
cloud_provider = tf.data.Dataset.from_generator(provide_data, output_types=(tf.float32, tf.float32), \
output_shapes=(tf.TensorShape([32, 1024, 3]), tf.TensorShape([32,40])))
point_clouds, cloud_labels = cloud_provider.make_one_shot_iterator().get_next()
iterator = Iterator.from_structure(cloud_provider.output_types,
cloud_provider.output_shapes)
training_init_op = iterator.make_initializer(cloud_provider)
noise = tf.random_normal([FLAGS.batch_size, FLAGS.noise_dims])
#with tf.variable_scope("my_scope", reuse=tf.AUTO_REUSE):
# Build the generator and discriminator.
gan_model = tfgan.gan_model(
generator_fn=conditional_generator, # you define
discriminator_fn=conditional_discriminator, # you define
real_data=point_clouds,
generator_inputs=(noise, cloud_labels))
# Build the GAN loss.
gan_loss = tfgan.gan_loss(
gan_model,
#gradient_penalty_weight=1.0,
return img_decoded, label_decoded
# training & validation 경로
train_imgs = tf.constant([("./dataset/train_in/%d_bi.png" % i)
for i in range(1, 40001)])
label_imgs = tf.constant([("./dataset/train_out/%d_gt.png" % i)
for i in range(1, 40001)])
dataset = tf.data.Dataset.from_tensor_slices((train_imgs, label_imgs))
dataset = dataset.map(input_parser)
dataset = dataset.shuffle(buffer_size=20000)
dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
iterator = Iterator.from_structure(dataset.output_types, dataset.output_shapes)
training_init_op = iterator.make_initializer(dataset)
next_element = iterator.get_next()
#valid_imgs = tf.constant([("./dataset/validation_bp/%d_bi.png" % i) for i in range(1,501)])
#valid_label_imgs = tf.constant([("./dataset/validation_gp/%d_gt.png" % i) for i in range(1,501)])
#dataset2 = Dataset.from_tensor_slices((valid_imgs,valid_label_imgs))
#dataset2= dataset2.map(input_parser)
#dataset2 = dataset2.shuffle(buffer_size=20000)
#dataset2 = dataset2.repeat()
#dataset2 = dataset2.batch(batch_size)
#iterator2 = Iterator.from_structure(dataset2.output_types,dataset2.output_shapes)
#training_init_op2 = iterator2.make_initializer(dataset2)
#next_element2 = iterator2.get_next()
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output