def _load_data(self):
self.config_data['batch_size'] = self.batch_size
self.config_data['patch_type_x'] = self.config_net['patch_type_x']
self.config_data['patch_type_y'] = self.config_net['patch_type_y']
self.config_data['patch_shape_x'] = self.config_net['patch_shape_x']
self.config_data['patch_shape_y'] = self.config_net['patch_shape_y']
if(self.mode == 'train'):
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
self.data_agent = ImageLoader(self.config_data)
# create an reinitializable iterator given the dataset structure
train_dataset = self.data_agent.get_dataset('train')
valid_dataset = self.data_agent.get_dataset('valid')
train_iterator = Iterator.from_structure(train_dataset.output_types,
train_dataset.output_shapes)
valid_iterator = Iterator.from_structure(valid_dataset.output_types,
valid_dataset.output_shapes)
self.next_train_batch = train_iterator.get_next()
self.next_valid_batch = valid_iterator.get_next()
# Ops for initializing the two different iterators
self.train_init_op = train_iterator.make_initializer(train_dataset)
self.valid_init_op = valid_iterator.make_initializer(valid_dataset)
else:
self.data_agent = ImageLoader(self.config_data)
self.test_dataset = self.data_agent.get_dataset('test')
def init_tfdata(self, batch_size, main_dir, resize_shape, mode='train'):
self.data_session = tf.Session()
print("Creating the iterator for training data")
with tf.device('/cpu:0'):
segdl = SegDataLoader(main_dir, batch_size, (resize_shape[0], resize_shape[1]), resize_shape,
# * 2), resize_shape,
'data/cityscapes_tfdata/train.txt')
iterator = Iterator.from_structure(segdl.data_tr.output_types, segdl.data_tr.output_shapes)
next_batch = iterator.get_next()
self.init_op = iterator.make_initializer(segdl.data_tr)
self.data_session.run(self.init_op)
print("Loading Validation data in memoryfor faster training..")
self.val_data = {'X': np.load(self.args.data_dir + "X_val.npy"),
'Y': np.load(self.args.data_dir + "Y_val.npy")}
# self.crop()
# import cv2
# cv2.imshow('crop1', self.val_data['X'][0,:,:,:])
# cv2.imshow('crop2', self.val_data['X'][1,:,:,:])
# cv2.imshow('seg1', self.val_data['Y'][0,:,:])
# cv2.imshow('seg2', self.val_data['Y'][1,:,:])
# cv2.waitKey()
self.val_data_len = self.val_data['X'].shape[0] - self.val_data['X'].shape[0] % self.args.batch_size
# self.num_iterations_validation_per_epoch = (
# self.val_data_len + self.args.batch_size - 1) // self.args.batch_size
self.num_iterations_validation_per_epoch = self.val_data_len // self.args.batch_size
print("Val-shape-x -- " + str(self.val_data['X'].shape) + " " + str(self.val_data_len))
print("Val-shape-y -- " + str(self.val_data['Y'].shape))
print("Num of iterations on validation data in one epoch -- " + str(self.num_iterations_validation_per_epoch))
print("Validation data is loaded")
return next_batch, segdl.data_len
def tf_create_iterator(dataset, batch_size):
""" """
dataset_prefix = os.path.join(SNPX_DATASET_ROOT, dataset, dataset)
train_rec_file = dataset_prefix + "_train.tfrecords"
val_rec_file = dataset_prefix + "_val.tfrecords"
# Create the training dataset object
train_set = TFRecordDataset(train_rec_file)
train_set = train_set.map(tf_parse_record,
num_threads=4,
output_buffer_size=1000)
train_set = train_set.shuffle(buffer_size=50000)
train_set = train_set.batch(batch_size)
# Create the validation dataset object
val_set = TFRecordDataset(val_rec_file)
val_set = val_set.map(tf_parse_record)
val_set = val_set.batch(batch_size)
# Create a reinitializable iterator from both datasets
iterator = Iterator.from_structure(train_set.output_types,
train_set.output_shapes)
train_init_op = iterator.make_initializer(train_set)
val_init_op = iterator.make_initializer(val_set)
iter_op = iterator.get_next()
return train_init_op, val_init_op, iter_op
def generate_input_pipline(tfrecords_path, filenames, batch_size,
test_batch_size, capacity, _parse_function_train,
_parse_function_test):
'''
:param sess:
:param tfrecords_path:
:param filenames:
:param epochs:
:param batch_size:
:param capacity:
:param _parse_function: tf dataset api parse function, you should provide this function by yourself
:return:
'''
file_paths = [os.path.join(tfrecords_path, name) for name in filenames]
train_dataset = tf.contrib.data.TFRecordDataset(file_paths[0])
train_dataset = train_dataset.map(_parse_function_train)
train_dataset = train_dataset.repeat(1)
train_dataset = train_dataset.batch(batch_size)
train_dataset = train_dataset.shuffle(buffer_size=capacity)
test_dataset = tf.contrib.data.TFRecordDataset(file_paths[1])
test_dataset = test_dataset.map(_parse_function_test)
test_dataset = test_dataset.repeat(1)
test_dataset = test_dataset.batch(test_batch_size)
iterator = Iterator.from_structure(train_dataset.output_types,
train_dataset.output_shapes)
next_element = iterator.get_next()
training_init_op = iterator.make_initializer(train_dataset)
test_init_op = iterator.make_initializer(test_dataset)
return next_element, training_init_op, test_init_op
def validate(model_def):
"""
Validate my alexnet implementation
Args:
model_def: the model class/definition
"""
img_dir = os.path.join('.', 'images')
images = []
print "loading images ..."
files = fnmatch.filter(os.listdir(img_dir), '*.jpeg')
for f in files:
print "> " + f
img_file = tf.read_file(os.path.join(img_dir, f))
img_decoded = tf.image.decode_jpeg(img_file, channels=3)
img_processed = model_def.image_prep.preprocess_image(
image=img_decoded,
output_height=model_def.image_size,
output_width=model_def.image_size,
is_training=False)
images.append(img_processed)
# create TensorFlow Iterator object
images = Dataset.from_tensors(images)
iterator = Iterator.from_structure(images.output_types,
images.output_shapes)
next_element = iterator.get_next()
iterator_init_op = iterator.make_initializer(images)
# create the model and get scores (pipe to softmax)
model = model_def(next_element)
scores = model.get_final_op()
softmax = tf.nn.softmax(scores)
print 'start validation ...'
with tf.Session() as sess:
# Initialize all variables and the iterator
sess.run(tf.global_variables_initializer())
sess.run(iterator_init_op)
# Load the pretrained weights into the model
model.load_initial_weights(sess)
# run the graph
probs = sess.run(softmax)
if model_def is ResNetV2:
probs = prep_resnet_results(probs)
# sometime we have an offset
offset = len(class_names) - len(probs[0])
# print the results
for prob in probs:
best_index = np.argmax(prob)
print "> " + class_names[best_index +
offset] + " -> %.4f" % prob[best_index]
def get_dataset_ops(data_train,
data_val,
batch_size,
train_size,
val_size,
shuffle=True):
"""
"""
# shuffle the dataset and create batches
if shuffle:
data_train = data_train.shuffle(train_size)
data_val = data_val.shuffle(val_size)
data_train = data_train.batch(batch_size)
data_val = data_val.batch(batch_size)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(data_train.output_types,
data_train.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
init_op_train = iterator.make_initializer(data_train)
init_op_val = iterator.make_initializer(data_val)
return init_op_train, init_op_val, next_batch
def finalize(self, shapes, batchsize):
assert not hasattr(self, 'output_shapes')
self.output_shapes = shapes
self.batchsize = batchsize
ishapes = [[None] + list(s) for s in self.output_shapes]
itypes = (tf.float32, tf.float32)
handle = tf.placeholder(tf.string, shape=[])
iterator = Iterator.from_string_handle(handle, itypes, tuple(ishapes))
getter = iterator.get_next()
return getter, handle
def train(self):
tf.global_variables_initializer().run()
could_load = 0
# load ckpt
#could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
tr_data = ImageDataGenerator(
FLAGS.train_file,
img_size=[self.DATA_HEIGHT, self.DATA_WIDTH],
label_size=[self.LABEL_HEIGHT, self.LABEL_WIDTH],
batch_size=self.BATCH_SIZE,
shuffle=True)
print('[Model] Data done ...')
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
batch_idxs = int(np.floor(tr_data.data_size / self.BATCH_SIZE))
print(tr_data.data_size)
for epoch in range(self.num_epochs): #
print('[Ves9] Begin epoch ', epoch, '...')
for iteration in range(batch_idxs):
start_time = time.time()
batch_images, batch_gts = self.sess.run(next_batch)
print(batch_images[0].shape, batch_images[0])
print(batch_gts[0].shape, batch_gts[0])
'''
_cost, result, acc_, _ = self.sess.run(
[self.cost, self.output, self.accuracy, self.train_op],
feed_dict={self.images: batch_images,
self.ground_truth: batch_gts})
'''
tm = time.time() - start_time
'''
if np.mod(iteration, 5000) == 0 and iteration != 0:
print('[INFO] Save checkpoint...')
self.save(self.checkpoint_dir, iteration)
if (iteration % 50 == 0):
print(iteration, ': ', '%.2f' % (tm), _cost)
'''
print('[INFO] Save the epoch checkpoint ... ')
def eval_data_generator(file_path, batch_size, sub_img_folder):
temp = imageDataGenerator(file_path, batch_size, sub_img_folder)
iterator = Iterator.from_structure(temp.data.output_types,
temp.data.output_shapes)
next_batch = iterator.get_next()
dataset_init_op = iterator.make_initializer(temp.data)
with tf.Session() as sess:
sess.run(dataset_init_op)
if sub_img_folder == '/test/':
img, label = sess.run(next_batch)
img, label = sess.run(next_batch)
else:
img, label = sess.run(next_batch)
# print(img.shape, label)
# imgplot = plt.imshow(img[0])
# plt.show()
return img, label
def confusion_matrix(val_file, num_classes, checkpoint_path):
with tf.device('/cpu:0'):
val_data = ImageDataGenerator(val_file, mode='inference', shuffle=True)
iterator = Iterator.from_structure(val_data.data.output_types,
val_data.data.output_shapes)
next_batch = iterator.get_next()
val_init_op = iterator.make_initializer(val_data.data)
x = tf.placeholder(tf.float32,
shape=[1, None, None, 3],
name="input_image")
keep_prob = tf.placeholder(tf.float32)
model = VGG16_FCN(x, num_classes, keep_prob)
cm = np.zeros([num_classes, num_classes], dtype=np.uint32)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(val_init_op)
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
for step in range(val_data.data_size):
image, label = sess.run(next_batch)
result_label = sess.run(model.pred_up,
feed_dict={
x: image,
keep_prob: 1.0
})
cml = confusion_matrix_label(result_label[0], label[0],
num_classes)
cm += cml
return cm
def get_dataset_tensors(args):
with tf.device('/cpu:0'), tf.variable_scope('input_pipeline'):
# TODO move this to hem.init()
# find all dataset plugins available
p = get_dataset(args.dataset)
# ensure that the dataset exists
if not p.check_prepared_datasets(args.dataset_dir):
if not p.check_raw_datasets(args.raw_dataset_dir):
print('Downloading dataset...')
# TODO: datasets should be able to be marked as non-downloadable
p.download(args.raw_dataset_dir)
print('Converting to tfrecord...')
p.convert_to_tfrecord(args.raw_dataset_dir, args.dataset_dir)
# load the dataset
datasets = p.get_datasets(args)
dataset_iterators = {}
# tensor to hold which training/eval phase we are in
handle = tf.placeholder(tf.string, shape=[])
# add a dataset for train, validation, and testing
for k, v in datasets.items():
# skip test set if not needed
if len(args.test_epochs) == 0 and k == 'test':
continue
d = v[0]
n = sum([1 for r in tf.python_io.tf_record_iterator(v[1])])
cache_fn = '{}.cache.{}'.format(args.dataset, k)
d = d.cache(os.path.join(
args.cache_dir, cache_fn)) if args.cache_dir else d.cache()
d = d.repeat()
d = d.shuffle(buffer_size=args.buffer_size, seed=args.seed)
d = d.batch(args.batch_size * args.n_gpus)
x_iterator = d.make_initializable_iterator()
dataset_iterators[k] = {
'x': x_iterator,
'n': n,
'batches': int(n / (args.batch_size * args.n_gpus)),
'handle': x_iterator.string_handle()
}
# feedable dataset that will swap between train/test/val
iterator = Iterator.from_string_handle(handle, d.output_types,
d.output_shapes)
return iterator.get_next(), handle, dataset_iterators
def input_pipeline(data_dir):
BATCH_SIZE = 16
filenames = [
join(data_dir, filename) for filename in os.listdir(data_dir)
if isfile(join(data_dir, filename))
]
np.random.shuffle(filenames)
train_images = tf.constant(filenames)
tr_data = Dataset.from_tensor_slices(train_images)
tr_data = tr_data.map(input_parser).map(distort_input)
tr_data = tr_data.batch(16)
# create TensorFlow Iterator object
iterator = Iterator.from_structure(tr_data.output_types,
tr_data.output_shapes)
training_init_op = iterator.make_initializer(tr_data)
next_element = iterator.get_next()
with tf.Session() as sess:
# initialize the iterator on the training data
sess.run(training_init_op)
# get each element of the training dataset until the end is reached
while True:
try:
elem = sess.run(next_element)
print(elem)
except tf.errors.OutOfRangeError:
print("End of training dataset.")
break
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
val_data = ImageDataGenerator(val_file,
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
# Mapping train_dataset
train_dataset = train_dataset.map(data_generator,
num_threads=4,
output_buffer_size=8 * batch_size)
train_dataset = train_dataset.shuffle(8 * batch_size)
train_dataset = train_dataset.batch(batch_size)
# Mapping valid_dataset
valid_dataset = valid_dataset.map(data_generator,
num_threads=4,
output_buffer_size=8 * batch_size)
valid_dataset = valid_dataset.shuffle(8 * batch_size)
valid_dataset = valid_dataset.batch(batch_size)
# create TensorFlow Iterator object
iterator = Iterator.from_structure(train_dataset.output_types,
train_dataset.output_shapes)
next_element = iterator.get_next()
# create two initialization ops to switch between the datasets
training_init_op = iterator.make_initializer(train_dataset)
validation_init_op = iterator.make_initializer(valid_dataset)
###### Simplified Object detection model #####
# Define single layers
# convolution
def conv2d(name, input_layer, kernel_size, filters, padding='same', relu=True):
if relu:
output = tf.layers.conv2d(inputs=input_layer,
filters=filters,
def train(model, saver, sess, exp_string, train_file_list, test_file, resume_itr=0):
SUMMARY_INTERVAL = 100
SAVE_INTERVAL = 10000
PRINT_INTERVAL = 100
TEST_PRINT_INTERVAL = 100
dropout_rate = 0.5
if FLAGS.log:
train_writer = tf.summary.FileWriter(FLAGS.logdir + '/' + exp_string, sess.graph)
print('Done initializing, starting training.')
prelosses, postlosses = [], []
num_classes = FLAGS.num_classes # for classification, 1 otherwise
# Defining data loaders
with tf.device('/cpu:0'):
tr_data_list = []
train_iterator_list = []
train_next_list = []
for i in range(len(train_file_list)):
tr_data = ImageDataGenerator(train_file_list[i],
dataroot=FLAGS.dataroot,
mode='training',
batch_size=FLAGS.meta_batch_size,
num_classes=num_classes,
shuffle=True)
tr_data_list.append(tr_data)
train_iterator_list.append(Iterator.from_structure(tr_data_list[i].data.output_types,
tr_data_list[i].data.output_shapes))
train_next_list.append(train_iterator_list[i].get_next())
test_data = ImageDataGenerator(test_file,
dataroot=FLAGS.dataroot,
mode='inference',
batch_size=1,
num_classes=num_classes,
shuffle=False)
test_iterator = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
test_next_batch = test_iterator.get_next()
# create an reinitializable iterator given the dataset structure
# Ops for initializing different iterators
training_init_op = []
train_batches_per_epoch = []
for i in range(len(train_file_list)):
training_init_op.append(train_iterator_list[i].make_initializer(tr_data_list[i].data))
train_batches_per_epoch.append(int(np.floor(tr_data_list[i].data_size/FLAGS.meta_batch_size)))
test_init_op = test_iterator.make_initializer(test_data.data)
test_batches_per_epoch = int(np.floor(test_data.data_size / 1))
# Training begins
for itr in range(resume_itr, FLAGS.pretrain_iterations + FLAGS.metatrain_iterations):
feed_dict = {}
# Sampling training and test tasks
num_training_tasks = len(train_file_list)
num_meta_train = num_training_tasks-1
num_meta_test = num_training_tasks-num_meta_train
# Randomly choosing meta train and meta test domains
task_list = np.random.permutation(num_training_tasks)
meta_train_index_list = task_list[:num_meta_train]
meta_test_index_list = task_list[num_meta_train:]
for i in range(len(train_file_list)):
if itr%train_batches_per_epoch[i] == 0:
sess.run(training_init_op[i])
# Populating input tensors
# Sampling meta train data
for i in range(num_meta_train):
task_ind = meta_train_index_list[i]
if i == 0:
inputa, labela = sess.run(train_next_list[task_ind])
else:
inp_tmp, lab_tmp = sess.run(train_next_list[task_ind])
inputa = np.concatenate((inputa, inp_tmp), axis=0)
labela = np.concatenate((labela, lab_tmp), axis=0)
inputs_all = list(zip(inputa, labela))
shuffle(inputs_all)
inputa, labela = zip(*inputs_all)
# Sampling meta test data
for i in range(num_meta_test):
task_ind = meta_test_index_list[i]
if i == 0:
inputb, labelb = sess.run(train_next_list[task_ind])
else:
inp_tmp, lab_tmp = sess.run(train_next_list[task_ind])
inputb = np.concatenate((inputb, inp_tmp), axis=0)
labelb = np.concatenate((labelb, lab_tmp), axis=0)
feed_dict = {model.inputa: inputa, model.inputb: inputb, model.labela: labela, model.labelb: labelb, model.KEEP_PROB: dropout_rate}
if itr<FLAGS.pretrain_iterations:
input_tensors = [model.pretrain_op]
else:
input_tensors = [model.metatrain_op]
if (itr % SUMMARY_INTERVAL == 0 or itr % PRINT_INTERVAL == 0):
input_tensors.extend([model.summ_op, model.total_loss1, model.total_losses2[FLAGS.num_updates-1]])
input_tensors.extend([model.total_accuracy1, model.total_accuracies2[FLAGS.num_updates-1]])
result = sess.run(input_tensors, feed_dict)
if itr % SUMMARY_INTERVAL == 0:
prelosses.append(result[-2])
if FLAGS.log:
train_writer.add_summary(result[1], itr)
postlosses.append(result[-1])
if (itr!=0) and itr % PRINT_INTERVAL == 0:
print_str = 'Iteration ' + str(itr - FLAGS.pretrain_iterations)
print_str += ': ' + str(np.mean(prelosses)) + ', ' + str(np.mean(postlosses))
print(print_str)
prelosses, postlosses = [], []
if (itr!=0) and itr % SAVE_INTERVAL == 0:
saver.save(sess, FLAGS.logdir + '/' + exp_string + '/model' + str(itr))
# Testing periodically
if itr % TEST_PRINT_INTERVAL == 0:
test_acc = 0.
test_loss = 0.
test_count = 0
sess.run(test_init_op)
for it in range(test_batches_per_epoch):
test_input, test_label = sess.run(test_next_batch)
feed_dict = {model.test_input: test_input, model.test_label: test_label, model.KEEP_PROB: 1.}
input_tensors = [model.test_loss, model.test_acc]
result = sess.run(input_tensors, feed_dict)
test_loss += result[0]
test_acc += result[1]
test_count += 1
print('Validation results: Iteration %d, Loss: %f, Accuracy: %f' %(itr, test_loss/test_count, test_acc/test_count))
saver.save(sess, FLAGS.logdir + '/' + exp_string + '/model' + str(itr))
def main():
"""Run the demo."""
data = fetch_gpml_sarcos_data()
Xr = data.train.data.astype(np.float32)
Yr = data.train.targets.astype(np.float32)[:, np.newaxis]
Xs = data.test.data.astype(np.float32)
Ys = data.test.targets.astype(np.float32)[:, np.newaxis]
N, D = Xr.shape
print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE))))
# Scale and centre the data, as per the original experiment
ss = StandardScaler()
Xr = ss.fit_transform(Xr)
Xs = ss.transform(Xs)
ym = Yr.mean()
Yr -= ym
Ys -= ym
# Training batches
data_tr = Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \
.shuffle(buffer_size=1000) \
.batch(BATCH_SIZE)
# Testing iterators
data_ts = Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat()
with tf.name_scope("DataIterators"):
iterator = Iterator.from_structure(data_tr.output_types,
data_tr.output_shapes)
data = iterator.get_next()
training_init = iterator.make_initializer(data_tr)
testing_init = iterator.make_initializer(data_ts)
with tf.name_scope("Deepnet"):
phi, kl = net(X=data['X'])
std = tf.Variable(NOISE, name="noise")
lkhood = tf.distributions.Normal(phi, scale=ab.pos(std))
loss = ab.elbo(lkhood, data['Y'], N, kl)
tf.summary.scalar('loss', loss)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
with tf.name_scope("Test"):
r2 = rsquare(data['Y'], phi)
# Logging
log = tf.train.LoggingTensorHook(
{'step': global_step, 'loss': loss},
every_n_iter=1000
)
with tf.train.MonitoredTrainingSession(
config=CONFIG,
scaffold=tf.train.Scaffold(local_init_op=training_init),
checkpoint_dir="./sarcos/",
save_summaries_steps=None,
save_checkpoint_secs=20,
save_summaries_secs=20,
hooks=[log]
) as sess:
summary_writer = sess._hooks[1]._summary_writer
for i in range(NEPOCHS):
# Train for one epoch
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
pass
# Init testing and assess and log R-square score on test set
sess.run(testing_init)
r2_score = sess.run(r2)
score_sum = tf.Summary(value=[
tf.Summary.Value(tag='r-square', simple_value=r2_score)
])
summary_writer.add_summary(score_sum, sess.run(global_step))
# Re-init training
sess.run(training_init)
# Prediction
sess.run(testing_init)
Ey = ab.predict_samples(phi, feed_dict=None, n_groups=NPREDICTSAMPLES,
session=sess)
sigma = sess.run(std)
r2_score = sess.run(r2)
# Score mean standardised log likelihood
Eymean = Ey.mean(axis=0)
Eyvar = Ey.var(axis=0) + sigma**2 # add sigma2 for obervation noise
snlp = msll(Ys.flatten(), Eymean, Eyvar, Yr.flatten())
print("------------")
print("r-square: {:.4f}, smse: {:.4f}, msll: {:.4f}."
.format(r2_score, 1 - r2_score, snlp))
def main():
parser = argparse.ArgumentParser(description='Finetune Alexnet')
parser.add_argument('--kfold', type=int, default=1)
parser.add_argument('--images_dirname', type=str, default='images')
parser.add_argument('--from_checkpoint', action='store_true', default=False)
parser.add_argument('--no-from_checkpoint', action='store_false', dest='from_checkpoint')
parser.add_argument('--prototype_run', action='store_true', default=False)
parser.add_argument('--save_every', type=int, default=10)
parser.add_argument('--display_step', type=int, default=20, help='how often to write the tf.summary to disk')
parser.add_argument('--loss', type=str, default='features')
parser.add_argument('--learning_rate', type=float, default=0.01)
parser.add_argument('--dropout_rate', type=float, default=0.5)
parser.add_argument('--num_epochs', type=int, default=1000)
parser.add_argument('--batch_size', type=int, default=16) # TODO: make sure this still trains well (was 128)
parser.add_argument('--patience', type=int, default=10)
args = parser.parse_args()
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logger.info('Running with args {:s}'.format(', '.join(
['{}={}'.format(key, value) for (key, value) in vars(args).items()])))
"""
Configuration Part.
"""
# Path to the textfiles for the trainings and validation set
data_path = os.path.join(os.path.dirname(__file__),
args.images_dirname + ('' if not args.prototype_run else '.proto'))
train_file = os.path.join(data_path, 'train{}.txt'.format(args.kfold))
val_file = os.path.join(data_path, 'val{}.txt'.format(args.kfold))
test_file = os.path.join(data_path, 'test{}.txt'.format(args.kfold))
predictions_file = os.path.join(data_path, "predictions{}.txt".format(args.kfold))
# Network params
skip_layers = [] # train everything, not just ['fc8', 'fc7', 'fc6']
num_classes = 325 if args.loss == 'categorical' else None
logger.debug("Num classes: {}".format(num_classes))
# Path for tf.summary.FileWriter and to store model checkpoints
storage_path = os.path.join(os.path.dirname(__file__), "storage", "kfold{}".format(args.kfold))
filewriter_path = os.path.join(storage_path, "tensorboard")
checkpoint_path = os.path.join(storage_path, "checkpoints")
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.makedirs(checkpoint_path, exist_ok=True)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
logger.info('Loading data')
tr_data = ImageDataGenerator(train_file,
mode='training',
num_classes=num_classes,
batch_size=args.batch_size,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
num_classes=num_classes,
batch_size=args.batch_size,
shuffle=False)
test_data = ImageDataGenerator(test_file,
mode='inference',
num_classes=num_classes,
batch_size=args.batch_size,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
test_init_op = iterator.make_initializer(test_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [args.batch_size, 227, 227, 3], name='x')
y = tf.placeholder(tf.float32, [args.batch_size, 4096 if args.loss == 'features' else num_classes], name='y')
keep_prob = tf.placeholder(tf.float32)
# Initialize model
logger.info('Creating model')
model = AlexNet(x, keep_prob, skip_layer=skip_layers + ['fc8'] if args.loss == 'categorical' else [],
num_classes=num_classes)
# Link variable to model output
class_output = model.fc8
features_output = model.fc7
# List of trainable variables of the layers we want to train
var_list = [v for v in tf.trainable_variables() if not skip_layers or v.name.split('/')[0] in skip_layers]
# Op for calculating the loss
with tf.name_scope("loss"):
if args.loss == 'features':
loss = tf.losses.mean_squared_error(predictions=features_output, labels=y)
elif args.loss == 'categorical':
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=class_output, labels=y))
# loss = -tf.reduce_sum(y * tf.log(class_output + 1e-10))
else:
raise ValueError('Invalid value for loss provided: {}'.format(args.loss))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
gradients = [(g, v) for g, v in gradients if g is not None]
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(args.learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('loss', loss)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = max(int(np.floor(tr_data.data_size / args.batch_size)), 1)
val_batches_per_epoch = max(int(np.floor(val_data.data_size / args.batch_size)), 1)
test_batches_per_epoch = max(int(np.floor(test_data.data_size / args.batch_size)), 1)
# Start Tensorflow session
with tf.Session() as sess:
start_epoch = 0
if args.from_checkpoint:
checkpoints = [f[:-6] for f in
[os.path.join(checkpoint_path, f) for f in os.listdir(checkpoint_path)
if f.endswith(".ckpt.index")] if os.path.isfile(f)]
if len(checkpoints) == 0:
logger.warning("No checkpoints found - starting from scratch")
args.from_checkpoint = False
else:
last_checkpoint = sorted(checkpoints)[-1]
try:
saver.restore(sess, last_checkpoint)
start_epoch = int(
last_checkpoint[-6]) + 1 # TODO: fix hard-coded single-digit (doesn't work for 11)
except Exception:
logger.warning("Unable to recover checkpoint {} - starting from scratch".format(last_checkpoint))
args.from_checkpoint = False
if not args.from_checkpoint:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
logger.info("{} Start training...".format(datetime.now()))
logger.info("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path))
# Loop over number of epochs
best_val_epoch, best_val_loss = None, math.inf
for epoch in range(start_epoch, args.num_epochs):
logger.info("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
sess.run(train_op, feed_dict={x: img_batch,
y: label_batch,
keep_prob: args.dropout_rate})
# Generate summary with the current batch of data and write to file
if True or step % args.display_step == 0: # TODO: remove True
s = sess.run(merged_summary, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
writer.add_summary(s, epoch * train_batches_per_epoch + step)
# save checkpoint of the model
if epoch % args.save_every == 0:
logger.info("{} Saving checkpoint of model...".format(datetime.now()))
checkpoint_name = os.path.join(checkpoint_path, 'model_epoch{:d}.ckpt'.format(epoch + 1))
save_path = saver.save(sess, checkpoint_name)
logger.info("{} Model checkpoint saved at {}".format(datetime.now(), save_path))
# Validate the model on the entire validation set
logger.info("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
val_loss = 0.
val_count = 0
for _ in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
_loss = sess.run(loss, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
val_loss += _loss
val_count += 1
val_loss /= val_count
logger.info("{} Validation Loss = {:.4f}".format(datetime.now(), val_loss))
if val_loss < best_val_loss:
best_val_loss, best_val_epoch = val_loss, epoch
saver.save(sess, os.path.join(checkpoint_path, 'model_best.ckpt'))
# Test the model
logger.info("{} Start test".format(datetime.now()))
test_predictions = np.empty((test_data.data_size, num_classes or 4096))
sess.run(test_init_op)
for batch_num in range(test_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
preds = sess.run(features_output, feed_dict={x: img_batch, keep_prob: 1.})
test_predictions[batch_num * args.batch_size:(batch_num + 1) * args.batch_size, :] = preds
with open(predictions_file, 'w') as f:
for img_path, prediction in zip(test_data._img_paths, test_predictions):
f.write('{} {}\n'.format(img_path, ",".join([str(p) for p in prediction])))
logger.info("Wrote predictions to {}".format(predictions_file))
elif epoch - best_val_epoch > args.patience:
logger.info("Validation loss has not decreased for {:d} epochs - stop (best epoch: {:d})".format(
epoch + 1 - best_val_epoch, best_val_epoch))
break
num_classes=num_classes)
val_text_k = TextualDataGenerator(valid_text_k,valid_k_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
tes_text_i = TextualDataGenerator(test_text_i,test_i_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
tes_text_j = TextualDataGenerator(test_text_j,test_j_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
tes_text_k = TextualDataGenerator(test_text_k,test_k_label,
batch_size_text=int(batch_size/3),
num_classes=num_classes)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
i_text_ite = Iterator.from_structure(tra_text_i.data.output_types,
tra_text_i.data.output_shapes)
j_text_ite = Iterator.from_structure(tra_text_j.data.output_types,
tra_text_j.data.output_shapes)
k_text_ite = Iterator.from_structure(tra_text_k.data.output_types,
tra_text_k.data.output_shapes)
next_batch = iterator.get_next()
next_batchi = i_text_ite.get_next()
next_batchj = j_text_ite.get_next()
next_batchk = k_text_ite.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
from alexnet import AlexNet
from dataprocess import ImageDataGenerator
from datetime import datetime
from tensorflow.contrib.data import Iterator
from config import *
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
with tf.device('/cpu:0'):
val_data = ImageDataGenerator(rest_file,
mode='inference',
batch_size=test_batch_size,
num_classes=num_classes,
shuffle=False)
iterator = Iterator.from_structure(val_data.data.output_types,
val_data.data.output_shapes)
next_batch = iterator.get_next()
validation_init_op = iterator.make_initializer(val_data.data)
x = tf.placeholder(tf.float32, [test_batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [test_batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
model = AlexNet(x, keep_prob, num_classes, train_layers)
score = model.fc8_softmax
with tf.name_scope("accuracy"):
index=tf.argmax(score, 1)
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
saver = tf.train.Saver()
batch_size = FLAGS.bs
# Network params
num_classes = FLAGS.num_class
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
test_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator_test = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
images_batch_test, labels_batch_test = iterator_test.get_next()
# Ops for initializing the two different iterators
test_init_op = iterator_test.make_initializer(test_data.data)
images_batch = tf.concat([images_batch_test, images_batch_test], axis=0)
y = tf.concat([labels_batch_test, labels_batch_test], axis=0)
with slim.arg_scope(resnet.resnet_arg_scope()):
Gap5, CONV5, net, _ = resnet.resnet_v2_50(images_batch, is_training=False)
net = tf.nn.dropout(net, 1.0)
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
val_labels = tf.constant([0, 0, 1])
tr_dataset = Dataset.from_tensor_slices((train_imgs, train_labels))
val_dataset = Dataset.from_tensor_slices((val_imgs, val_labels))
# iterator = Iterator.from_structure(tr_dataset.output_types, tr_dataset.output_shapes)
tr_dataset = tr_dataset.map(input_parser)
val_dataset = val_dataset.map(input_parser)
batch_size = 2
# dataset = tr_dataset.batch(batch_size)
batched_dataset = tr_dataset.batch(batch_size)
iterator = Iterator.from_dataset(batched_dataset)
next_elements = iterator.get_next()
training_init_op = iterator.make_initializer(batched_dataset)
# validation_init_op = iterator.make_initializer(val_dataset)
with tf.Session() as sess:
sess.run(training_init_op)
while True:
try:
(source, label) = sess.run(next_elements)
print(source)
print(label)
except tf.errors.OutOfRangeError:
break
def build_loss(patch, patch_est):
loss = tf.losses.mean_squared_error(labels=patch, predictions=patch_est)
return loss
# create tf dataset from input data
training_dataset = tf.contrib.data.Dataset.from_tensor_slices((training_measurements, training_patches))
validation_dataset = tf.contrib.data.Dataset.from_tensor_slices((validation_measurements, validation_patches))
nEpochs = 10
training_dataset = training_dataset.batch(1000)
validation_dataset = validation_dataset.batch(3000)
# initialize iterator
iterator = Iterator.from_structure(training_dataset.output_types, training_dataset.output_shapes)
next_measurement, next_patch = iterator.get_next()
training_init_op = iterator.make_initializer(training_dataset)
validation_init_op = iterator.make_initializer(validation_dataset)
patch_est = build_linear_mapping(next_measurement)
loss = build_loss(patch_est, next_patch)
# define optimization procedure
learning_rate = 0.01
learning_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)
with tf.Session() as sess:
noise2 = tf.random_normal([32, 1024, 128])
cloud = generate_cloud(tf.expand_dims(feature, axis=1), noise2)
return cloud
######################################### main #############################################
######################################### main #############################################
######################################### main #############################################
######################################### main #############################################
######################################### main #############################################
cloud_provider = tf.data.Dataset.from_generator(provide_data, output_types=(tf.float32, tf.float32), \
output_shapes=(tf.TensorShape([32, 1024, 3]), tf.TensorShape([32,40])))
point_clouds, cloud_labels = cloud_provider.make_one_shot_iterator().get_next()
iterator = Iterator.from_structure(cloud_provider.output_types,
cloud_provider.output_shapes)
training_init_op = iterator.make_initializer(cloud_provider)
noise = tf.random_normal([FLAGS.batch_size, FLAGS.noise_dims])
#with tf.variable_scope("my_scope", reuse=tf.AUTO_REUSE):
# Build the generator and discriminator.
gan_model = tfgan.gan_model(
generator_fn=conditional_generator, # you define
discriminator_fn=conditional_discriminator, # you define
real_data=point_clouds,
generator_inputs=(noise, cloud_labels))
# Build the GAN loss.
gan_loss = tfgan.gan_loss(
gan_model,
#gradient_penalty_weight=1.0,
def train(hps, design):
"""Training loop."""
train_records = _get_tfrecord_files_from_dir(
FLAGS.train_data_path) #get tfrecord files for train
train_iterator = petct_input.build_input(train_records, hps.batch_size,
hps.num_epochs, FLAGS.mode)
train_iterator_handle = train_iterator.string_handle()
if not FLAGS.val_data_path == '': # skip validation if no path
val_records = _get_tfrecord_files_from_dir(
FLAGS.val_data_path) # get tfrecord files for val
val_iterator = petct_input.build_input(val_records, hps.batch_size,
hps.num_epochs, 'valid')
val_iterator_handle = val_iterator.string_handle()
handle = tf.placeholder(tf.string, shape=[], name='data')
iterator = Iterator.from_string_handle(handle, train_iterator.output_types,
train_iterator.output_shapes)
ct, pt, ctlb, ptlb, bglb = iterator.get_next()
model = fuse_cnn_petct.FuseNet(hps, design, ct, pt, ctlb, ptlb, bglb,
FLAGS.mode)
model.build_cross_modal_model()
# for use in loading later
#tf.get_collection('model')
#tf.add_to_collection('model',model)
# put get metrics ops here for train and val
with tf.variable_scope('metrics'):
tr_summary_op, tr_precision_op, tr_recall_op, tr_accuracy_op, tr_rmse_op = get_metrics_ops(
model, 'train')
val_summary_op, val_precision_op, val_recall_op, val_accuracy_op, val_rmse_op = get_metrics_ops(
model, 'valid')
# needed for input handlers
g_init_op = tf.global_variables_initializer()
l_init_op = tf.local_variables_initializer()
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True, device_count={'GPU': 1})) as mon_sess:
# Need a saver to save and restore all the variables.
saver = tf.train.Saver()
if FLAGS.DEBUG:
print('ENABLING DEBUG')
mon_sess = tf_debug.LocalCLIDebugWrapperSession(mon_sess)
mon_sess.add_tensor_filter("has_inf_or_nan",
tf_debug.has_inf_or_nan)
training_handle = mon_sess.run(train_iterator_handle)
if not FLAGS.val_data_path == '': # skip validation if no path
validation_handle = mon_sess.run(val_iterator_handle)
train_writer = tf.summary.FileWriter(FLAGS.log_root + '/train',
mon_sess.graph)
if not FLAGS.val_data_path == '': # skip validation if no path
valid_writer = tf.summary.FileWriter(FLAGS.log_root + '/valid')
mon_sess.run([g_init_op, l_init_op])
summary = None
step = None
val_summary = None
#check = 1
while True:
try:
## FIRST RUN TRAINING OP BASED ON OUTPUT STYLE
if FLAGS.output_style == fuse_cnn_petct.STYLE_SPLIT:
# get PET and CT recons separately
_, summary, step, loss, p, r, a, e, cts, pts, trcts, trpts, trbgs, recon_cts, recon_pts, ct_preds, pt_preds = mon_sess.run(
[
model.train_op, tr_summary_op, model.global_step,
model.cost, tr_precision_op, tr_recall_op,
tr_accuracy_op, tr_rmse_op, model.ct, model.pt,
model.lbct, model.lbpt, model.lbbg, model.ct_pred,
model.pt_pred, model.ct_probabilities,
model.pt_probabilities
],
feed_dict={
handle: training_handle,
model.is_training: True
})
elif FLAGS.output_style == fuse_cnn_petct.STYLE_SINGLE:
# get PET and CT recons together
_, summary, step, loss, p, r, a, e, cts, pts, trallpos, trbgs, recon_all, all_preds = mon_sess.run(
[
model.train_op, tr_summary_op, model.global_step,
model.cost, tr_precision_op, tr_recall_op,
tr_accuracy_op, tr_rmse_op, model.ct, model.pt,
model.lb_pos_gt, model.lbbg, model.all_pred,
model.all_probabilities
],
feed_dict={
handle: training_handle,
model.is_training: True
})
if step % FLAGS.train_iter == 0:
print(
'[TRAIN] STEP: %d, LOSS: %.5f, PRECISION: %.5f, RECALL: %.5f, ACCURACY: %.5f, RMSE: %.5f'
% (step, loss, p, r, a, e))
train_writer.add_summary(summary, step)
train_writer.flush()
if FLAGS.IMSAVE > 0:
if step % FLAGS.IMSAVE == 0:
print('SAVING IMAGES')
if FLAGS.output_style == fuse_cnn_petct.STYLE_SPLIT:
_saveImages(hps.batch_size,
step,
cts,
pts,
trcts=trcts,
trpts=trpts,
trbgs=trbgs,
recon_cts=recon_cts,
recon_pts=recon_pts,
ct_preds=ct_preds,
pt_preds=pt_preds)
elif FLAGS.output_style == fuse_cnn_petct.STYLE_SINGLE:
_saveImages(hps.batch_size,
step,
cts,
pts,
trallpos=trallpos,
trbgs=trbgs,
recon_all=recon_all,
all_preds=all_preds)
if not FLAGS.val_data_path == '': # skip validation if no path
if step % FLAGS.val_iter == 0:
_, val_summary, loss, p, r, a, e = mon_sess.run(
[
model.val_op, val_summary_op, model.cost,
val_precision_op, val_recall_op,
val_accuracy_op, val_rmse_op
],
feed_dict={
handle: validation_handle,
model.is_training: False
})
val_step = step
print(
'[VALID] STEP: %d, LOSS: %.5f, PRECISION: %.5f, RECALL: %.5f, ACCURACY: %.5f, RMSE: %.5f'
% (step, loss, p, r, a, e))
valid_writer.add_summary(val_summary, step)
valid_writer.flush()
if step % FLAGS.chkpt_iter == 0:
save_loc = FLAGS.log_root + '/' + FLAGS.chkpt_file + str(
step) + '.ckpt'
save_path = saver.save(mon_sess, save_loc)
print('Model saved in path: %s' % save_path)
except tf.errors.OutOfRangeError:
print('OUT OF DATA - ENDING')
# now finished training (either train or validation has run out)
train_writer.add_summary(summary, step)
train_writer.flush()
if not FLAGS.val_data_path == '': # skip validation if no path
valid_writer.add_summary(val_summary, val_step)
valid_writer.flush()
save_loc = FLAGS.log_root + '/' + FLAGS.chkpt_file + str(
step) + '-end.ckpt'
save_path = saver.save(mon_sess, save_loc)
print('Model saved in path: %s' % save_path)
break
if __name__ == '__main__':
batch_size = 5
attr_label_num = 30
img_loader = image_bleach.ImageLoader(
[global_configs.pic2attr_tfrecord_train_path], batch_size, 100, 40000)
train_dataset = img_loader.launch_tfrecord_dataset()
train_iterator = train_dataset.make_one_shot_iterator()
# ========================= 数据导入 =========================
# =================== 用handle导入，feedble ===================
# 构造一个可导入(feedble)的句柄占位符，可以通过这个将训练集的句柄或者验证集的句柄传入
handle = tf.placeholder(tf.string, shape=[])
iterator = Iterator.from_string_handle(handle, train_dataset.output_types,
train_dataset.output_shapes)
pic_name_batch, pic_class_batch, attr_label_batch, img_batch = iterator.get_next(
)
# 从迭代器中出来的是一个二维数组，而用到的id、effect_len和label是要一个一维数组，需要reshape以下
pic_name_batch = tf.reshape(pic_name_batch, [batch_size])
pic_class_batch = tf.reshape(pic_class_batch, [batch_size])
attr_label_batch = tf.reshape(attr_label_batch,
[batch_size, attr_label_num])
# ==================/ 用handle导入，feedble /==================
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
sess.run(tf.global_variables_initializer()) # 所有变量初始化
# 获得训练集和验证集的引用句柄，后面导入数据到模型用
def train(config_file):
# 1, load configuration parameters
config = parse_config(config_file)
config_data = config['data']
config_net = config['network']
config_train = config['training']
random.seed(config_train.get('random_seed', 1))
assert(config_data['with_ground_truth'])
net_type = config_net['net_type']
net_name = config_net['net_name']
class_num = config_net['class_num']
batch_size = config_data.get('batch_size', 5)
# 2, construct graph
full_data_shape = [batch_size] + config_data['data_shape']
full_label_shape = [batch_size] + config_data['label_shape']
x = tf.placeholder(tf.float32, shape = full_data_shape)
w = tf.placeholder(tf.float32, shape = full_label_shape)
y = tf.placeholder(tf.int64, shape = full_label_shape)
w_regularizer = regularizers.l2_regularizer(config_train.get('decay', 1e-7))
b_regularizer = regularizers.l2_regularizer(config_train.get('decay', 1e-7))
net_class = NetFactory.create(net_type)
net = net_class(num_classes = class_num,
w_regularizer = w_regularizer,
b_regularizer = b_regularizer,
name = net_name)
net.set_params(config_net)
predicty = net(x, is_training = True)
proby = tf.nn.softmax(predicty)
loss_func = LossFunction(n_class=class_num)
loss = loss_func(predicty, y, weight_map = w)
print('size of predicty:',predicty)
# 3, initialize session and saver
lr = config_train.get('learning_rate', 1e-3)
opt_step = tf.train.AdamOptimizer(lr).minimize(loss)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
loader = DataLoader(config_data)
train_data = loader.get_dataset('train', shuffle = True)
batch_per_epoch = loader.get_batch_per_epoch()
train_iterator = Iterator.from_structure(train_data.output_types,
train_data.output_shapes)
next_train_batch = train_iterator.get_next()
train_init_op = train_iterator.make_initializer(train_data)
# 4, start to train
loss_file = config_train['model_save_prefix'] + "_loss.txt"
start_it = config_train.get('start_iteration', 0)
if( start_it > 0):
saver.restore(sess, config_train['model_pre_trained'])
loss_list, temp_loss_list = [], []
for n in range(start_it, config_train['maximal_iteration']):
if((n-start_it)%batch_per_epoch == 0):
sess.run(train_init_op)
one_batch = sess.run(next_train_batch)
feed_dict = {x:one_batch['image'], w:one_batch['weight'], y:one_batch['label']}
opt_step.run(session = sess, feed_dict=feed_dict)
loss_value = loss.eval(feed_dict = feed_dict)
temp_loss_list.append(loss_value)
if((n+1)%config_train['loss_display_iteration'] == 0):
avg_loss = np.asarray(temp_loss_list, np.float32).mean()
t = time.strftime('%X %x %Z')
print(t, 'iter', n+1,'loss', avg_loss)
loss_list.append(avg_loss)
np.savetxt(loss_file, np.asarray(loss_list))
temp_loss_list = []
if((n+1)%config_train['snapshot_iteration'] == 0):
saver.save(sess, config_train['model_save_prefix']+"_{0:}.ckpt".format(n+1))
sess.close()
target_val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False,
mean='/home/camalab/caffe/models/vgg19/dog129sandt_mean.npy')
tst_data = ImageDataGenerator(
test_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False,
mean='/home/camalab/caffe/models/vgg19/dog129sandt_mean.npy')
# create an reinitializable iterator given the dataset structure
source_iterator = Iterator.from_structure(
source_tr_data.data.output_types, source_tr_data.data.output_shapes)
target_iterator = Iterator.from_structure(
target_tr_data.data.output_types, target_tr_data.data.output_shapes)
source_next_batch = source_iterator.get_next()
target_next_batch = target_iterator.get_next()
# Ops for initializing the two different iterators
source_training_init_op = source_iterator.make_initializer(source_tr_data.data)
target_training_init_op = target_iterator.make_initializer(target_tr_data.data)
source_validation_init_op = source_iterator.make_initializer(
source_val_data.data)
target_validation_init_op = target_iterator.make_initializer(
target_val_data.data)
testing_init_op = target_iterator.make_initializer(tst_data.data)
print("loading data [train and test] \n")
filename_train = os.path.join(conf.DATA_DIR, "train.tfrecords")
filename_test = os.path.join(conf.DATA_DIR, "test.tfrecords")
#---------------read TFRecords data for training
data_train = tf.data.TFRecordDataset(filename_train)
data_train = data_train.map(data.parser_tfrecord)
data_train = data_train.batch(conf.BATCH_SIZE)
data_train = data_train.shuffle(conf.ESTIMATED_NUMBER_OF_BATCHES)
#---------------read TFRecords data for validation
data_test = tf.data.TFRecordDataset(filename_test)
data_test = data_test.map(data.parser_tfrecord)
data_test = data_test.batch(conf.BATCH_SIZE)
data_test = data_test.shuffle(conf.ESTIMATED_NUMBER_OF_BATCHES_TEST)
#defining saver to save snapshots
#defining a reinitializable iterator
iterator = Iterator.from_structure(data_train.output_types,
data_train.output_shapes)
iterator_test = Iterator.from_structure(data_test.output_types,
data_test.output_shapes)
next_batch = iterator.get_next()
next_batch_test = iterator_test.get_next()
#tensor that initialize the iterator:
training_init_op = iterator.make_initializer(data_train)
testing_init_op = iterator_test.make_initializer(data_test)
print("OK")
with tf.device(device_name):
net = mnet.net()
print("train")
#to save snapshots
saver = tf.train.Saver()
#load mean
def main():
"""
Configuration Part.
"""
# Path to the textfiles for the trainings and validation set
train_file = './train.txt'
val_file = './val.txt'
# Learning params
learning_rate = 0.01
num_epochs = 100
batch_size = 10
# Network params
dropout_rate = 0.5
num_classes = 2
train_layers = ['fc8', 'fc7', 'fc6']
# How often we want to write the tf.summary data to disk
display_step = 20
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "./tmp/tensorboard"
checkpoint_path = "./tmp/checkpoints"
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
score = model.fc8
pred = tf.nn.softmax(score)
# List of trainable variables of the layers we want to train
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
# optimizer = tf.train.GradientDescentOptimizer(learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size))
# Start Tensorflow session
with tf.Session() as sess:
print("# isTrain : ", args.istrain)
if not args.istrain:
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
print("Reading model parameters from %s" %
ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("checkpoint path isn't correct")
return
file = tf.read_file(args.input_path)
decoded_img = tf.image.decode_jpeg(file, channels=3)
resized_img = tf.image.resize_images(decoded_img, [227, 227])
resized_img = tf.reshape(resized_img, [1, 227, 227, 3])
print("# decoded img : ", decoded_img.eval().shape)
pred_ = sess.run(pred,
feed_dict={
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("P(man|data) : ", pred_[0][0])
print("P(woman|data) : ", pred_[0][1])
img = decoded_img.eval()
plt.imshow(img)
plt.show()
if args.visualize:
w1, w2, w3, w4, w5 = sess.run(model.weight,
feed_dict={
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("W1 : ", w1.shape)
visualize(w1[:, :, 0, :25])
print("W2 : ", w2.shape)
visualize(w2[:, :, 0, :25])
print("W3 : ", w3.shape)
visualize(w3[:, :, 0, :25])
print("W4 : ", w4.shape)
visualize(w4[:, :, 0, :25])
print("W5 : ", w5.shape)
visualize(w5[:, :, 0, :25])
f1, f2, f3, f4, f5 = sess.run(model.fm, {
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("F1 : ", f1.shape)
visualize(f1[0][:, :, :25])
print("F2 : ", f2.shape)
visualize(f2[0][:, :, :25])
print("F3 : ", f3.shape)
visualize(f3[0][:, :, :25])
print("F4 : ", f4.shape)
visualize(f4[0][:, :, :25])
print("F5 : ", f5.shape)
visualize(f5[0][:, :, :25])
return
else:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
train_acc = 0.
train_count = 0.
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
acc, _ = sess.run([accuracy, train_op],
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: dropout_rate
})
train_acc += acc
train_count += 1
# Generate summary with the current batch of data and write to file
# if step % display_step == 0:
# s = sess.run(merged_summary, feed_dict={x: img_batch,
# y: label_batch,
# keep_prob: 1.})
#
# writer.add_summary(s, epoch*train_batches_per_epoch + step)
train_acc /= train_count
print("{} Train Accuracy = {:.4f}".format(datetime.now(),
train_acc))
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
for ind in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
acc = sess.run(accuracy,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
test_acc += acc
test_count += 1
if epoch is 2 and ind is 0:
fm = sess.run(model.fm,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
weight = sess.run(model.weight,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
# print("fm0 : ", np.array(fm[0]).shape)
# print("fm1 : ", np.array(fm[1]).shape)
# print("fm2 : ", np.array(fm[2]).shape)
# print("fm3 : ", np.array(fm[3]).shape)
# print("fm4 : ", np.array(fm[4]).shape)
#
# print("weight0 : ", np.array(weight[0]).shape)
# print("weight1 : ", np.array(weight[1]).shape)
# print("weight2 : ", np.array(weight[2]).shape)
# print("weight3 : ", np.array(weight[3]).shape)
# print("weight4 : ", np.array(weight[4]).shape)
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(
datetime.now(), test_acc))
print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
return img_decoded, label_decoded
# training & validation 경로
train_imgs = tf.constant([("./dataset/train_in/%d_bi.png" % i)
for i in range(1, 40001)])
label_imgs = tf.constant([("./dataset/train_out/%d_gt.png" % i)
for i in range(1, 40001)])
dataset = tf.data.Dataset.from_tensor_slices((train_imgs, label_imgs))
dataset = dataset.map(input_parser)
dataset = dataset.shuffle(buffer_size=20000)
dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
iterator = Iterator.from_structure(dataset.output_types, dataset.output_shapes)
training_init_op = iterator.make_initializer(dataset)
next_element = iterator.get_next()
#valid_imgs = tf.constant([("./dataset/validation_bp/%d_bi.png" % i) for i in range(1,501)])
#valid_label_imgs = tf.constant([("./dataset/validation_gp/%d_gt.png" % i) for i in range(1,501)])
#dataset2 = Dataset.from_tensor_slices((valid_imgs,valid_label_imgs))
#dataset2= dataset2.map(input_parser)
#dataset2 = dataset2.shuffle(buffer_size=20000)
#dataset2 = dataset2.repeat()
#dataset2 = dataset2.batch(batch_size)
#iterator2 = Iterator.from_structure(dataset2.output_types,dataset2.output_shapes)
#training_init_op2 = iterator2.make_initializer(dataset2)
#next_element2 = iterator2.get_next()
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
