def train_normal():
simple_train_savepath = "/home/oole/tf_normal_save/tfnet_full"
initial_epoch = 0
train_datapath = "/home/oole/Data/training/patient_patches_jpg"
# train_datapath = '/home/oole/tf_test_data/validation'
val_datapath = "/home/oole/Data/validation/patient_patches_jpg"
logfile_path = "/home/oole/tfnetsave/tfnet_em_full_log.csv"
logreg_savepath = "/home/oole/tfnetsave/tfnet_em_full_logreg"
model_name = "model"
train_slidelist, train_slide_dimensions, old_disc_patches, _ = data_tf.collect_data(
train_datapath, batch_size)
val_slidelist, _, _, _ = data_tf.collect_data(val_datapath, batch_size)
train_patches = dataset.slidelist_to_patchlist(train_slidelist)
val_patches = dataset.slidelist_to_patchlist(val_slidelist)
np.random.shuffle(train_patches)
np.random.shuffle(val_patches)
train_accuracy, val_accuracy = train.train_net(
train_patches,
val_patches,
num_epochs=20,
batch_size=batch_size,
savepath=simple_train_savepath,
do_augment=True,
model_name=model_name)
def train_augment():
# simple_train_savepath = "/home/oole/tfnetsave/tfnet_full"
simple_train_savepath = "/home/oole/tfnetsave/tfnet_em_full"
em_train_savepath = "/home/oole/tfnetsave/tfnet_em_full"
initial_epoch = 0
train_datapath = "/home/oole/Data/training/patient_patches_jpg"
# train_datapath = '/home/oole/tf_test_data/validation'
val_datapath = "/home/oole/Data/validation/patient_patches_jpg"
logfile_path = "/home/oole/tfnetsave/tfnet_em_full_log.csv"
logreg_savepath = "/home/oole/tfnetsave/tfnet_em_full_logreg"
model_name = "model"
label_encoder = data_tf.labelencoder()
train_slidelist, train_slide_dimensions, old_disc_patches, _ = data_tf.collect_data(
train_datapath, batch_size)
val_slidelist, _, _, _ = data_tf.collect_data(val_datapath, batch_size)
train_patches = dataset.slidelist_to_patchlist(train_slidelist)
val_patches = dataset.slidelist_to_patchlist(val_slidelist)
np.random.shuffle(train_patches)
np.random.shuffle(val_patches)
# Initial training
#train_accuracy, val_accuracy = train.train_net(train_patches, val_patches, num_epochs=2, batch_size=batch_size,
# savepath=simple_train_savepath, do_augment=True, model_name=model_name)
#util.write_log_file(logfile_path, train_accuracy=train_accuracy, val_accuracy=val_accuracy)
# Test continue training
# train.train_net(train_patches, val_patches, num_epochs=2, batch_size=batch_size, savepath=simple_train_savepath,
# loadpath=simple_train_savepath, do_augment=False, model_name="model")
train_em.emtrain(train_datapath,
val_datapath,
simple_train_savepath,
em_train_savepath,
label_encoder,
batch_size,
initial_epochnum=initial_epoch,
model_name=model_name,
spatial_smoothing=False,
do_augment=True,
num_epochs=2,
dropout_ratio=0.5,
learning_rate=0.0005,
sanity_check=False,
logfile_path=logfile_path,
logreg_savepath=logreg_savepath)
def test_existing_net(slide_datapath, net_loadpath, model_name, dropout_ratio, batch_size, do_augment=False, shuffle_buffer_size=2000):
slide_list, slide_dimensions, num_patches, slide_label = data_tf.collect_data(
slide_datapath, batch_size)
patches = dataset.slidelist_to_patchlist(slide_list)
with tf.Session() as sess:
if do_augment:
input_dataset = dataset.img_dataset_augment(patches, batch_size=batch_size,
shuffle_buffer_size=shuffle_buffer_size, shuffle=True)
else:
input_dataset = dataset.img_dataset(patches, batch_size=batch_size,
shuffle_buffer_size=shuffle_buffer_size, shuffle=True)
input_iterator = input_dataset.make_initializable_iterator()
iterator_handle = sess.run(input_iterator.string_handle())
proxy_iterator_handle_ph = tf.placeholder(tf.string, shape=[])
proxy_iterator = tf.data.Iterator.from_string_handle(proxy_iterator_handle_ph,
output_types=input_iterator.output_types,
output_shapes=input_iterator.output_shapes)
x, y = proxy_iterator.get_next()
train_op, loss_op, y, accuracy_op, x, keep_prob_ph, learning_rate_ph, is_training_ph, y_pred_op, y_argmax_op = \
netutil.build_model(model_name, x, y, use_bn_1=True, use_bn_2=True, use_dropout_1=True, use_dropout_2=True)
# model saver
saver = tf.train.Saver()
########################
# load model from disc
saver.restore(sess, net_loadpath)
validate.validate_existing_net(iterator_handle, input_iterator, num_patches,
dropout_ratio, batch_size,
loss_op, accuracy_op,
keep_prob_ph, is_training_ph, proxy_iterator_handle_ph, sess)
# net_loadpath = "/home/oole/tfnetsave/tfnet_em_full_premod"
# slide_datapath = "/home/oole/Data/training/patient_patches_premod_jpg"
# model_name="model"
# dropout_ratio=0.5
# batch_size=64
# shuffle_buffer_size=2000
#
# test_existing_net(slide_datapath, net_loadpath, model_name, dropout_ratio, batch_size, do_augment=False)
import dataset
import netutil
import util
import numpy as np
""" CONFIGURATION """
dropout_ratio = 0.5
number_of_epochs = 5
batch_size = 64
lr = 0.0001
#### log dir
log_directory = "/home/oole/tf_log"
""" LOAD DATA """
# training_data_path = 'D:/Data/tf_test_data/validation'
training_data_path = '/home/oole/tf_test_data/validation'
train_slidelist, train_slide_dimensions, train_num_patches, train_slide_label = data_tf.collect_data(
training_data_path, batch_size)
patches = dataset.slidelist_to_patchlist(train_slidelist)
no_patches = len(patches)
""" CREATE TRAINING AND VALIDATION DATASET """
train_dataset = dataset.img_dataset(patches,
batch_size,
shuffle=True,
shuffle_buffer_size=no_patches)
train_iterator = train_dataset.make_initializable_iterator()
val_dataset = dataset.img_dataset(patches,
batch_size,
shuffle=False,
shuffle_buffer_size=no_patches)