def tfreloader(mode, ep, bs, ctr, cte):
filename = data_dir + '/' + mode + '.tfrecords'
if mode == 'train':
ct = ctr
else:
ct = cte
datasets = data_input.DataSet(bs, ct, ep=ep, mode=mode, filename=filename)
return datasets
def tfreloader(mode, ep, bs, cls, ctr, cte, cva, data_dir):
filename = data_dir + '/' + mode + '.tfrecords'
if mode == 'train':
ct = ctr
elif mode == 'test':
ct = cte
else:
ct = cva
datasets = data_input.DataSet(bs,
ct,
ep=ep,
cls=cls,
mode=mode,
filename=filename)
return datasets
def train(self, data_dir, out_dir, slides, top_k=10,
valid_data_path=None, sample_rate=None, n_epoch=10, batch_size=128, save=True):
if save:
saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
outfile = os.path.join(os.path.abspath(out_dir), "{}_MIL_{}_lr_{}_drop_{}".format(
str(self.datetime), str(self.architecture),
str(self.learning_rate), str(self.dropout)))
now = datetime.now().strftime(r"%y-%m-%d %H:%M:%S")
print("------- Training begin: {} -------\n".format(now))
slide_fn = tf.placeholder(tf.string, shape=None)
slide_dataset = data_input.DataSet(inputs=slide_fn, batch_size=64)
rand_ph = tf.placeholder(tf.float32, shape=None)
def sample_slide(ds, rand, rate): # random sample from the whole slide, based on the sample_rate argument
if not sample_rate:
iter = ds.shuffled_iter()
else:
def sample_fn(data, rind): # random sample from the whole slide, based on the sample_rate argument
return rind < rate
dat = ds.get_data()
rds = tf.data.Dataset.from_tensor_slices(rand)
dat_sample = tf.data.Dataset.zip((dat, rds))
dat_sample = dat_sample.filter(sample_fn)
dat_sample = dat_sample.map(lambda dat_, rds_: dat_)
iter = dat_sample.batch(batch_size=batch_size, drop_remainder=False).make_initializable_iterator()
return iter
slide_tfr_iter = sample_slide(slide_dataset, rand_ph, sample_rate)
next_tfr_batch = slide_tfr_iter.get_next()
trn_img_ph = tf.placeholder(tf.uint8)
trn_lab_ph = tf.placeholder(tf.int64)
trn_ds = tf.data.Dataset.from_tensor_slices((trn_img_ph, trn_lab_ph))
trn_ds = trn_ds.shuffle(buffer_size=2000).batch(batch_size=batch_size)
trn_iter = trn_ds.make_initializable_iterator()
next_trn_batch = trn_iter.get_next()
if valid_data_path:
valid_data = data_input.DataSet(inputs=valid_data_path, batch_size=64)
valid_iter = valid_data.shuffled_iter()
next_val_batch = valid_iter.get_next()
try:
for epoch in range(n_epoch):
"""
Inference run: get top score tiles from each slide
"""
now = datetime.now().strftime(r"%y-%m-%d %H:%M:%S")
print('----------epoch {}: {}----------'.format(epoch, now))
trn_img_subsets = []
trn_lab_subsets = []
for slide in slides:
# s_id = slide.split('.')[0]
slide_prob = []
slide_img = []
slide_lab = []
slide_counter = 0
slide_path = data_dir + '/' + slide
self.sesh.run(slide_tfr_iter.initializer,
feed_dict={slide_fn: slide_path, rand_ph: np.random.uniform(0., 1., 200000)})
while True:
try:
imgs, labs = self.sesh.run(next_tfr_batch)
batch_pred = self.inference(imgs)[:, 1]
batch_top_ind = batch_pred.argsort()[-top_k:] # index of largest k probabilities
slide_counter += imgs.shape[0]
for top_ind in batch_top_ind:
top_prob = batch_pred[top_ind]
#print(top_ind)
#print(top_prob)
if (slide_counter <= top_k*batch_size or
top_prob >= np.sort(np.array(slide_prob))[-top_k]):
slide_prob.append(batch_pred[top_ind])
slide_img.append(imgs[top_ind])
slide_lab.append(labs[top_ind])
else:
pass
except tf.errors.OutOfRangeError:
break
slide_prob_ind = np.asarray(slide_prob).argsort()[:-top_k]
for ind in sorted(slide_prob_ind, reverse=True):
del slide_prob[ind]
del slide_img[ind]
del slide_lab[ind]
print('{}: {} tiles inferred from slide.'.format(slide, slide_counter))
print('Top {} probabilities: '.format(top_k))
print(slide_prob)
for i in range(len(slide_prob)):
trn_img_subsets.append(slide_img[i])
trn_lab_subsets.append(slide_lab[i])
print('Filtered images: {}'.format(len(trn_img_subsets)))
#print('Filtered labels:{}'.format(len(lab_subsets)))
trn_img_subsets = np.asarray(trn_img_subsets)
trn_lab_subsets = np.asarray(trn_lab_subsets)
self.sesh.run(trn_iter.initializer, feed_dict={trn_img_ph: trn_img_subsets,
trn_lab_ph: trn_lab_subsets})
err_train = 0
epoch_batch = 0
while True:
try:
train_X, train_Y =self.sesh.run(next_trn_batch)
train_X = utils.input_preprocessing(train_X, model=self.architecture)
feed = {self.x_in: train_X, self.y_in: train_Y}
fetches = [self.merged_summary, self.logits, self.pred,
self.cost, self.global_step, self.train_op]
summary, logits, pred, cost, i, _ = self.sesh.run(fetches=fetches, feed_dict=feed)
err_train += cost
epoch_batch += 1
except tf.errors.OutOfRangeError:
print('MIL training epoch {} finished.'.format(epoch))
print('Global step {}: average train error {}'.format(i, err_train / epoch_batch))
break
self.epoch_trained = epoch
if valid_data_path:
self.sesh.run(valid_iter.initializer)
valid_X, valid_Y = self.sesh.run(next_val_batch)
valid_X = utils.input_preprocessing(valid_X, model=self.architecture)
feed = {self.x_in: valid_X, self.y_in: valid_Y, self.training_status: False}
fetches = [self.merged_summary, self.pred,
self.cost, self.global_step]
summary, pred, cost, i = self.sesh.run(fetches=fetches, feed_dict=feed)
self.validation_logger.add_summary(summary, i)
print('MIL training epoch {} validation cost: {}'.format(self.epoch_trained, cost))
if save:
saver.save(self.sesh, outfile, global_step=None)
print('Trained model saved to {}'.format(outfile))
try:
self.train_logger.flush()
self.train_logger.close()
self.validation_logger.flush()
self.validation_logger.close()
except AttributeError: # not logging
print('Not logging')
except KeyboardInterrupt:
pass
now = datetime.now().strftime(r"%y-%m-%d %H:%M:%S")
print("------- Training end: {} -------\n".format(now), flush=True)
print('Epochs trained: {}'.format(str(self.epoch_trained)))
i = self.global_step.eval(session = self.sesh)
print('Global steps: {}'.format(str(i)))
if save:
saver.save(self.sesh, outfile, global_step=None)
print('Trained model saved to {}'.format(outfile))
def pre_train(self, pretrain_data_path, out_dir, valid_data_path=None, n_epoch=10, batch_size=128, save=True):
"""
Pretrain the model with tile level labels before MIL (the train method below)
"""
pretrain_data = data_input.DataSet(inputs=pretrain_data_path, batch_size=batch_size)
if save:
saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
outfile = os.path.join(os.path.abspath(out_dir), "{}_preMIL_{}_lr_{}_drop_{}".format(
str(self.datetime), str(self.architecture),
str(self.learning_rate), str(self.dropout)))
valid_costs = []
pretrain_iter = pretrain_data.shuffled_iter()
next_pretrn_batch = pretrain_iter.get_next()
if valid_data_path:
preval_data = data_input.DataSet(inputs=valid_data_path, batch_size=batch_size)
preval_iter = preval_data.shuffled_iter()
next_preval_batch = preval_iter.get_next()
now = datetime.now().strftime(r"%y-%m-%d %H:%M:%S")
print("------- Pre-training begin: {} -------\n".format(now))
try:
for epoch in range(n_epoch):
err_train = 0
epoch_batch = 0
self.sesh.run(pretrain_iter.initializer)
while True:
try:
pretrain_X, pretrain_Y = self.sesh.run(next_pretrn_batch)
pretrain_X = utils.input_preprocessing(pretrain_X, model=self.architecture)
feed = {self.x_in: pretrain_X, self.y_in: pretrain_Y}
fetches = [self.merged_summary, self.logits, self.pred,
self.cost, self.global_step, self.train_op]
summary, logits, pred, cost, i, _ = self.sesh.run(fetches=fetches, feed_dict=feed)
self.train_logger.add_summary(summary, i)
err_train += cost
epoch_batch += 1
except tf.errors.OutOfRangeError:
i = self.global_step.eval(session=self.sesh)
print('Epoch {} finished.'.format(epoch))
print('Global step {}: average train error {}'.format(i, err_train / epoch_batch))
break
self.epoch_pretrained = epoch
if valid_data_path:
self.sesh.run(preval_iter.initializer)
valid_X, valid_Y = self.sesh.run(next_preval_batch)
valid_X = utils.input_preprocessing(valid_X, model=self.architecture)
feed = {self.x_in: valid_X, self.y_in: valid_Y, self.training_status: False}
fetches = [self.merged_summary, self.pred,
self.cost, self.global_step]
summary, pred, cost, i = self.sesh.run(fetches=fetches, feed_dict=feed)
self.validation_logger.add_summary(summary, i)
print('Tile pre-training epoch {} validation cost: {}'.format(self.epoch_pretrained, cost))
valid_costs.append(cost)
min_valid_cost = min(valid_costs)
if cost > min_valid_cost:
print('Validation cost reached plateau. Pre-training stopped.')
break
try:
self.pretrain_logger.flush()
self.pretrain_logger.close()
self.validation_logger.flush()
self.validation_logger.close()
except AttributeError: # not logging
print('Not logging')
except KeyboardInterrupt:
pass
now = datetime.now().strftime(r"%y-%m-%d %H:%M:%S")
print("------- Pre-training end: {} -------\n".format(now), flush=True)
print('Epochs trained: {}'.format(str(self.epoch_pretrained)))
i = self.global_step.eval(session=self.sesh)
print('Global steps: {}'.format(str(i)))
if save:
saver.save(self.sesh, outfile, global_step=None)
print('Pre-trained model saved to {}'.format(outfile))
def tfreloader(mode, ep, bs):
filename = data_dir + '/' + mode + '.tfrecords'
datasets = data_input.DataSet(bs, ep=ep, mode=mode, filename=filename)
return datasets
def loader(images, bs, ct):
dataset = data_input.DataSet(bs, ct, images=images)
return dataset