def get_input_flow(self):
ds_train = CellImageDataManagerTrain()
# ds_train = MapDataComponent(ds_train, random_affine) # TODO : no improvement?
ds_train = MapDataComponent(ds_train, random_color)
# ds_train = MapDataComponent(ds_train, random_scaling)
ds_train = MapDataComponent(
ds_train,
mask_size_normalize) # Resize by instance size - normalization
ds_train = MapDataComponent(
ds_train, lambda x: resize_shortedge_if_small(x, self.img_size))
ds_train = MapDataComponent(
ds_train, lambda x: random_crop(x, self.img_size, self.img_size))
ds_train = MapDataComponent(ds_train, random_flip_lr)
ds_train = MapDataComponent(ds_train, random_flip_ud)
# ds_train = MapDataComponent(ds_train, data_to_elastic_transform_wrapper)
ds_train = MapDataComponent(ds_train, erosion_mask)
ds_train = MapData(
ds_train, lambda x: data_to_segment_input(
x, is_gray=False, unet_weight=True))
ds_train = PrefetchData(ds_train, 256, 24)
ds_train = BatchData(ds_train, self.batchsize)
ds_train = MapDataComponent(ds_train, data_to_normalize1)
ds_valid = CellImageDataManagerValid()
ds_valid = MapDataComponent(
ds_valid, lambda x: resize_shortedge_if_small(x, self.img_size))
ds_valid = MapDataComponent(
ds_valid, lambda x: random_crop(x, self.img_size, self.img_size))
ds_valid = MapDataComponent(ds_valid, erosion_mask)
ds_valid = MapData(
ds_valid, lambda x: data_to_segment_input(
x, is_gray=False, unet_weight=True))
ds_valid = PrefetchData(ds_valid, 20, 12)
ds_valid = BatchData(ds_valid, self.batchsize, remainder=True)
ds_valid = MapDataComponent(ds_valid, data_to_normalize1)
ds_valid2 = CellImageDataManagerValid()
ds_valid2 = MapDataComponent(
ds_valid2, lambda x: resize_shortedge_if_small(x, self.img_size))
ds_valid2 = MapDataComponent(
ds_valid2,
lambda x: center_crop_if_tcga(x, self.img_size, self.img_size))
# ds_valid2 = MapDataComponent(ds_valid2, lambda x: resize_shortedge(x, self.img_size))
ds_valid2 = MapData(ds_valid2,
lambda x: data_to_segment_input(x, is_gray=False))
ds_valid2 = MapDataComponent(ds_valid2, data_to_normalize1)
ds_test = CellImageDataManagerTest()
ds_test = MapDataComponent(
ds_test, lambda x: resize_shortedge_if_small(x, self.img_size))
# ds_test = MapDataComponent(ds_test, lambda x: resize_shortedge(x, self.img_size))
ds_test = MapData(ds_test, lambda x: data_to_image(x, is_gray=False))
ds_test = MapDataComponent(ds_test, data_to_normalize1)
return ds_train, ds_valid, ds_valid2, ds_test
def main():
cifar = cifarDataLoader()
cifar.load('../cifar-100-python/train', '../cifar-100-python/meta')
train_generator = ImageDataGenerator( # rotation_range=30,
# width_shift_range=0.2,
# height_shift_range=0.2,
rescale=1.,
# zoom_range=0.2,
# horizontal_flip=True,
fill_mode='nearest',
cval=0)
batch_input = train_generator.flow(cifar.X, cifar.Y, 64)
with tf.device('/device:GPU:1'):
# CNN = ResNet(height=32, width=32, depth=3, nums_classes=100, lr=0.1)
CNN = DenseNet(height=32,
width=32,
depth=3,
nums_classes=100,
lr=0.001,
prob=0.)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
# sess.run(tf.global_variables_initializer())
# saver.restore(sess, tf.train.latest_checkpoint('./ckpt_resnet_rp/'))
# saver.restore(sess, './ckpt_resnet_rp/MyModel-50000')
# saver.restore(sess, tf.train.latest_checkpoint('./ckpt_densenet_rp/'))
saver.restore(sess, './ckpt_densenet_rp/MyModel-32000')
for x, y in batch_input:
batch_x, batch_y = batch_input.next()
for i in range(batch_x.shape[0]):
batch_x[i] = random_crop(batch_x[i],
padding=4,
is_flip=True,
prob=0.5,
is_crop=True)
batch_x[i] = random_erase(batch_x[i],
prob=0.5,
min=0.1,
max=0.4)
batch_y = one_hot_encode(batch_y, 100)
batch_loss, batch_accuracy, _, train_step = sess.run(
[CNN.loss, CNN.accuracy, CNN.train_op, CNN.global_step],
feed_dict={
CNN.X: batch_x,
CNN.Y: batch_y
})
if train_step % 50 == 0:
print("traing_step: ", train_step, "training_loss: ",
batch_loss, "batch_accuracy: ", batch_accuracy)
if train_step % 2000 == 0:
# saver.save(sess, './ckpt_resnet_rp/MyModel', global_step=train_step)
saver.save(sess,
'./ckpt_densenet_rp/MyModel',
global_step=train_step)
if train_step == 100000:
break
def get_image(self, image_url):
img = cv2.imread(image_url)
img = np.reshape(img, [1, 1920, 2560, 3])
for images in gen_data.flow(img, batch_size=1, shuffle=False):
img = images
break
img = np.reshape(img, [1920, 2560, 3])
img = random_crop(img)
img = cv2.resize(img, (224, 224))
return img
def main():
cifar = cifarDataLoader()
cifar.load('../cifar-100-python/test', '../cifar-100-python/meta')
batch_size = 50
data_size = cifar.X.shape[0]
with tf.device('/device:GPU:1'):
# CNN = ResNet(height=32, width=32, depth=3, nums_classes=100, trainable=False)
CNN = DenseNet(height=32, width=32, depth=3, nums_classes=100, trainable=False)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=50)
# saver.restore(sess, tf.train.latest_checkpoint('./ckpt_resnet_rp/'))
# saver.restore(sess, './ckpt_resnet_rp/MyModel-100000')
# saver.restore(sess, tf.train.latest_checkpoint('./ckpt_densenet_rp/'))
saver.restore(sess, './ckpt_densenet_rp/MyModel-36000')
total_assemble_ac = []
total_accuracy = []
for i in range(data_size//batch_size):
batch_x = cifar.X[i*batch_size:(i+1)*batch_size]
batch_x = np.repeat(batch_x, 4, axis=0)
for j in range(batch_size):
batch_x[j*4+1] = random_crop(batch_x[j*4+1], padding=0, is_flip=True, prob=0., is_crop=False)
batch_x[j*4+2] = random_crop(batch_x[j*4+2], padding=4, is_flip=False, is_crop=True)
batch_x[j*4+3] = random_crop(batch_x[j*4+3], padding=4, is_flip=True, prob=0., is_crop=True)
batch_x = np.asarray(batch_x, np.float32)
batch_y = cifar.Y[i*batch_size:(i+1)*batch_size]
batch_y = one_hot_encode(batch_y, 100)
# pdb.set_trace()
batch_y = np.repeat(batch_y, 4, axis=0)
batch_pred, batch_accuracy = sess.run([CNN.pred, CNN.accuracy],
feed_dict={CNN.X: batch_x, CNN.Y: batch_y})
ensemble_labels = []
for j in range(batch_size):
ensemble_labels.append(stats.mode(batch_pred[4*j:4*(j+1)])[0][0])
ensemble_labels = np.asarray(ensemble_labels, np.int32)
# pdb.set_trace()
nums_correct = np.equal(ensemble_labels, cifar.Y[i*batch_size:(i+1)*batch_size])
batch_ensemble_ac = np.mean(nums_correct)
print("test_step: ", i+1, " test_loss: ", batch_ensemble_ac, " test_accuracy: ", batch_accuracy)
total_ensemble_ac.append(batch_ensemble_ac)
total_accuracy.append(batch_accuracy)
print("Finally, final_ensemble_ac:", np.mean(total_ensemble_ac), " final_test_accuracy:", np.mean(total_accuracy))
def get_input_flow(self):
ds_train = CellImageDataManagerTrain()
# Augmentation :
ds_train = MapDataComponent(ds_train, random_affine)
ds_train = MapDataComponent(ds_train, random_color)
# ds_train = MapDataComponent(ds_train, random_color2) # not good
ds_train = MapDataComponent(ds_train, random_scaling)
ds_train = MapDataComponent(
ds_train, lambda x: resize_shortedge_if_small(x, 224))
ds_train = MapDataComponent(ds_train,
lambda x: random_crop(x, 224, 224))
ds_train = MapDataComponent(ds_train, random_flip_lr)
# ds_train = MapDataComponent(ds_train, data_to_elastic_transform_wrapper)
ds_train = MapDataComponent(ds_train, random_flip_ud)
if self.unet_weight:
ds_train = MapDataComponent(ds_train, erosion_mask)
ds_train = PrefetchData(ds_train, 1000, 24)
ds_train = MapData(
ds_train, lambda x: data_to_segment_input(x, not self.is_color,
self.unet_weight))
ds_train = BatchData(ds_train, self.batchsize)
ds_train = MapDataComponent(ds_train, data_to_normalize1)
ds_train = PrefetchData(ds_train, 10, 2)
ds_valid = CellImageDataManagerValid()
ds_valid = MapDataComponent(ds_valid,
lambda x: center_crop(x, 224, 224))
if self.unet_weight:
ds_valid = MapDataComponent(ds_valid, erosion_mask)
ds_valid = MapData(
ds_valid, lambda x: data_to_segment_input(x, not self.is_color,
self.unet_weight))
ds_valid = BatchData(ds_valid, self.batchsize, remainder=True)
ds_valid = MapDataComponent(ds_valid, data_to_normalize1)
ds_valid = PrefetchData(ds_valid, 20, 24)
ds_valid2 = CellImageDataManagerValid()
ds_valid2 = MapDataComponent(
ds_valid2, lambda x: resize_shortedge_if_small(x, 224))
ds_valid2 = MapData(
ds_valid2, lambda x: data_to_segment_input(x, not self.is_color))
ds_valid2 = MapDataComponent(ds_valid2, data_to_normalize1)
ds_test = CellImageDataManagerTest()
ds_test = MapDataComponent(ds_test,
lambda x: resize_shortedge_if_small(x, 224))
ds_test = MapData(ds_test,
lambda x: data_to_image(x, not self.is_color))
ds_test = MapDataComponent(ds_test, data_to_normalize1)
return ds_train, ds_valid, ds_valid2, ds_test
def main():
cifar = cifarDataLoader()
cifar.load('../cifar-100-python/test', '../cifar-100-python/meta')
batch_size = 250
data_size = cifar.X.shape[0]
with tf.device('/device:GPU:1'):
# CNN = ResNet(height=32, width=32, depth=3, nums_classes=100, trainable=False)
CNN = DenseNet(height=32,
width=32,
depth=3,
nums_classes=100,
trainable=False)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=50)
# saver.restore(sess, tf.train.latest_checkpoint('./ckpt_resnet_rp/'))
# saver.restore(sess, './ckpt_resnet_rp/MyModel-100000')
# saver.restore(sess, tf.train.latest_checkpoint('./ckpt_densenet_rp/'))
saver.restore(sess, './ckpt_densenet_rp/MyModel-36000')
total_loss = []
total_accuracy = []
for i in range(data_size // batch_size):
batch_x = cifar.X[i * batch_size:(i + 1) * batch_size]
batch_x = batch_x.astype(np.float32)
for j in range(batch_x.shape[0]):
batch_x[j] = random_crop(batch_x[j],
padding=4,
is_flip=False,
is_crop=False)
# batch_x[i] = random_erase(batch_x[i], prob=0.5, min=0.1, max=0.4)
batch_y = cifar.Y[i * batch_size:(i + 1) * batch_size]
batch_y = one_hot_encode(batch_y, 100)
batch_loss, batch_accuracy = sess.run([CNN.loss, CNN.accuracy],
feed_dict={
CNN.X: batch_x,
CNN.Y: batch_y
})
print("test_step: ", i + 1, " test_loss: ", batch_loss,
" test_accuracy: ", batch_accuracy)
total_loss.append(batch_loss)
total_accuracy.append(batch_accuracy)
print("Finally, final_test_loss:", np.mean(total_loss),
" final_test_accuracy:", np.mean(total_accuracy))
def get_input_flow(self):
ds_train = CellImageDataManagerTrain()
# ds_train = MapDataComponent(ds_train, random_affine) # TODO : no improvement?
ds_train = MapDataComponent(ds_train, random_color)
# ds_train = MapDataComponent(ds_train, random_scaling)
ds_train = MapDataComponent(
ds_train,
mask_size_normalize) # Resize by instance size - normalization
ds_train = MapDataComponent(
ds_train, lambda x: resize_shortedge_if_small(x, self.img_size))
# ds_train = MapDataComponent(ds_train, lambda x: pad_if_small(x, self.img_size)) # preseve cell's size
ds_train = MapDataComponent(
ds_train, lambda x: random_crop(
x, self.img_size, self.img_size, padding=self.pad_size))
# ds_train = MapDataComponent(ds_train, random_add_thick_area) # TODO : worth?
ds_train = MapDataComponent(ds_train, random_flip_lr)
ds_train = MapDataComponent(ds_train, random_flip_ud)
# ds_train = MapDataComponent(ds_train, data_to_elastic_transform_wrapper)
if self.unet_weight:
ds_train = MapDataComponent(ds_train, erosion_mask)
ds_train = MapData(
ds_train, lambda x: data_to_segment_input(x, not self.is_color,
self.unet_weight))
ds_train = PrefetchData(ds_train, 256, 24)
ds_train = BatchData(ds_train, self.batchsize)
ds_train = MapDataComponent(ds_train, data_to_normalize1)
ds_valid = CellImageDataManagerValid()
ds_valid = MapDataComponent(
ds_valid, lambda x: resize_shortedge_if_small(x, self.img_size))
ds_valid = MapDataComponent(
ds_valid, lambda x: random_crop(
x, self.img_size, self.img_size, padding=self.pad_size))
if self.unet_weight:
ds_valid = MapDataComponent(ds_valid, erosion_mask)
ds_valid = MapData(
ds_valid, lambda x: data_to_segment_input(x, not self.is_color,
self.unet_weight))
ds_valid = PrefetchData(ds_valid, 32, 8)
ds_valid = BatchData(ds_valid, self.batchsize, remainder=True)
ds_valid = MapDataComponent(ds_valid, data_to_normalize1)
ds_valid2 = CellImageDataManagerValid()
ds_valid2 = MapDataComponent(
ds_valid2, lambda x: resize_shortedge_if_small(x, self.img_size))
ds_valid2 = MapDataComponent(
ds_valid2,
lambda x: center_crop_if_tcga(x, self.img_size, self.img_size))
# ds_valid2 = MapDataComponent(ds_valid2, lambda x: resize_shortedge(x, self.img_size))
ds_valid2 = MapData(
ds_valid2, lambda x: data_to_segment_input(x, not self.is_color))
ds_valid2 = MapDataComponent(ds_valid2, data_to_normalize1)
ds_test = CellImageDataManagerTest()
ds_test = MapDataComponent(
ds_test, lambda x: resize_shortedge_if_small(x, self.img_size))
# ds_test = MapDataComponent(ds_test, lambda x: resize_shortedge(x, self.img_size))
ds_test = MapData(ds_test,
lambda x: data_to_image(x, not self.is_color))
ds_test = MapDataComponent(ds_test, data_to_normalize1)
return ds_train, ds_valid, ds_valid2, ds_test