def get_input_flow(self): ds_train = CellImageDataManagerTrain() # ds_train = MapDataComponent(ds_train, random_affine) # TODO : no improvement? ds_train = MapDataComponent(ds_train, random_color) # ds_train = MapDataComponent(ds_train, random_scaling) ds_train = MapDataComponent( ds_train, mask_size_normalize) # Resize by instance size - normalization ds_train = MapDataComponent( ds_train, lambda x: resize_shortedge_if_small(x, self.img_size)) ds_train = MapDataComponent( ds_train, lambda x: random_crop(x, self.img_size, self.img_size)) ds_train = MapDataComponent(ds_train, random_flip_lr) ds_train = MapDataComponent(ds_train, random_flip_ud) # ds_train = MapDataComponent(ds_train, data_to_elastic_transform_wrapper) ds_train = MapDataComponent(ds_train, erosion_mask) ds_train = MapData( ds_train, lambda x: data_to_segment_input( x, is_gray=False, unet_weight=True)) ds_train = PrefetchData(ds_train, 256, 24) ds_train = BatchData(ds_train, self.batchsize) ds_train = MapDataComponent(ds_train, data_to_normalize1) ds_valid = CellImageDataManagerValid() ds_valid = MapDataComponent( ds_valid, lambda x: resize_shortedge_if_small(x, self.img_size)) ds_valid = MapDataComponent( ds_valid, lambda x: random_crop(x, self.img_size, self.img_size)) ds_valid = MapDataComponent(ds_valid, erosion_mask) ds_valid = MapData( ds_valid, lambda x: data_to_segment_input( x, is_gray=False, unet_weight=True)) ds_valid = PrefetchData(ds_valid, 20, 12) ds_valid = BatchData(ds_valid, self.batchsize, remainder=True) ds_valid = MapDataComponent(ds_valid, data_to_normalize1) ds_valid2 = CellImageDataManagerValid() ds_valid2 = MapDataComponent( ds_valid2, lambda x: resize_shortedge_if_small(x, self.img_size)) ds_valid2 = MapDataComponent( ds_valid2, lambda x: center_crop_if_tcga(x, self.img_size, self.img_size)) # ds_valid2 = MapDataComponent(ds_valid2, lambda x: resize_shortedge(x, self.img_size)) ds_valid2 = MapData(ds_valid2, lambda x: data_to_segment_input(x, is_gray=False)) ds_valid2 = MapDataComponent(ds_valid2, data_to_normalize1) ds_test = CellImageDataManagerTest() ds_test = MapDataComponent( ds_test, lambda x: resize_shortedge_if_small(x, self.img_size)) # ds_test = MapDataComponent(ds_test, lambda x: resize_shortedge(x, self.img_size)) ds_test = MapData(ds_test, lambda x: data_to_image(x, is_gray=False)) ds_test = MapDataComponent(ds_test, data_to_normalize1) return ds_train, ds_valid, ds_valid2, ds_test
def main(): cifar = cifarDataLoader() cifar.load('../cifar-100-python/train', '../cifar-100-python/meta') train_generator = ImageDataGenerator( # rotation_range=30, # width_shift_range=0.2, # height_shift_range=0.2, rescale=1., # zoom_range=0.2, # horizontal_flip=True, fill_mode='nearest', cval=0) batch_input = train_generator.flow(cifar.X, cifar.Y, 64) with tf.device('/device:GPU:1'): # CNN = ResNet(height=32, width=32, depth=3, nums_classes=100, lr=0.1) CNN = DenseNet(height=32, width=32, depth=3, nums_classes=100, lr=0.001, prob=0.) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) saver = tf.train.Saver(tf.global_variables(), max_to_keep=100) # sess.run(tf.global_variables_initializer()) # saver.restore(sess, tf.train.latest_checkpoint('./ckpt_resnet_rp/')) # saver.restore(sess, './ckpt_resnet_rp/MyModel-50000') # saver.restore(sess, tf.train.latest_checkpoint('./ckpt_densenet_rp/')) saver.restore(sess, './ckpt_densenet_rp/MyModel-32000') for x, y in batch_input: batch_x, batch_y = batch_input.next() for i in range(batch_x.shape[0]): batch_x[i] = random_crop(batch_x[i], padding=4, is_flip=True, prob=0.5, is_crop=True) batch_x[i] = random_erase(batch_x[i], prob=0.5, min=0.1, max=0.4) batch_y = one_hot_encode(batch_y, 100) batch_loss, batch_accuracy, _, train_step = sess.run( [CNN.loss, CNN.accuracy, CNN.train_op, CNN.global_step], feed_dict={ CNN.X: batch_x, CNN.Y: batch_y }) if train_step % 50 == 0: print("traing_step: ", train_step, "training_loss: ", batch_loss, "batch_accuracy: ", batch_accuracy) if train_step % 2000 == 0: # saver.save(sess, './ckpt_resnet_rp/MyModel', global_step=train_step) saver.save(sess, './ckpt_densenet_rp/MyModel', global_step=train_step) if train_step == 100000: break
def get_image(self, image_url): img = cv2.imread(image_url) img = np.reshape(img, [1, 1920, 2560, 3]) for images in gen_data.flow(img, batch_size=1, shuffle=False): img = images break img = np.reshape(img, [1920, 2560, 3]) img = random_crop(img) img = cv2.resize(img, (224, 224)) return img
def main(): cifar = cifarDataLoader() cifar.load('../cifar-100-python/test', '../cifar-100-python/meta') batch_size = 50 data_size = cifar.X.shape[0] with tf.device('/device:GPU:1'): # CNN = ResNet(height=32, width=32, depth=3, nums_classes=100, trainable=False) CNN = DenseNet(height=32, width=32, depth=3, nums_classes=100, trainable=False) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) saver = tf.train.Saver(tf.global_variables(), max_to_keep=50) # saver.restore(sess, tf.train.latest_checkpoint('./ckpt_resnet_rp/')) # saver.restore(sess, './ckpt_resnet_rp/MyModel-100000') # saver.restore(sess, tf.train.latest_checkpoint('./ckpt_densenet_rp/')) saver.restore(sess, './ckpt_densenet_rp/MyModel-36000') total_assemble_ac = [] total_accuracy = [] for i in range(data_size//batch_size): batch_x = cifar.X[i*batch_size:(i+1)*batch_size] batch_x = np.repeat(batch_x, 4, axis=0) for j in range(batch_size): batch_x[j*4+1] = random_crop(batch_x[j*4+1], padding=0, is_flip=True, prob=0., is_crop=False) batch_x[j*4+2] = random_crop(batch_x[j*4+2], padding=4, is_flip=False, is_crop=True) batch_x[j*4+3] = random_crop(batch_x[j*4+3], padding=4, is_flip=True, prob=0., is_crop=True) batch_x = np.asarray(batch_x, np.float32) batch_y = cifar.Y[i*batch_size:(i+1)*batch_size] batch_y = one_hot_encode(batch_y, 100) # pdb.set_trace() batch_y = np.repeat(batch_y, 4, axis=0) batch_pred, batch_accuracy = sess.run([CNN.pred, CNN.accuracy], feed_dict={CNN.X: batch_x, CNN.Y: batch_y}) ensemble_labels = [] for j in range(batch_size): ensemble_labels.append(stats.mode(batch_pred[4*j:4*(j+1)])[0][0]) ensemble_labels = np.asarray(ensemble_labels, np.int32) # pdb.set_trace() nums_correct = np.equal(ensemble_labels, cifar.Y[i*batch_size:(i+1)*batch_size]) batch_ensemble_ac = np.mean(nums_correct) print("test_step: ", i+1, " test_loss: ", batch_ensemble_ac, " test_accuracy: ", batch_accuracy) total_ensemble_ac.append(batch_ensemble_ac) total_accuracy.append(batch_accuracy) print("Finally, final_ensemble_ac:", np.mean(total_ensemble_ac), " final_test_accuracy:", np.mean(total_accuracy))
def get_input_flow(self): ds_train = CellImageDataManagerTrain() # Augmentation : ds_train = MapDataComponent(ds_train, random_affine) ds_train = MapDataComponent(ds_train, random_color) # ds_train = MapDataComponent(ds_train, random_color2) # not good ds_train = MapDataComponent(ds_train, random_scaling) ds_train = MapDataComponent( ds_train, lambda x: resize_shortedge_if_small(x, 224)) ds_train = MapDataComponent(ds_train, lambda x: random_crop(x, 224, 224)) ds_train = MapDataComponent(ds_train, random_flip_lr) # ds_train = MapDataComponent(ds_train, data_to_elastic_transform_wrapper) ds_train = MapDataComponent(ds_train, random_flip_ud) if self.unet_weight: ds_train = MapDataComponent(ds_train, erosion_mask) ds_train = PrefetchData(ds_train, 1000, 24) ds_train = MapData( ds_train, lambda x: data_to_segment_input(x, not self.is_color, self.unet_weight)) ds_train = BatchData(ds_train, self.batchsize) ds_train = MapDataComponent(ds_train, data_to_normalize1) ds_train = PrefetchData(ds_train, 10, 2) ds_valid = CellImageDataManagerValid() ds_valid = MapDataComponent(ds_valid, lambda x: center_crop(x, 224, 224)) if self.unet_weight: ds_valid = MapDataComponent(ds_valid, erosion_mask) ds_valid = MapData( ds_valid, lambda x: data_to_segment_input(x, not self.is_color, self.unet_weight)) ds_valid = BatchData(ds_valid, self.batchsize, remainder=True) ds_valid = MapDataComponent(ds_valid, data_to_normalize1) ds_valid = PrefetchData(ds_valid, 20, 24) ds_valid2 = CellImageDataManagerValid() ds_valid2 = MapDataComponent( ds_valid2, lambda x: resize_shortedge_if_small(x, 224)) ds_valid2 = MapData( ds_valid2, lambda x: data_to_segment_input(x, not self.is_color)) ds_valid2 = MapDataComponent(ds_valid2, data_to_normalize1) ds_test = CellImageDataManagerTest() ds_test = MapDataComponent(ds_test, lambda x: resize_shortedge_if_small(x, 224)) ds_test = MapData(ds_test, lambda x: data_to_image(x, not self.is_color)) ds_test = MapDataComponent(ds_test, data_to_normalize1) return ds_train, ds_valid, ds_valid2, ds_test
def main(): cifar = cifarDataLoader() cifar.load('../cifar-100-python/test', '../cifar-100-python/meta') batch_size = 250 data_size = cifar.X.shape[0] with tf.device('/device:GPU:1'): # CNN = ResNet(height=32, width=32, depth=3, nums_classes=100, trainable=False) CNN = DenseNet(height=32, width=32, depth=3, nums_classes=100, trainable=False) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) saver = tf.train.Saver(tf.global_variables(), max_to_keep=50) # saver.restore(sess, tf.train.latest_checkpoint('./ckpt_resnet_rp/')) # saver.restore(sess, './ckpt_resnet_rp/MyModel-100000') # saver.restore(sess, tf.train.latest_checkpoint('./ckpt_densenet_rp/')) saver.restore(sess, './ckpt_densenet_rp/MyModel-36000') total_loss = [] total_accuracy = [] for i in range(data_size // batch_size): batch_x = cifar.X[i * batch_size:(i + 1) * batch_size] batch_x = batch_x.astype(np.float32) for j in range(batch_x.shape[0]): batch_x[j] = random_crop(batch_x[j], padding=4, is_flip=False, is_crop=False) # batch_x[i] = random_erase(batch_x[i], prob=0.5, min=0.1, max=0.4) batch_y = cifar.Y[i * batch_size:(i + 1) * batch_size] batch_y = one_hot_encode(batch_y, 100) batch_loss, batch_accuracy = sess.run([CNN.loss, CNN.accuracy], feed_dict={ CNN.X: batch_x, CNN.Y: batch_y }) print("test_step: ", i + 1, " test_loss: ", batch_loss, " test_accuracy: ", batch_accuracy) total_loss.append(batch_loss) total_accuracy.append(batch_accuracy) print("Finally, final_test_loss:", np.mean(total_loss), " final_test_accuracy:", np.mean(total_accuracy))
def get_input_flow(self): ds_train = CellImageDataManagerTrain() # ds_train = MapDataComponent(ds_train, random_affine) # TODO : no improvement? ds_train = MapDataComponent(ds_train, random_color) # ds_train = MapDataComponent(ds_train, random_scaling) ds_train = MapDataComponent( ds_train, mask_size_normalize) # Resize by instance size - normalization ds_train = MapDataComponent( ds_train, lambda x: resize_shortedge_if_small(x, self.img_size)) # ds_train = MapDataComponent(ds_train, lambda x: pad_if_small(x, self.img_size)) # preseve cell's size ds_train = MapDataComponent( ds_train, lambda x: random_crop( x, self.img_size, self.img_size, padding=self.pad_size)) # ds_train = MapDataComponent(ds_train, random_add_thick_area) # TODO : worth? ds_train = MapDataComponent(ds_train, random_flip_lr) ds_train = MapDataComponent(ds_train, random_flip_ud) # ds_train = MapDataComponent(ds_train, data_to_elastic_transform_wrapper) if self.unet_weight: ds_train = MapDataComponent(ds_train, erosion_mask) ds_train = MapData( ds_train, lambda x: data_to_segment_input(x, not self.is_color, self.unet_weight)) ds_train = PrefetchData(ds_train, 256, 24) ds_train = BatchData(ds_train, self.batchsize) ds_train = MapDataComponent(ds_train, data_to_normalize1) ds_valid = CellImageDataManagerValid() ds_valid = MapDataComponent( ds_valid, lambda x: resize_shortedge_if_small(x, self.img_size)) ds_valid = MapDataComponent( ds_valid, lambda x: random_crop( x, self.img_size, self.img_size, padding=self.pad_size)) if self.unet_weight: ds_valid = MapDataComponent(ds_valid, erosion_mask) ds_valid = MapData( ds_valid, lambda x: data_to_segment_input(x, not self.is_color, self.unet_weight)) ds_valid = PrefetchData(ds_valid, 32, 8) ds_valid = BatchData(ds_valid, self.batchsize, remainder=True) ds_valid = MapDataComponent(ds_valid, data_to_normalize1) ds_valid2 = CellImageDataManagerValid() ds_valid2 = MapDataComponent( ds_valid2, lambda x: resize_shortedge_if_small(x, self.img_size)) ds_valid2 = MapDataComponent( ds_valid2, lambda x: center_crop_if_tcga(x, self.img_size, self.img_size)) # ds_valid2 = MapDataComponent(ds_valid2, lambda x: resize_shortedge(x, self.img_size)) ds_valid2 = MapData( ds_valid2, lambda x: data_to_segment_input(x, not self.is_color)) ds_valid2 = MapDataComponent(ds_valid2, data_to_normalize1) ds_test = CellImageDataManagerTest() ds_test = MapDataComponent( ds_test, lambda x: resize_shortedge_if_small(x, self.img_size)) # ds_test = MapDataComponent(ds_test, lambda x: resize_shortedge(x, self.img_size)) ds_test = MapData(ds_test, lambda x: data_to_image(x, not self.is_color)) ds_test = MapDataComponent(ds_test, data_to_normalize1) return ds_train, ds_valid, ds_valid2, ds_test