H = Convolution2D(1, 1, 1, border_mode='same', init='glorot_uniform')(H)
g_V = Activation('sigmoid')(H)
generator = Model(g_input, g_V)
generator.compile(loss='binary_crossentropy',
optimizer=Adam(lr=0.0002, beta_1=0.5))
generator.summary()
# Generator
adam = Adam(lr=0.0002, beta_1=0.5)
g = Sequential()
layer = g.add(Dense(7 * 7 * 112, input_dim=z_dim))
g.add(Reshape((7, 7, 112)))
g.add(BatchNormalization())
g.add(Activation(LeakyReLU(alpha=0.2)))
g.add(Conv2DTranspose(56, 5, strides=2, padding='same'))
g.add(BatchNormalization())
g.add(Activation(LeakyReLU(alpha=0.2)))
g.add(Conv2DTranspose(1, 5, strides=2, padding='same', activation='sigmoid'))
g.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
g.summary()
d = Sequential()
d.add(
Conv2D(56,
5,
strides=2,
padding='same',
input_shape=(28, 28, 1),
activation=LeakyReLU(alpha=0.2)))
d.add(Conv2D(112, 5, strides=2, padding='same'))
Convolution2D(200, (5, 5),
strides=(2, 2),
activation='elu',
padding='same'))
model.add(MaxPooling2D())
model.add(Convolution2D(300, (3, 3), activation='elu', padding='same'))
model.add(Convolution2D(400, (3, 3), activation='elu', padding='same'))
model.add(Dropout(0.1))
model.add(Convolution2D(400, (3, 3), activation='elu', padding='same'))
model.add(Convolution2D(300, (3, 3), activation='elu', padding='same'))
model.add(Dropout(0.1))
model.add(Convolution2D(
2, (1, 1))) # this is called upscore layer for some reason?
model.add(
Conv2DTranspose(2, (31, 31),
strides=(16, 16),
activation='softmax',
padding='same'))
#model = multi_gpu_model(model, gpus=2)
#opt = SGD(lr=1e-4, nesterov=True)
opt = Adam(lr=1e-6) # nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
#model.load_weights('./allsamples-keras-improvement-03-0.62.hdf')
# In[3]:
model = load_model('./fast-allsamples-keras-improvement-01-0.37.hdf')
train_samples = pd.read_table(
'/Z/personal-folders/interns/saket/github/pyvirchow/data/patch_df/train_df_with_mask.tsv'
def build_model(input_layer, start_neurons, DropoutRatio=0.5):
# 101 -> 50
conv1 = Conv2D(start_neurons * 1, (3, 3), activation=None,
padding="same")(input_layer)
conv1 = residual_block(conv1, start_neurons * 1)
conv1 = residual_block(conv1, start_neurons * 1, True)
pool1 = MaxPooling2D((2, 2))(conv1)
pool1 = Dropout(DropoutRatio / 2)(pool1)
# 50 -> 25
conv2 = Conv2D(start_neurons * 2, (3, 3), activation=None,
padding="same")(pool1)
conv2 = residual_block(conv2, start_neurons * 2)
conv2 = residual_block(conv2, start_neurons * 2, True)
pool2 = MaxPooling2D((2, 2))(conv2)
pool2 = Dropout(DropoutRatio)(pool2)
# 25 -> 12
conv3 = Conv2D(start_neurons * 4, (3, 3), activation=None,
padding="same")(pool2)
conv3 = residual_block(conv3, start_neurons * 4)
conv3 = residual_block(conv3, start_neurons * 4, True)
pool3 = MaxPooling2D((2, 2))(conv3)
pool3 = Dropout(DropoutRatio)(pool3)
# 12 -> 6
conv4 = Conv2D(start_neurons * 8, (3, 3), activation=None,
padding="same")(pool3)
conv4 = residual_block(conv4, start_neurons * 8)
conv4 = residual_block(conv4, start_neurons * 8, True)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(DropoutRatio)(pool4)
# Middle
convm = Conv2D(start_neurons * 16, (3, 3), activation=None,
padding="same")(pool4)
convm = residual_block(convm, start_neurons * 16)
convm = residual_block(convm, start_neurons * 16, True)
# 6 -> 12
deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(convm)
uconv4 = concatenate([deconv4, conv4])
uconv4 = Dropout(DropoutRatio)(uconv4)
uconv4 = Conv2D(start_neurons * 8, (3, 3), activation=None,
padding="same")(uconv4)
uconv4 = residual_block(uconv4, start_neurons * 8)
uconv4 = residual_block(uconv4, start_neurons * 8, True)
# 12 -> 25
#deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv4)
deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3),
strides=(2, 2),
padding="valid")(uconv4)
uconv3 = concatenate([deconv3, conv3])
uconv3 = Dropout(DropoutRatio)(uconv3)
uconv3 = Conv2D(start_neurons * 4, (3, 3), activation=None,
padding="same")(uconv3)
uconv3 = residual_block(uconv3, start_neurons * 4)
uconv3 = residual_block(uconv3, start_neurons * 4, True)
# 25 -> 50
deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3),
strides=(2, 2),
padding="same")(uconv3)
uconv2 = concatenate([deconv2, conv2])
uconv2 = Dropout(DropoutRatio)(uconv2)
uconv2 = Conv2D(start_neurons * 2, (3, 3), activation=None,
padding="same")(uconv2)
uconv2 = residual_block(uconv2, start_neurons * 2)
uconv2 = residual_block(uconv2, start_neurons * 2, True)
# 50 -> 101
#deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv2)
deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3),
strides=(2, 2),
padding="valid")(uconv2)
uconv1 = concatenate([deconv1, conv1])
uconv1 = Dropout(DropoutRatio)(uconv1)
uconv1 = Conv2D(start_neurons * 1, (3, 3), activation=None,
padding="same")(uconv1)
uconv1 = residual_block(uconv1, start_neurons * 1)
uconv1 = residual_block(uconv1, start_neurons * 1, True)
#uconv1 = Dropout(DropoutRatio/2)(uconv1)
#output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1)
output_layer_noActi = Conv2D(1, (1, 1), padding="same",
activation=None)(uconv1)
output_layer = Activation('sigmoid')(output_layer_noActi)
return output_layer
def Unet(imgheight, imgwidth):
inputs = Input((imgheight, imgwidth, 3))
s = Lambda(lambda x: x / 255)(inputs)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(s)
c1 = Dropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Dropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p2)
c3 = Dropout(0.2)(c3)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p3)
c4 = Dropout(0.2)(c4)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p4)
c5 = Dropout(0.3)(c5)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u6)
c6 = Dropout(0.2)(c6)
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u7)
c7 = Dropout(0.2)(c7)
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u8)
c8 = Dropout(0.1)(c8)
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u9)
c9 = Dropout(0.1)(c9)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[inputs], outputs=[outputs])
return model
def runner_lung_segmentation():
get_ipython().system('pip install imgaug') # for image augmentation')
# In[ ]:
get_ipython().system('pip install -U segmentation-models') # ONLY used for dice metric and IOU metric computation, models are made from scratch')
# In[1]:
import glob
import pandas as pd
import numpy as np
import nibabel as nib
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from zipfile import ZipFile
from shutil import copyfile, copyfileobj
import gzip
from IPython.display import clear_output
import cv2
import os
from pylab import rcParams
import PIL
from PIL import Image
import scipy
from google.colab import files
from sklearn.model_selection import train_test_split
from google.colab import drive
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans, MeanShift
import imgaug as ia
import imgaug.augmenters as iaa
print("Version: ", tf.__version__)
print("Eager mode: ", tf.executing_eagerly())
print("GPU is", "available" if tf.config.experimental.list_physical_devices("GPU") else "NOT AVAILABLE")
# In[ ]:
import sys
import random
import warnings
import math
import seaborn as sns; sns.set()
from keras.callbacks import Callback
from keras.losses import binary_crossentropy
from tqdm import tqdm_notebook, tnrange
from itertools import chain
from skimage.io import imread, imshow, concatenate_images
from skimage.transform import resize
from skimage.morphology import label
from keras.models import Model, load_model
from keras.layers import Input, BatchNormalization, Activation, Dense, Dropout
from keras.layers.core import Lambda, RepeatVector, Reshape
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D, GlobalMaxPool2D
from keras.layers.merge import concatenate, add
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras import backend as K
import joblib
import gc
import segmentation_models as sm
# * Setting up environment to connect kaggle and colab
# In[ ]:
os.environ['KAGGLE_USERNAME'] = "******" # username from the json file
os.environ['KAGGLE_KEY'] = "396fa22ac6546fab343d3b7f3a8e547b" # key from the json file
# * kaggle dataset api
# In[3]:
get_ipython().system('kaggle datasets download -d andrewmvd/covid19-ct-scans')
# In[4]:
get_ipython().system('kaggle datasets download -d deadskull7/unetpp')
# In[ ]:
#copyfile("/content/drive/My Drive/covid19-ct-scans.zip","/content/covid19-ct-scans.zip")
with ZipFile('unetpp.zip', 'r') as zipObj:
# Extract all the contents of zip file in current directory
zipObj.extractall('unetpp')
# In[6]:
os.listdir()
# * Extracting zip file here
# In[ ]:
#copyfile("/content/drive/My Drive/covid19-ct-scans.zip","/content/covid19-ct-scans.zip")
with ZipFile('covid19-ct-scans.zip', 'r') as zipObj:
# Extract all the contents of zip file in current directory
zipObj.extractall('covid19-ct-scans')
# In[8]:
# Read and examine metadata
raw_data = pd.read_csv('/content/covid19-ct-scans/metadata.csv')
raw_data = raw_data.replace('../input/covid19-ct-scans/','/content/covid19-ct-scans/',regex=True)
raw_data.head(5)
# In[9]:
raw_data.shape
# * img_size is the preferred image size to which the image is to be resized
# In[ ]:
img_size = 512
# * Used (CLAHE) Contrast Limited Adaptive Histogram Equalization to enhance the contrast of the images since medical images suffer a lot from the contrast problems.
# * Here Clip limit and the grid size are the hyperparameters to tune, generally clip limit should be between 2 to 4 as higher clip limit won't clip most of the histogram and treat it as AHE.
# * Higher clip limit might not even prevent the image from overamplifying the noise (the core advantage the CLAHE gives over AHE)
# * demo is a boolean variable, if 1 then giving the ability to plot the before and image after enhancement along with their respective histogram plots.
# In[ ]:
def clahe_enhancer(test_img, demo):
test_img = test_img*255
test_img = np.uint8(test_img)
test_img_flattened = test_img.flatten()
clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
clahe_image = clahe.apply(test_img)
clahe_image_flattened = clahe_image.flatten()
if demo == 1:
fig = plt.figure()
rcParams['figure.figsize'] = 10,10
plt.subplot(2, 2, 1)
plt.imshow(test_img, cmap='bone')
plt.title("Original CT-Scan")
plt.subplot(2, 2, 2)
plt.hist(test_img_flattened)
plt.title("Histogram of Original CT-Scan")
plt.subplot(2, 2, 3)
plt.imshow(clahe_image, cmap='bone')
plt.title("CLAHE Enhanced CT-Scan")
plt.subplot(2, 2, 4)
plt.hist(clahe_image_flattened)
plt.title("Histogram of CLAHE Enhanced CT-Scan")
return(clahe_image)
# * The images possess a lot of black space containing nothing and parts in which we are not interested like diaphragm below lungs. They will take the valuable RAM and also the unnecessary convolutions (computing power)
# * A possible solution is cropping the images and taking out the Region Of Interest (ROI) as per problem statement and use case.
# * Other pros are that we can now have greater area of ROI in the same resolution.
# * For cropping, though we can slice the rows and columns using trial and error but that process would be limited to this dataset only.
# * If I input the same images with the lungs translated vertically a bit, the above technique would fail miserably and would rather crop the lungs too.
# * So, a more better approach is to draw contours over the image and then cropping out the rectangle with the biggest contour with the largest area.
# * The contour (largest closed boundary) with the largest area would be the contour covering both the lungs.
# * This will require thresholding as a prior step. After plotting the histogram of the images, the images were found near bimodal i.e. roughly two peaks. So, used the Otsu's method of binarization
# * In global thresholding, we can use an arbitrary chosen value as a threshold. In contrast, Otsu's method avoids having to choose a value and determines it automatically and returns it.
# * Another important point is while cropping a CT scan we have to make sure that its corresponding segmentation map is also cropped by same limits otherwise pixel level labeling will go wrong and the model might map a good area with an infectious area and vice versa which we don't want.
# * So, in order to preserve the 4 end points, I am using a list by the name "points" for the same.
# In[ ]:
def cropper(test_img, demo):
test_img = test_img*255
test_img = np.uint8(test_img)
# ret, thresh = cv2.threshold(test_img, 50, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
# ret, thresh = cv2.threshold(test_img, ret, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
contours,hierarchy = cv2.findContours(test_img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
areas = [cv2.contourArea(c) for c in contours]
x = np.argsort(areas)
max_index = x[x.size - 1]
cnt1=contours[max_index]
second_max_index = x[x.size - 2]
cnt2 = contours[second_max_index]
# max_index = np.argmax(areas)
# cnt=contours[max_index]
x,y,w,h = cv2.boundingRect(cnt1)
p,q,r,s = cv2.boundingRect(cnt2)
cropped1 = test_img[y:y+h, x:x+w]
cropped1 = cv2.resize(cropped1, dsize=(125,250), interpolation=cv2.INTER_AREA)
cropped2 = test_img[q:q+s, p:p+r]
cropped2 = cv2.resize(cropped2, dsize=(125,250), interpolation=cv2.INTER_AREA)
fused = np.concatenate((cropped1, cropped2), axis=1)
# super_cropped = test_img[y+7:y+h-20, x+25:x+w-25]
points_lung1 = []
points_lung2 = []
points_lung1.append(x); points_lung1.append(y); points_lung1.append(w); points_lung1.append(h)
points_lung2.append(p); points_lung2.append(q); points_lung2.append(r); points_lung2.append(s)
if demo == 1:
fig = plt.figure()
rcParams['figure.figsize'] = 35, 35
plt.subplot(1, 3, 1)
plt.imshow(test_img, cmap='bone')
plt.title("Original CT-Scan")
plt.subplot(1, 3, 2)
plt.imshow(thresh, cmap='bone')
plt.title("Binary Mask")
plt.subplot(1, 3, 3)
plt.imshow(fused, cmap='bone')
plt.title("Cropped CT scan after making bounding rectangle")
# plt.subplot(1, 4, 4)
# plt.imshow(super_cropped, cmap='bone')
# plt.title("Cropped further manually")
plt.show()
return(fused, points_lung1, points_lung2)
# * Dataset contain 20 files of .nii type, though each file contained multiple channels or silces each as a separate gray scale image.
# * Total slices are 3520. These have been sliced out by 20% in the front and by 20% in the last of each file since in general these didn't had any infection masks and some didn't had the lungs, removed as noise.
# * Also, images had pixel values from -998 to 1000+. Did min-max scaling.
# In[ ]:
def read_nii_demo(filepath, data):
'''
Reads .nii file and returns pixel array
'''
ct_scan = nib.load(filepath)
array = ct_scan.get_fdata()
array = np.rot90(np.array(array))
slices = array.shape[2]
array = array[:,:,round(slices*0.2):round(slices*0.8)]
array = np.reshape(np.rollaxis(array, 2),(array.shape[2],array.shape[0],array.shape[1], 1))
for img_no in range(0, array.shape[0]):
# array = Image.resize(array[...,img_no], (img_size,img_size))
img = cv2.resize(array[img_no], dsize=(img_size, img_size), interpolation=cv2.INTER_AREA)
xmax, xmin = img.max(), img.min()
img = (img - xmin)/(xmax - xmin)
data.append(img)
# In[ ]:
cts = []
lungs = []
# In[15]:
for i in range(0, 20):
read_nii_demo(raw_data.loc[i,'lung_mask'], lungs)
read_nii_demo(raw_data.loc[i,'ct_scan'], cts)
# In[16]:
print(len(cts), len(lungs))
# In[ ]:
new_cts = []
new_lungs = []
# In[ ]:
del lungs[1368:1372]
del cts[1368:1372]
del lungs[1924:1926]
del cts[1924:1926]
# In[ ]:
for img_no in range(len(lungs)):
lung_img = lungs[img_no]
lung_img[lung_img>0]=1
cropped_lung, points1, points2 = cropper(lung_img, demo = 0)
new_lungs.append(cropped_lung)
cts_img = cts[img_no]
cts_img = clahe_enhancer(cts_img, demo = 0)
# img, points1, points2 = cropper(img, demo = 0)
# all_points1.append((points1[0], points1[1], points1[2], points1[3]))
# all_points2.append((points2[0], points2[1], points2[2], points2[3]))
a,b,c,d = points1[0], points1[1], points1[2], points1[3]
e,f,g,h = points2[0], points2[1], points2[2], points2[3]
img1 = cts_img[b:b+d, a:a+c]
img1 = cv2.resize(img1, dsize=(125,250), interpolation=cv2.INTER_AREA)
img2 = cts_img[f:f+h, e:e+g]
img2 = cv2.resize(img2, dsize=(125,250), interpolation=cv2.INTER_AREA)
cropped_cts = np.concatenate((img1, img2), axis=1)
new_cts.append(cropped_cts)
# In[20]:
print(len(new_cts), len(new_lungs))
# In[33]:
x = 100
rcParams['figure.figsize'] = 10,10
plt.subplot(1, 2, 1)
plt.imshow(new_cts[x], cmap='bone')
plt.title("Final preprocessed (CLAHE Enhanced + Cropped) Image")
plt.subplot(1, 2, 2)
plt.imshow(new_lungs[x], cmap='bone')
plt.title("Final preprocessed corresponding Mask")
print(cts[x].shape, lungs[x].shape)
# * Also, figuring out that 498 slices were of complete black masks i.e. no infection. For right now, kept out as didn't want to bother the segmentation model with this.
# * A better approach would be to make a sub-task and use that sub-task's output as input in our main task. Train a separate binary classifier to classify the CT scan as complete black or not, then the not ones to be passed from the segmentation model trained in this notebook.
# In[ ]:
no_masks = []
for i in range(0, len(lungs)):
if np.unique(lungs[i]).size == 1:
no_masks.append(i)
print("Number of complete black masks :" , len(no_masks))
# * Following is the demo for the CLAHE enhanced images with histograms.
# * Notice how the seahorse shaped infection in the left lung can be distinguised clearly after enhancement.
# In[ ]:
test_file = []
read_nii_demo(raw_data.loc[0,'ct_scan'], test_file)
test_file = np.array(test_file)
rcParams['figure.figsize'] = 10, 10
clahe_image = clahe_enhancer(test_file[100], demo = 1)
# * A demo for the cropped images, notice how the unwanted part including the diaphragm got cut
# In[ ]:
# fig = plt.figure()
# rcParams['figure.figsize'] = 35, 35
# cropped_image, points1, points2 = cropper(test_file[120], demo = 1)
# #print(ret)
# print(points1, points2)
# In[ ]:
# test_mask = []
# read_nii_demo(raw_data.loc[0,'infection_mask'], test_mask)
# test_mask = np.array(test_mask)
# test_mask = np.uint8(test_mask*255)
# rcParams['figure.figsize'] = 10,10
# plt.imshow(test_mask[120][20:155, 4:217], cmap = 'bone')
# test_mask[120][20:155, 4:217].shape
# * Finally 1614 samples which will later be split into train and test
# * Also, since we copped the images and masks, all cannot be of same size, so again a dimension needs to be decided to which all could be resized to.
# * Used median of the all width and height but couldn't fit the RAM so reduced the size to 224, though images with larger resolution with more clear features will possibly give better results on the same model.
# In[ ]:
dim1=[]
dim2=[]
for i in range(0, len(cts)):
dim1.append(cts[i].shape[0])
dim2.append(cts[i].shape[1])
dim1 = np.array(dim1)
dim2 = np.array(dim2)
print("An idea about the new net dimension to which all must be resized to (some will increase and some decrease) --->", np.median(dim1),'x', np.median(dim2))
# In[ ]:
# 32*11 = 352
# In[ ]:
new_dim = 224
# In[ ]:
for i in range(0,len(new_cts)):
new_cts[i] = cv2.resize(new_cts[i], dsize=(new_dim, new_dim), interpolation=cv2.INTER_LINEAR)
# cts[i] = cts[i]/255
new_lungs[i] = cv2.resize(new_lungs[i], dsize=(new_dim, new_dim), interpolation=cv2.INTER_LINEAR)
# infections[i] = infections[i]/255
# In[ ]:
# for i in range(0, len(cts)):
# cts[i] = cv2.cvtColor(cts[i], cv2.COLOR_GRAY2RGB)
# for i in range(0, len(infections)):
# infections[i] = cv2.cvtColor(infections[i], cv2.COLOR_GRAY2RGB)
# In[ ]:
new_cts = np.array(new_cts)
new_lungs = np.array(new_lungs)
# * Saving the numpy arrays to later reuse the same preprocessing for other models rather than doing it again and again.
# In[ ]:
# cts = cts.reshape( len(cts), new_dim, new_dim)
# infections = infections.reshape( len(infections), new_dim, new_dim)
# In[ ]:
new_cts = np.uint8(new_cts)
new_lungs = np.uint8(new_lungs)
# In[ ]:
# In[ ]:
# In[ ]:
# No Augmentation added this time
# * Data augmentation pipeline
# In[ ]:
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
rotate=(-40, 40), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
))
], random_order=True)
# In[ ]:
no_of_aug_imgs = 50
random_indices = np.random.randint(0, new_cts.shape[0], size=no_of_aug_imgs)
sample_new_cts = new_cts[random_indices]
sample_new_lungs = new_lungs[random_indices]
# In[ ]:
# sample_cts = np.floor(sample_cts)
# sample_inf = np.floor(sample_inf)
# sample_cts = np.uint64(sample_cts)
# sample_inf = np.uint64(sample_inf)
# In[ ]:
new_cts_aug, new_lungs_aug = seq(images=sample_new_cts,
segmentation_maps=sample_new_lungs)
# In[44]:
rcParams['figure.figsize'] = 60,60
rand = np.random.randint(0, no_of_aug_imgs, size=8)
cells1 = new_cts_aug[rand]
grid_image1 = np.hstack(cells1)
plt.imshow(grid_image1, cmap = 'bone')
# In[46]:
cells2 = new_lungs_aug[rand]
grid_image2 = np.hstack(cells2)
plt.imshow(grid_image2, cmap = 'bone')
# In[47]:
print(new_cts_aug.shape, new_lungs_aug.shape)
# In[ ]:
# cts = np.concatenate((cts, cts_aug), axis=0)
# infections = np.concatenate((infections, infections_aug), axis = 0)
# np.random.shuffle(cts)
# np.random.shuffle(infections)
# print(cts.shape, infections.shape)
# In[ ]:
new_cts_aug = new_cts_aug/255
new_lungs_aug = new_lungs_aug/255
new_cts_aug = new_cts_aug.reshape(len(new_cts_aug), new_dim, new_dim, 1)
new_lungs_aug = new_lungs_aug.reshape(len(new_lungs_aug), new_dim, new_dim, 1)
# In[ ]:
# In[ ]:
# * Normalizing images and masks from 0 to 1
# In[ ]:
joblib.dump(cts, 'cts_cropped_lungs_224.pkl')
# In[ ]:
files.download('cts_cropped_lungs_224.pkl')
# In[ ]:
joblib.dump(infections, 'infections_cropped_lungs_224.pkl')
# In[ ]:
files.download('infections_cropped_lungs_224.pkl')
# In[ ]:
# temp = joblib.load('infections_cropped_lungs_224.pkl')
# In[ ]:
new_cts = new_cts/255
new_lungs = new_lungs/255
# In[ ]:
new_cts = new_cts.reshape(len(new_cts), new_dim, new_dim, 1)
new_lungs = new_lungs.reshape(len(new_lungs), new_dim, new_dim, 1)
# In[ ]:
# cts_new = []
# # lungs_infections_new = []
# infections_new = []
# In[ ]:
# for i in range(0, 2112):
# cts_new.append(np.array(cts[i]))
# # lungs_infections_new.append(np.array(lungs_infections[i]))
# infections_new.append(np.array(infections[i]))
# In[ ]:
# cts_new = np.array(cts_new)
# # lungs_infections_new = np.array(lungs_infections_new)
# infections_new = np.array(infections_new)
# * Just overlaying infection masks over the corresponding CT scans
# In[ ]:
def plot_sample(array_list, color_map = 'nipy_spectral'):
'''
Plots and a slice with all available annotations
'''
fig = plt.figure(figsize=(10,30))
plt.subplot(1,2,1)
plt.imshow(array_list[0].reshape(new_dim, new_dim), cmap='bone')
plt.title('Original Image')
# plt.subplot(1,2,2)
# plt.imshow(array_list[0], cmap='bone')
# plt.imshow(array_list[1], alpha=0.5, cmap=color_map)
# plt.title('Lung Mask')
plt.subplot(1,2,2)
plt.imshow(array_list[0].reshape(new_dim, new_dim), cmap='bone')
plt.imshow(array_list[1].reshape(new_dim, new_dim), alpha=0.5, cmap=color_map)
plt.title('Infection Mask')
# plt.subplot(1,2,2)
# plt.imshow(array_list[0].reshape(img_size,img_size), cmap='bone')
# plt.imshow(array_list[1].reshape(img_size, img_size), alpha=0.5, cmap=color_map)
# plt.title('Lung and Infection Mask')
# plt.subplot(1,4,4)
# plt.imshow(array_list[0], cmap='bone')
# plt.imshow(array_list[3], alpha=0.5, cmap=color_map)
# plt.title('Lung and Infection Mask')
plt.show()
# In[ ]:
for index in [100,110,120,130,140,150]:
plot_sample([new_cts[index], new_lungs[index]])
# In[ ]:
x_train, x_valid, y_train, y_valid = train_test_split(new_cts, new_lungs, test_size=0.3, random_state=42)
# In[30]:
print(x_train.shape, x_valid.shape)
# In[ ]:
# x_train = np.concatenate((x_train, new_cts_aug), axis=0)
# y_train = np.concatenate((y_train, new_lungs_aug), axis = 0)
# print(x_train.shape, x_valid.shape)
# In[ ]:
# * Loss functions and metrics
# In[ ]:
def dice_coeff(y_true, y_pred):
smooth = 1.
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
return score
def dice_loss(y_true, y_pred):
loss = 1 - dice_coeff(y_true, y_pred)
return loss
def bce_dice_loss(y_true, y_pred):
loss = 0.5*binary_crossentropy(y_true, y_pred) + 0.5*dice_loss(y_true, y_pred)
return loss
def tversky_loss(y_true, y_pred):
alpha = 0.5
beta = 0.5
ones = K.ones(K.shape(y_true))
p0 = y_pred # proba that voxels are class i
p1 = ones-y_pred # proba that voxels are not class i
g0 = y_true
g1 = ones-y_true
num = K.sum(p0*g0, (0,1,2))
den = num + alpha*K.sum(p0*g1,(0,1,2)) + beta*K.sum(p1*g0,(0,1,2))
T = K.sum(num/den) # when summing over classes, T has dynamic range [0 Ncl]
Ncl = K.cast(K.shape(y_true)[-1], 'float32')
return Ncl-T
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
logit_y_pred = K.log(y_pred / (1. - y_pred))
loss = weight * (logit_y_pred * (1. - y_true) +
K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.))
return K.sum(loss) / K.sum(weight)
def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
w, m1, m2 = weight, y_true, y_pred
intersection = (m1 * m2)
score = (2. * K.sum(w * intersection) + smooth) / (K.sum(w * m1) + K.sum(w * m2) + smooth)
loss = 1. - K.sum(score)
return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
y_pred = K.cast(y_pred, 'float32')
# if we want to get same size of output, kernel size must be odd
averaged_mask = K.pool2d(
y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
weight = K.ones_like(averaged_mask)
w0 = K.sum(weight)
weight = 5. * K.exp(-5. * K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
weight *= (w0 / w1)
loss = 0.5*weighted_bce_loss(y_true, y_pred, weight) + 0.5*dice_loss(y_true, y_pred)
return loss
# * All the hyperparameters are put in place after repeating trial and error for a fixed number of epochs.
# In[33]:
inputs = Input((new_dim, new_dim, 1))
# s = Lambda(lambda x: x / 255) (inputs)
# def mish(inputs):
# return inputs * tf.math.tanh(tf.math.softplus(inputs))
c1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (inputs)
c1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c1)
c1 = BatchNormalization()(c1)
p1 = MaxPooling2D((2, 2)) (c1)
p1 = Dropout(0.25)(p1)
c2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (p1)
c2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c2)
c2 = BatchNormalization()(c2)
p2 = MaxPooling2D((2, 2)) (c2)
p2 = Dropout(0.25)(p2)
c3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (p2)
c3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c3)
c3 = BatchNormalization()(c3)
p3 = MaxPooling2D((2, 2)) (c3)
p3 = Dropout(0.25)(p3)
c4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (p3)
c4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c4)
c4 = BatchNormalization()(c4)
p4 = MaxPooling2D(pool_size=(2, 2)) (c4)
p4 = Dropout(0.25)(p4)
c5 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (p4)
c5 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c5)
u6 = Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
u6 = BatchNormalization()(u6)
c6 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (u6)
c6 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c6)
u7 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same') (c6)
u7 = concatenate([u7, c3])
u7 = BatchNormalization()(u7)
c7 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (u7)
c7 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c7)
u8 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c7)
u8 = concatenate([u8, c2])
u8 = BatchNormalization()(u8)
c8 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (u8)
c8 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c8)
u9 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c8)
u9 = concatenate([u9, c1], axis=3)
u9 = BatchNormalization()(u9)
c9 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (u9)
c9 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal") (c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid') (c9)
model = Model(inputs=[inputs], outputs=[outputs])
model.summary()
# * Some callbacks (model checkpointing with least validation loss, highest validation dice coefficient, learning rate reduction after some patience number of epochs)
# * Also experimented with exponential decaying learning rate but found ReduceLROnPlateau a bit effective in this case.
# In[ ]:
# import math
# def step_decay(epoch):
# initial_lrate = 0.0008
# drop = 0.8
# epochs_drop = 10
# lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop))
# print('New learning rate', lrate)
# return lrate
# lrate = LearningRateScheduler(step_decay)
# In[ ]:
# In[ ]:
class CosineAnnealingScheduler(Callback):
"""Cosine annealing scheduler.
"""
def __init__(self, T_max, eta_max, eta_min=0, verbose=1):
super(CosineAnnealingScheduler, self).__init__()
self.T_max = T_max
self.eta_max = eta_max
self.eta_min = eta_min
self.verbose = verbose
def on_epoch_begin(self, epoch, logs=None):
if not hasattr(self.model.optimizer, 'lr'):
raise ValueError('Optimizer must have a "lr" attribute.')
lr = self.eta_min + (self.eta_max - self.eta_min) * (1 + math.cos(math.pi * epoch / self.T_max)) / 2
K.set_value(self.model.optimizer.lr, lr)
print('\nEpoch %05d: CosineAnnealingScheduler setting learning ''rate to %s.' % (epoch + 1, lr))
def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
logs['lr'] = K.get_value(self.model.optimizer.lr)
# In[ ]:
cosine_annealer = CosineAnnealingScheduler(T_max=7, eta_max=0.0005, eta_min=0.0001)
# In[ ]:
rcParams['figure.figsize'] = 5,5
T_max=7
eta_max=0.002
eta_min = 0.0001
lr=[]
for epoch in range(100):
lr.append(eta_min + (eta_max - eta_min) * (1 + math.cos(math.pi * epoch / T_max)) / 2)
lr = np.array(lr)
plt.plot(lr)
# In[ ]:
batch_size = 32
epochs = 80
#lr_reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.7, patience=7, verbose=1)
filepath_dice_coeff="unet_covid_weights_dice_coeff.hdf5"
filepath_loss = "unet_covid_weights_val_loss.hdf5"
checkpoint_dice = ModelCheckpoint(filepath_dice_coeff, monitor='val_dice_coeff', verbose=1, save_best_only=True, mode='max')
checkpoint_loss = ModelCheckpoint(filepath_loss, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
# In[ ]:
model.compile(optimizer=Adam(lr = 0.0005), loss=bce_dice_loss, metrics=[dice_coeff])
# In[48]:
results = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
validation_data=(x_valid, y_valid),
callbacks = [checkpoint_dice, checkpoint_loss])
# In[ ]:
gc.collect()
# In[ ]:
model.load_weights(filepath_dice_coeff)
# * Model saved in json format and its weight in .hdf5 format at local
# In[ ]:
model.save_weights('unet_lung_segmentation_0.9813.h5')
# In[ ]:
files.download('unet_lung_segmentation_0.9813.h5')
# In[ ]:
model_json = model.to_json()
with open("unet_lung_segmentation_0.9813.json","w") as json_file:
json_file.write(model_json)
files.download("unet_lung_segmentation_0.9813.json")
# * Saved model results --> validation loss: 0.1504 and validation dice coefficient: 0.8780
# * Though highest validation dice coefficient was 0.8789 but didn't saved it
# because of its high val loss of 0.1547
# In[62]:
score = model.evaluate(x_valid, y_valid, batch_size=32)
print("test loss, test dice coefficient:", score)
# * I have specifically saved the least loss model rather than highest dice coefficient because, loss is generally a better predictor for overfitting, as you can see the validation dice coefficient still increased in the 79 th epoch when there was overfitting but loss was last saved on 51 st epoch, thus refrained more from overfitting.
# In[ ]:
plt.rcParams["axes.grid"] = True
rcParams['figure.figsize'] = 7, 5
plt.xlim(0, 40)
plt.plot(results.history['dice_coeff'])
plt.plot(results.history['val_dice_coeff'])
plt.title('Dice Coefficient')
plt.ylabel('Dice coefficient')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.ylim(0, 2)
plt.xlim(0, 40)
plt.plot(results.history['loss'])
plt.plot(results.history['val_loss'])
plt.title('Dice Loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# * Some actual vs predicted samples to check the performance of the model visually. Notice, how overlapping they are with each other, thus justifying the high dice coefficient.
# In[ ]:
plt.rcParams["axes.grid"] = False
# In[ ]:
def compare_actual_and_predicted(image_no):
temp = model.predict(new_cts[image_no].reshape(1,new_dim, new_dim, 1))
fig = plt.figure(figsize=(15,15))
plt.subplot(1,3,1)
plt.imshow(new_cts[image_no].reshape(new_dim, new_dim), cmap='bone')
plt.title('Original Image (CT)')
plt.subplot(1,3,2)
plt.imshow(new_lungs[image_no].reshape(new_dim,new_dim), cmap='bone')
plt.title('Actual mask')
plt.subplot(1,3,3)
plt.imshow(temp.reshape(new_dim,new_dim), cmap='bone')
plt.title('Predicted mask')
plt.show()
# plt.imshow(temp.reshape(img_size, img_size), cmap = 'bone')
# plt.imshow(infections_scaled[120].reshape(img_size, img_size), cmap ='summer')
# In[ ]:
for i in [440,269,555, 355, 380, 90]:
compare_actual_and_predicted(i)
# In[ ]:
gc.collect()
# In[ ]:
# # Some Significant Post-Processing
# In[54]:
drive.mount('/content/drive')
# In[ ]:
# model.load_weights(filepath_dice_coeff)
model.load_weights('/content/drive/My Drive/cts and infections/unet_lung_segmentation_0.9813.h5')
# In[ ]:
the_range = np.arange(0.10,0.80, 0.05)
# In[64]:
dices=[]
ious=[]
for t in the_range:
iou = sm.metrics.IOUScore(threshold=t)
dice = sm.metrics.FScore(threshold=t)
model.compile(optimizer=Adam(lr = 0.0005), loss=bce_dice_loss, metrics=[dice, iou])
score = model.evaluate(x_valid, y_valid, batch_size=32)
dices.append(score[1])
ious.append(score[2])
# In[65]:
print('DICES:',dices)
print("IOUS:",ious)
print("Best Threshold:", the_range[np.argmax(dices)])
print("Best dice score:", dices[np.argmax(dices)])
print("Best iou score:", ious[np.argmax(ious)])
# In[68]:
print("Best Threshold:", the_range[np.argmax(dices)])
fig = plt.figure(figsize=(15,5))
plt.subplot(1, 2, 1)
plt.plot(the_range, dices)
plt.title("Validation Dice coefficient vs Threshold")
plt.subplot(1, 2, 2)
plt.plot(the_range, ious)
plt.title("Validation IOU vs Threshold")
plt.show()
# In[ ]:
the_new_range = np.arange(0.43,0.53, 0.001)
# In[71]:
new_dices=[]
new_ious=[]
for t in the_new_range:
iou = sm.metrics.IOUScore(threshold=t)
dice = sm.metrics.FScore(threshold=t)
model.compile(optimizer=Adam(lr = 0.0005), loss=bce_dice_loss, metrics=[dice, iou])
score = model.evaluate(x_valid, y_valid, batch_size=32)
new_dices.append(score[1])
new_ious.append(score[2])
# In[ ]:
# print("We just checked for",len(the_new_range), "steps between 0.39 and 0.47")
# In[73]:
print('NEW DICES:',new_dices)
print("NEW IOUS:",new_ious)
print("New Best Threshold:", the_new_range[np.argmax(new_dices)])
print("Best new dice score:", new_dices[np.argmax(new_dices)])
print("Best new iou score:", new_ious[np.argmax(new_ious)])
# In[74]:
print("Best Threshold:", the_new_range[np.argmax(new_dices)])
fig = plt.figure(figsize=(20,7))
plt.subplot(1, 2, 1)
plt.plot(the_new_range, new_dices)
plt.title("Validation Dice coefficient vs Threshold")
plt.subplot(1, 2, 2)
plt.plot(the_new_range, new_ious)
plt.title("Validation IOU vs Threshold")
plt.show()
# In[ ]:
the_prec_rec_range = np.arange(0,1, 0.05)
# In[76]:
precisions=[]
recalls=[]
for t in the_prec_rec_range:
precision = sm.metrics.Precision(threshold=t)
recall = sm.metrics.Recall(threshold=t)
model.compile(optimizer=Adam(lr = 0.0005), loss=bce_dice_loss, metrics=[precision, recall])
score = model.evaluate(x_valid, y_valid, batch_size=32)
precisions.append(score[1])
recalls.append(score[2])
# In[77]:
print('PRECISIONS:',precisions)
print("RRECALLS:",recalls)
print("Best Threshold for Precision:", the_prec_rec_range[np.argmax(precisions)])
print("Best Threshold for Recall:", the_prec_rec_range[np.argmax(recalls)])
print("Best precision score:", precisions[np.argmax(precisions)])
print("Best recall score:", recalls[np.argmax(recalls)])
# In[78]:
rcParams['figure.figsize'] = 7,7
plt.rcParams["axes.grid"] = True
plt.title("Precision and Recalls vs Thresholds")
plt.xlabel('Threshold')
plt.ylabel('Precision and Recall Values')
plt.plot(the_prec_rec_range,precisions, color='red')
plt.plot(the_prec_rec_range, recalls, color='green')
plt.legend(['Precision', 'Recall'])
# In[ ]:
return
def decoder(input_img,
n_filters=16,
dropout=1.0,
batchnorm=True,
training=True):
# expansive path
u7 = Conv2DTranspose(n_filters * 32, (3, 3),
strides=(2, 2),
padding='same')(input_img)
u7 = Dropout(dropout)(u7, training=training)
c8 = conv2d_block(u7,
n_filters=n_filters * 32,
kernel_size=3,
batchnorm=batchnorm)
u8 = Conv2DTranspose(n_filters * 16, (3, 3),
strides=(2, 2),
padding='same')(c8)
u8 = Dropout(dropout)(u8, training=training)
c9 = conv2d_block(u8,
n_filters=n_filters * 16,
kernel_size=3,
batchnorm=batchnorm)
u9 = Conv2DTranspose(n_filters * 8, (3, 3), strides=(2, 2),
padding='same')(c9)
u9 = Dropout(dropout)(u9, training=training)
c10 = conv2d_block(u9,
n_filters=n_filters * 8,
kernel_size=3,
batchnorm=batchnorm)
u10 = Conv2DTranspose(n_filters * 4, (3, 3),
strides=(2, 2),
padding='same')(c10)
u10 = Dropout(dropout)(u10, training=training)
c11 = conv2d_block(u10,
n_filters=n_filters * 4,
kernel_size=3,
batchnorm=batchnorm)
u11 = Conv2DTranspose(n_filters * 2, (3, 3),
strides=(2, 2),
padding='same')(c11)
u11 = Dropout(dropout)(u11, training=training)
c12 = conv2d_block(u11,
n_filters=n_filters * 2,
kernel_size=3,
batchnorm=batchnorm)
u12 = Conv2DTranspose(n_filters * 1, (3, 3),
strides=(2, 2),
padding='same')(c12)
u12 = Dropout(dropout)(u12, training=training)
c13 = conv2d_block(u12,
n_filters=n_filters * 1,
kernel_size=3,
batchnorm=batchnorm)
prediction = Conv2D(1, (1, 1))(c13)
variances = Conv2D(1, (1, 1), activation="softplus")(c13)
outputs = concatenate([prediction, variances])
return outputs
def _get_unet(self, input_img, n_filters=16, dropout=0.1, batchnorm=True):
"""Function to define the UNET Model"""
# Contracting Path
c1 = self._conv2d_block(input_img,
n_filters * 1,
kernel_size=3,
batchnorm=batchnorm)
p1 = MaxPooling2D((2, 2))(c1)
p1 = Dropout(dropout)(p1)
c2 = self._conv2d_block(p1,
n_filters * 2,
kernel_size=3,
batchnorm=batchnorm)
p2 = MaxPooling2D((2, 2))(c2)
p2 = Dropout(dropout)(p2)
c3 = self._conv2d_block(p2,
n_filters * 4,
kernel_size=3,
batchnorm=batchnorm)
p3 = MaxPooling2D((2, 2))(c3)
p3 = Dropout(dropout)(p3)
c4 = self._conv2d_block(p3,
n_filters * 8,
kernel_size=3,
batchnorm=batchnorm)
p4 = MaxPooling2D((2, 2))(c4)
p4 = Dropout(dropout)(p4)
c5 = self._conv2d_block(p4,
n_filters=n_filters * 16,
kernel_size=3,
batchnorm=batchnorm)
# Expansive Path
u6 = Conv2DTranspose(n_filters * 8, (3, 3),
strides=(2, 2),
padding='same')(c5)
u6 = concatenate([u6, c4])
u6 = Dropout(dropout)(u6)
c6 = self._conv2d_block(u6,
n_filters * 8,
kernel_size=3,
batchnorm=batchnorm)
u7 = Conv2DTranspose(n_filters * 4, (3, 3),
strides=(2, 2),
padding='same')(c6)
u7 = concatenate([u7, c3])
u7 = Dropout(dropout)(u7)
c7 = self._conv2d_block(u7,
n_filters * 4,
kernel_size=3,
batchnorm=batchnorm)
u8 = Conv2DTranspose(n_filters * 2, (3, 3),
strides=(2, 2),
padding='same')(c7)
u8 = concatenate([u8, c2])
u8 = Dropout(dropout)(u8)
c8 = self._conv2d_block(u8,
n_filters * 2,
kernel_size=3,
batchnorm=batchnorm)
u9 = Conv2DTranspose(n_filters * 1, (3, 3),
strides=(2, 2),
padding='same')(c8)
u9 = concatenate([u9, c1])
u9 = Dropout(dropout)(u9)
c9 = self._conv2d_block(u9,
n_filters * 1,
kernel_size=3,
batchnorm=batchnorm)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[input_img], outputs=[outputs])
return model
def __init__(self, input_channel_count, output_channel_count,
first_layer_filter_count):
self.INPUT_IMAGE_SIZE = 256
self.CONCATENATE_AXIS = -1
self.CONV_FILTER_SIZE = 4
self.CONV_STRIDE = 2
self.CONV_PADDING = (1, 1)
self.DECONV_FILTER_SIZE = 2
self.DECONV_STRIDE = 2
# (256 x 256 x input_channel_count)
inputs = Input((self.INPUT_IMAGE_SIZE, self.INPUT_IMAGE_SIZE,
input_channel_count))
# エンコーダーの作成
# (128 x 128 x N)
enc1 = ZeroPadding2D(self.CONV_PADDING)(inputs)
enc1 = Conv2D(first_layer_filter_count,
self.CONV_FILTER_SIZE,
strides=self.CONV_STRIDE)(enc1)
# (64 x 64 x 2N)
filter_count = first_layer_filter_count * 2
enc2 = self._add_encoding_layer(filter_count, enc1)
# (32 x 32 x 4N)
filter_count = first_layer_filter_count * 4
enc3 = self._add_encoding_layer(filter_count, enc2)
# (16 x 16 x 8N)
filter_count = first_layer_filter_count * 8
enc4 = self._add_encoding_layer(filter_count, enc3)
# (8 x 8 x 8N)
enc5 = self._add_encoding_layer(filter_count, enc4)
# (4 x 4 x 8N)
enc6 = self._add_encoding_layer(filter_count, enc5)
# (2 x 2 x 8N)
enc7 = self._add_encoding_layer(filter_count, enc6)
# (1 x 1 x 8N)
enc8 = self._add_encoding_layer(filter_count, enc7)
# デコーダーの作成
# (2 x 2 x 8N)
dec1 = self._add_decoding_layer(filter_count, True, enc8)
dec1 = concatenate([dec1, enc7], axis=self.CONCATENATE_AXIS)
# (4 x 4 x 8N)
dec2 = self._add_decoding_layer(filter_count, True, dec1)
dec2 = concatenate([dec2, enc6], axis=self.CONCATENATE_AXIS)
# (8 x 8 x 8N)
dec3 = self._add_decoding_layer(filter_count, True, dec2)
dec3 = concatenate([dec3, enc5], axis=self.CONCATENATE_AXIS)
# (16 x 16 x 8N)
dec4 = self._add_decoding_layer(filter_count, False, dec3)
dec4 = concatenate([dec4, enc4], axis=self.CONCATENATE_AXIS)
# (32 x 32 x 4N)
filter_count = first_layer_filter_count * 4
dec5 = self._add_decoding_layer(filter_count, False, dec4)
dec5 = concatenate([dec5, enc3], axis=self.CONCATENATE_AXIS)
# (64 x 64 x 2N)
filter_count = first_layer_filter_count * 2
dec6 = self._add_decoding_layer(filter_count, False, dec5)
dec6 = concatenate([dec6, enc2], axis=self.CONCATENATE_AXIS)
# (128 x 128 x N)
filter_count = first_layer_filter_count
dec7 = self._add_decoding_layer(filter_count, False, dec6)
dec7 = concatenate([dec7, enc1], axis=self.CONCATENATE_AXIS)
# (256 x 256 x output_channel_count)
dec8 = Activation(activation='relu')(dec7)
dec8 = Conv2DTranspose(output_channel_count,
self.DECONV_FILTER_SIZE,
strides=self.DECONV_STRIDE)(dec8)
dec8 = Activation(activation='sigmoid')(dec8)
self.UNET = Model(input=inputs, output=dec8)
kernel_initializer='he_normal',
activation='sigmoid')(Conv4)
########### Layer 6 ############
Conv6 = Conv2D(filters=32,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
kernel_initializer='he_normal',
activation='sigmoid')(Conv5)
########### Layer 7 ############
Conv7 = add([
Conv2DTranspose(filters=16,
kernel_size=[2, 2],
strides=[2, 2],
padding='same',
kernel_initializer='he_normal',
activation='sigmoid')(Conv6), Conv3
])
########### Layer 8 ############
Conv8 = Conv2D(filters=8,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
kernel_initializer='he_normal',
activation='sigmoid')(Conv7)
########### Layer 9 ############
Conv9 = add([
Conv2DTranspose(filters=8,
def make_generator(dense=True, labels_size=10):
"""Creates a generator model that takes a 100-dimensional noise vector as a "seed", and outputs images
of size 28x28x1."""
model = Sequential()
# ------------------------------ Layer 1: Dense + LeakyReLu ---------------------------------------
if dense:
model.add(Dense(1024, input_dim=100 + labels_size))
model.add(LeakyReLU())
model.add(Dense(128 * 7 * 7))
else:
model.add(Dense(128 * 7 * 7, input_dim=100 + labels_size))
# ------------------------------ Layer 2: Dense + LeakyReLu ---------------------------------------
model.add(BatchNormalization())
model.add(LeakyReLU())
# - - - - - - - - - - - - - - - - - - - Reshape - - - - - - - - - - - - - - - -
if K.image_data_format() == 'channels_first':
# size: 128 x 7 x 7
model.add(Reshape((128, 7, 7), input_shape=(128 * 7 * 7, )))
bn_axis = 1 # first
else:
# size: 7 x 7 x 128
model.add(Reshape((7, 7, 128), input_shape=(128 * 7 * 7, )))
bn_axis = -1 # last
# ------------------------------ Layer 3: DeConv2D + LeakyReLu ---------------------------------------
model.add(
Conv2DTranspose(filters=128,
kernel_size=(5, 5),
strides=2,
padding='same'))
model.add(BatchNormalization(axis=bn_axis))
model.add(LeakyReLU())
# ------------------------------ Layer 4: Conv2D + LeakyReLu ---------------------------------------
model.add(Convolution2D(64, (5, 5), padding='same'))
model.add(BatchNormalization(axis=bn_axis))
model.add(LeakyReLU())
# ------------------------------ Layer 5: DeConv2D + LeakyReLu ---------------------------------------
model.add(Conv2DTranspose(64, (5, 5), strides=2, padding='same'))
model.add(BatchNormalization(axis=bn_axis))
model.add(LeakyReLU())
# ------------------------------ Layer 6: Conv2D + Tanh ---------------------------------------
# Because we normalized training inputs to lie in the range [-1, 1],
# the tanh function should be used for the output of the generator to ensure its output
# also lies in this range.
model.add(Convolution2D(1, (5, 5), padding='same', activation='tanh'))
# our idea:
# seed 100
# layer1: dense 1024
# layer2: dense 7*7*128
# reshape 7 x 7 x 128
# layer3: Deconv 14 x 14 x 128
# layer4: Conv 14 x 14 x 64
# layer5: Deconv 28 x 28 x 64
# layer6: Conv 28 x 28 x 1
return model
def get_gen_normal(noise_shape):
noise_shape = noise_shape
"""
Changing padding = 'same' in the first layer makes a lot fo difference!!!!
"""
#kernel_init = RandomNormal(mean=0.0, stddev=0.01)
kernel_init = 'glorot_uniform'
gen_input = Input(
shape=noise_shape) #if want to directly use with conv layer next
#gen_input = Input(shape = [noise_shape]) #if want to use with dense layer next
generator = Conv2DTranspose(filters=512,
kernel_size=(4, 4),
strides=(1, 1),
padding="valid",
data_format="channels_last",
kernel_initializer=kernel_init)(gen_input)
generator = BatchNormalization(momentum=0.5)(generator)
generator = LeakyReLU(0.2)(generator)
#generator = bilinear2x(generator,256,kernel_size=(4,4))
#generator = UpSampling2D(size=(2, 2))(generator)
#generator = SubPixelUpscaling(scale_factor=2)(generator)
#generator = Conv2D(filters = 256, kernel_size = (4,4), strides = (1,1), padding = "same", data_format = "channels_last", kernel_initializer = kernel_init)(generator)
generator = Conv2DTranspose(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
data_format="channels_last",
kernel_initializer=kernel_init)(generator)
generator = BatchNormalization(momentum=0.5)(generator)
generator = LeakyReLU(0.2)(generator)
#generator = bilinear2x(generator,128,kernel_size=(4,4))
#generator = UpSampling2D(size=(2, 2))(generator)
#generator = SubPixelUpscaling(scale_factor=2)(generator)
#generator = Conv2D(filters = 128, kernel_size = (4,4), strides = (1,1), padding = "same", data_format = "channels_last", kernel_initializer = kernel_init)(generator)
generator = Conv2DTranspose(filters=128,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
data_format="channels_last",
kernel_initializer=kernel_init)(generator)
generator = BatchNormalization(momentum=0.5)(generator)
generator = LeakyReLU(0.2)(generator)
#generator = bilinear2x(generator,64,kernel_size=(4,4))
#generator = UpSampling2D(size=(2, 2))(generator)
#generator = SubPixelUpscaling(scale_factor=2)(generator)
#generator = Conv2D(filters = 64, kernel_size = (4,4), strides = (1,1), padding = "same", data_format = "channels_last", kernel_initializer = kernel_init)(generator)
generator = Conv2DTranspose(filters=64,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
data_format="channels_last",
kernel_initializer=kernel_init)(generator)
generator = BatchNormalization(momentum=0.5)(generator)
generator = LeakyReLU(0.2)(generator)
generator = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
data_format="channels_last",
kernel_initializer=kernel_init)(generator)
generator = BatchNormalization(momentum=0.5)(generator)
generator = LeakyReLU(0.2)(generator)
#generator = bilinear2x(generator,3,kernel_size=(3,3))
#generator = UpSampling2D(size=(2, 2))(generator)
#generator = SubPixelUpscaling(scale_factor=2)(generator)
#generator = Conv2D(filters = 3, kernel_size = (4,4), strides = (1,1), padding = "same", data_format = "channels_last", kernel_initializer = kernel_init)(generator)
generator = Conv2DTranspose(filters=3,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
data_format="channels_last",
kernel_initializer=kernel_init)(generator)
generator = Activation('tanh')(generator)
gen_opt = Adam(lr=0.00015, beta_1=0.5)
generator_model = Model(input=gen_input, output=generator)
generator_model.compile(loss='binary_crossentropy',
optimizer=gen_opt,
metrics=['accuracy'])
generator_model.summary()
return generator_model
def network(input_train_tm_tens, input_train_tp_tens, output_train_t_tens,
dfat_train_t_tens, te_train_t_tens):
n_filters = 32
kernel_size = 2
c1 = Conv2D(12,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
padding='same',
kernel_initializer='glorot_uniform')(input_train_tm_tens)
c1 = BatchNormalization()(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(24,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(p1)
c2 = BatchNormalization()(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(48,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(p2)
c3 = BatchNormalization()(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(96,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(p3)
c4 = BatchNormalization()(c4)
p4 = MaxPooling2D((2, 2))(c4)
c5 = Conv2D(192,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(p4)
c5 = BatchNormalization()(c5)
p5 = MaxPooling2D((2, 2))(c5)
c6 = Conv2D(384,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(p5)
c6 = BatchNormalization()(c6)
c6 = UpSampling2D(size=(2, 2), data_format=None)(c6)
u7 = Conv2DTranspose(192,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(c6)
u7 = concatenate([u7, c5])
c7 = Conv2D(192,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(u7)
c7 = BatchNormalization()(c7)
c7 = UpSampling2D(size=(2, 2), data_format=None)(c7)
u8 = Conv2DTranspose(96,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(c7)
u8 = concatenate([u8, c4], axis=3)
c8 = Conv2D(96,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(u8)
c8 = BatchNormalization()(c8)
c8 = UpSampling2D(size=(2, 2), data_format=None)(c8)
u9 = Conv2DTranspose(48,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(c8)
u9 = concatenate([u9, c3], axis=3)
c9 = Conv2D(48,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(u9)
c9 = BatchNormalization()(c9)
c9 = UpSampling2D(size=(2, 2), data_format=None)(c9)
u10 = Conv2DTranspose(24,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(c9)
u10 = concatenate([u10, c2], axis=3)
c10 = Conv2D(24,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(u10)
c10 = BatchNormalization()(c10)
c10 = UpSampling2D(size=(2, 2), data_format=None)(c10)
u11 = Conv2DTranspose(12,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(c10)
u11 = concatenate([u11, c1], axis=3)
c11 = Conv2D(12,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(u11)
c11 = BatchNormalization()(c11)
c11 = UpSampling2D(size=(1, 1), data_format=None)(c11)
output_pred1 = Conv2D(3, (1, 1), activation='linear')(c11)
cc1 = Conv2D(12,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
padding='same',
kernel_initializer='glorot_uniform')(input_train_tp_tens)
cc1 = BatchNormalization()(cc1)
pp1 = MaxPooling2D((2, 2))(cc1)
cc2 = Conv2D(24,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(pp1)
cc2 = BatchNormalization()(cc2)
pp2 = MaxPooling2D((2, 2))(cc2)
cc3 = Conv2D(48,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(pp2)
cc3 = BatchNormalization()(cc3)
pp3 = MaxPooling2D((2, 2))(cc3)
cc4 = Conv2D(96,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(pp3)
cc4 = BatchNormalization()(cc4)
pp4 = MaxPooling2D((2, 2))(cc4)
cc5 = Conv2D(192,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(pp4)
cc5 = BatchNormalization()(cc5)
pp5 = MaxPooling2D((2, 2))(cc5)
cc6 = Conv2D(384,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(pp5)
cc6 = BatchNormalization()(cc6)
cc6 = UpSampling2D(size=(2, 2), data_format=None)(cc6)
uu7 = Conv2DTranspose(192,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(cc6)
uu7 = concatenate([uu7, cc5])
cc7 = Conv2D(192,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(uu7)
cc7 = BatchNormalization()(cc7)
cc7 = UpSampling2D(size=(2, 2), data_format=None)(cc7)
uu8 = Conv2DTranspose(96,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(cc7)
uu8 = concatenate([uu8, cc4])
cc8 = Conv2D(96,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(uu8)
cc8 = BatchNormalization()(cc8)
cc8 = UpSampling2D(size=(2, 2), data_format=None)(cc8)
uu9 = Conv2DTranspose(48,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(cc8)
uu9 = concatenate([uu9, cc3])
cc9 = Conv2D(48,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(uu9)
cc9 = BatchNormalization()(cc9)
cc9 = UpSampling2D(size=(2, 2), data_format=None)(cc9)
uu10 = Conv2DTranspose(24,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(cc9)
uu10 = concatenate([uu10, cc2])
cc10 = Conv2D(24,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(uu10)
cc10 = BatchNormalization()(cc10)
cc10 = UpSampling2D(size=(2, 2), data_format=None)(cc10)
uu11 = Conv2DTranspose(12,
kernel_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same')(cc10)
uu11 = concatenate([uu11, cc1], axis=3)
cc11 = Conv2D(12,
kernel_size=(kernel_size, kernel_size),
activation='sigmoid',
kernel_initializer='glorot_uniform',
padding='same')(uu11)
cc11 = BatchNormalization()(cc11)
cc11 = UpSampling2D(size=(1, 1), data_format=None)(cc11)
output_pred2 = Conv2D(3, (1, 1), activation='linear')(cc11)
output_pred = concatenate([output_pred1, output_pred2])
model = Model(inputs=[
input_train_tm_tens, input_train_tp_tens, output_train_t_tens,
dfat_train_t_tens, te_train_t_tens
],
outputs=[output_pred])
return model, output_pred
z = Lambda(sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_var])
encoder = Model(input_labels, [z_mean, z_log_var, z], name='encoder')
# encoder.summary()
# plot_model(encoder, to_file='qvg_encoder.png', show_shapes=True)
# build decoder model
filter = 256
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
x2 = Dense(sh[1] * sh[2] * sh[3])(latent_inputs)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x2 = Reshape((sh[1], sh[2], sh[3]))(x2)
for i in range(3):
x2 = Conv2DTranspose(filters=filter,
kernel_size=kernel_size,
strides=2,
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
filter //= 2 # Filter before division: 256, 128, 32
if (i == 1): filter //= 2 # 64 -> 32
outputs = Conv2D(filters=3,
kernel_size=kernel_size,
activation='tanh',
padding='same',
name='decoder_output',
use_bias=False)(x2)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
# decoder.summary()
def build_generator(latent_size):
# we will map a pair of (z, L), where z is a latent vector and L is a
# label drawn from P_c, to image space (..., 28, 28, 1)
# Coefficients of the rational approximating ReLU
alpha_initializer = [1.1915, 1.5957, 0.5, 0.0218]
beta_initializer = [2.383, 0.0, 1.0]
if UseRational:
cnn = Sequential()
cnn.add(Dense(3 * 3 * 384, input_dim=latent_size))
cnn.add(
RationalLayer(alpha_initializer, beta_initializer,
shared_axes=[1]))
cnn.add(Reshape((3, 3, 384)))
# upsample to (7, 7, ...)
cnn.add(
Conv2DTranspose(192,
5,
strides=1,
padding='valid',
kernel_initializer='glorot_normal'))
cnn.add(
RationalLayer(alpha_initializer,
beta_initializer,
shared_axes=[1, 2, 3]))
cnn.add(BatchNormalization())
# upsample to (14, 14, ...)
cnn.add(
Conv2DTranspose(96,
5,
strides=2,
padding='same',
kernel_initializer='glorot_normal'))
cnn.add(
RationalLayer(alpha_initializer,
beta_initializer,
shared_axes=[1, 2, 3]))
cnn.add(BatchNormalization())
# upsample to (28, 28, ...)
cnn.add(
Conv2DTranspose(1,
5,
strides=2,
padding='same',
activation='tanh',
kernel_initializer='glorot_normal'))
else:
cnn = Sequential()
cnn.add(Dense(3 * 3 * 384, input_dim=latent_size))
cnn.add(ReLU())
cnn.add(Reshape((3, 3, 384)))
# upsample to (7, 7, ...)
cnn.add(
Conv2DTranspose(192,
5,
strides=1,
padding='valid',
kernel_initializer='glorot_normal'))
cnn.add(ReLU())
cnn.add(BatchNormalization())
# upsample to (14, 14, ...)
cnn.add(
Conv2DTranspose(96,
5,
strides=2,
padding='same',
kernel_initializer='glorot_normal'))
cnn.add(ReLU())
cnn.add(BatchNormalization())
# upsample to (28, 28, ...)
cnn.add(
Conv2DTranspose(1,
5,
strides=2,
padding='same',
activation='tanh',
kernel_initializer='glorot_normal'))
# this is the z space commonly referred to in GAN papers
latent = Input(shape=(latent_size, ))
# this will be our label
image_class = Input(shape=(1, ), dtype='int32')
cls = Embedding(num_classes,
latent_size,
embeddings_initializer='glorot_normal')(image_class)
# hadamard product between z-space and a class conditional embedding
h = layers.multiply([latent, cls])
fake_image = cnn(h)
return Model([latent, image_class], fake_image)
def get_vgg_7conv(input_shape):
img_input = Input(input_shape)
vgg16_base = VGG16(input_tensor=img_input, include_top=False)
for l in vgg16_base.layers:
l.trainable = True
conv1 = vgg16_base.get_layer("block1_conv2").output
conv2 = vgg16_base.get_layer("block2_conv2").output
conv3 = vgg16_base.get_layer("block3_conv3").output
pool3 = vgg16_base.get_layer("block3_pool").output
conv4 = Conv2D(384, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block4_conv1")(pool3)
conv4 = Conv2D(384, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block4_conv2")(conv4)
pool4 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(conv4)
conv5 = Conv2D(512, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block5_conv1")(pool4)
conv5 = Conv2D(512, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block5_conv2")(conv5)
pool5 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(conv5)
conv6 = Conv2D(512, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block6_conv1")(pool5)
conv6 = Conv2D(512, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block6_conv2")(conv6)
pool6 = MaxPooling2D((2, 2), strides=(2, 2), name='block6_pool')(conv6)
conv7 = Conv2D(512, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block7_conv1")(pool6)
conv7 = Conv2D(512, (3, 3), activation="relu", padding='same', kernel_initializer="he_normal",
name="block7_conv2")(conv7)
up8 = concatenate([Conv2DTranspose(384, (3, 3), activation="relu", kernel_initializer="he_normal",
strides=(2, 2), padding='same')(conv7), conv6], axis=3)
conv8 = Conv2D(384, (3, 3), activation="relu", kernel_initializer="he_normal", padding='same')(up8)
up9 = concatenate([Conv2DTranspose(256, (3, 3), activation="relu", kernel_initializer="he_normal",
strides=(2, 2), padding='same')(conv8), conv5], axis=3)
conv9 = Conv2D(256, (3, 3), activation="relu", kernel_initializer="he_normal", padding='same')(up9)
up10 = concatenate([Conv2DTranspose(192, (3, 3), activation="relu", kernel_initializer="he_normal",
strides=(2, 2), padding='same')(conv9), conv4], axis=3)
conv10 = Conv2D(192, (3, 3), activation="relu", kernel_initializer="he_normal", padding='same')(up10)
up11 = concatenate([Conv2DTranspose(128, (3, 3), activation="relu", kernel_initializer="he_normal",
strides=(2, 2), padding='same')(conv10), conv3], axis=3)
conv11 = Conv2D(128, (3, 3), activation="relu", kernel_initializer="he_normal", padding='same')(up11)
up12 = concatenate([Conv2DTranspose(64, (3, 3), activation="relu", kernel_initializer="he_normal",
strides=(2, 2), padding='same')(conv11), conv2], axis=3)
conv12 = Conv2D(64, (3, 3), activation="relu", kernel_initializer="he_normal", padding='same')(up12)
up13 = concatenate([Conv2DTranspose(32, (3, 3), activation="relu", kernel_initializer="he_normal",
strides=(2, 2), padding='same')(conv12), conv1], axis=3)
conv13 = Conv2D(32, (3, 3), activation="relu", kernel_initializer="he_normal", padding='same')(up13)
conv13 = Conv2D(1, (1, 1))(conv13)
conv13 = Activation("sigmoid")(conv13)
model = Model(img_input, conv13)
return model
def UEfficientNet(input_shape=(None, None, 3), dropout_rate=0.1):
backbone = efn.EfficientNetB2(weights=None,
include_top=False,
input_shape=input_shape)
# backbone.load_weights("../input/efficientnet-keras-weights-b0b5/efficientnet-b2_imagenet_1000_notop.h5")
input = backbone.input
start_neurons = 8
i = 2
lr = []
for l in backbone.layers:
if l.name == 'block{}a_expand_activation'.format(i):
lr.append(l)
i += 1
conv4 = lr[-1].output
conv4 = LeakyReLU(alpha=0.1)(conv4)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(dropout_rate)(pool4)
# Middle
convm = Conv2D(start_neurons * 32, (3, 3), activation=None,
padding="same")(pool4)
convm = residual_block(convm, start_neurons * 32)
convm = residual_block(convm, start_neurons * 32)
convm = LeakyReLU(alpha=0.1)(convm)
deconv4 = Conv2DTranspose(start_neurons * 16, (3, 3),
strides=(2, 2),
padding="same")(convm)
deconv4_up1 = Conv2DTranspose(start_neurons * 16, (3, 3),
strides=(2, 2),
padding="same")(deconv4)
deconv4_up2 = Conv2DTranspose(start_neurons * 16, (3, 3),
strides=(2, 2),
padding="same")(deconv4_up1)
deconv4_up3 = Conv2DTranspose(start_neurons * 16, (3, 3),
strides=(2, 2),
padding="same")(deconv4_up2)
uconv4 = concatenate([deconv4, conv4])
uconv4 = Dropout(dropout_rate)(uconv4)
uconv4 = Conv2D(start_neurons * 16, (3, 3),
activation=None,
padding="same")(uconv4)
uconv4 = residual_block(uconv4, start_neurons * 16)
# uconv4 = residual_block(uconv4,start_neurons * 16)
uconv4 = LeakyReLU(alpha=0.1)(uconv4) #conv1_2
deconv3 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(uconv4)
deconv3_up1 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(deconv3)
deconv3_up2 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(deconv3_up1)
conv3 = lr[-2].output
uconv3 = concatenate([deconv3, deconv4_up1, conv3])
uconv3 = Dropout(dropout_rate)(uconv3)
uconv3 = Conv2D(start_neurons * 8, (3, 3), activation=None,
padding="same")(uconv3)
uconv3 = residual_block(uconv3, start_neurons * 8)
# uconv3 = residual_block(uconv3,start_neurons * 8)
uconv3 = LeakyReLU(alpha=0.1)(uconv3)
deconv2 = Conv2DTranspose(start_neurons * 4, (3, 3),
strides=(2, 2),
padding="same")(uconv3)
deconv2_up1 = Conv2DTranspose(start_neurons * 4, (3, 3),
strides=(2, 2),
padding="same")(deconv2)
conv2 = lr[-4].output
uconv2 = concatenate([deconv2, deconv3_up1, deconv4_up2, conv2])
uconv2 = Dropout(0.1)(uconv2)
uconv2 = Conv2D(start_neurons * 4, (3, 3), activation=None,
padding="same")(uconv2)
uconv2 = residual_block(uconv2, start_neurons * 4)
# uconv2 = residual_block(uconv2,start_neurons * 4)
uconv2 = LeakyReLU(alpha=0.1)(uconv2)
deconv1 = Conv2DTranspose(start_neurons * 2, (3, 3),
strides=(2, 2),
padding="same")(uconv2)
conv1 = lr[-5].output
uconv1 = concatenate(
[deconv1, deconv2_up1, deconv3_up2, deconv4_up3, conv1])
uconv1 = Dropout(0.1)(uconv1)
uconv1 = Conv2D(start_neurons * 2, (3, 3), activation=None,
padding="same")(uconv1)
uconv1 = residual_block(uconv1, start_neurons * 2)
# uconv1 = residual_block(uconv1,start_neurons * 2)
uconv1 = LeakyReLU(alpha=0.1)(uconv1)
uconv0 = Conv2DTranspose(start_neurons * 1, (3, 3),
strides=(2, 2),
padding="same")(uconv1)
uconv0 = Dropout(0.1)(uconv0)
uconv0 = Conv2D(start_neurons * 1, (3, 3), activation=None,
padding="same")(uconv0)
uconv0 = residual_block(uconv0, start_neurons * 1)
# uconv0 = residual_block(uconv0,start_neurons * 1)
uconv0 = LeakyReLU(alpha=0.1)(uconv0)
uconv0 = Dropout(dropout_rate / 2)(uconv0)
uconv0 = Conv2DTranspose(start_neurons * 1, (3, 3),
strides=(2, 2),
padding="same")(uconv0)
output_layer = Conv2D(4, (1, 1), padding="same",
activation="sigmoid")(uconv0)
model = Model(input, output_layer)
model.name = 'u-xception'
return model
def get_unet(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS, N_Cls):
# Build U-Net model
inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
s = Lambda(lambda x: x / 255)(inputs) #0〜1の範囲に正規化
# Contracting 1
c1 = Conv2D(8, (3, 3), activation='relu', padding='same')(s)
c1 = Conv2D(8, (3, 3), activation='relu', padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
# Contracting 2
c2 = Conv2D(16, (3, 3), activation='relu', padding='same')(p1)
c2 = Conv2D(16, (3, 3), activation='relu', padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
# Contracting 3
c3 = Conv2D(32, (3, 3), activation='relu', padding='same')(p2)
c3 = Conv2D(32, (3, 3), activation='relu', padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
# Contracting 4
c4 = Conv2D(64, (3, 3), activation='relu', padding='same')(p3)
c4 = Conv2D(64, (3, 3), activation='relu', padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
# Lowest resolution
c5 = Conv2D(128, (3, 3), activation='relu', padding='same')(p4)
c5 = Conv2D(128, (3, 3), activation='relu', padding='same')(c5)
# Up-sampling 1
u6 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(64, (3, 3), activation='relu', padding='same')(u6)
c6 = Conv2D(64, (3, 3), activation='relu', padding='same')(c6)
# Up-sampling 2
u7 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(32, (3, 3), activation='relu', padding='same')(u7)
c7 = Conv2D(32, (3, 3), activation='relu', padding='same')(c7)
# Up-sampling 3
u8 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(16, (3, 3), activation='relu', padding='same')(u8)
c8 = Conv2D(16, (3, 3), activation='relu', padding='same')(c8)
# Up-sampling 4
u9 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(8, (3, 3), activation='relu', padding='same')(u9)
c9 = Conv2D(8, (3, 3), activation='relu', padding='same')(c9)
outputs = Conv2D(N_Cls, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[mean_iou])
model.summary()
return model
def Unet_with_slice(input_img, n_filters=16, dropout=0.3, batch_norm=True):
c1 = Conv2D(16, kernel_size=(1, 6), strides=(1, 1), padding='valid')(input_img)
if batch_norm:
c1 = BatchNormalization()(c1)
# print(c1.shape)
c1 = Activation('relu')(c1)
c1 = Conv2D(n_filters, kernel_size=(3, 3), strides=(1, 1), padding='same')(c1)
if batch_norm:
c1 = BatchNormalization()(c1)
c1 = Activation('relu')(c1)
p1 = MaxPooling2D(pool_size=(2, 2), strides=2)(c1)
p1 = Dropout(dropout)(p1)
# print(p1.shape)
c2 = conv_block(p1, n_filters * 2, 3, batch_norm)
p2 = MaxPooling2D(pool_size=(3, 3), strides=3)(c2)
p2 = Dropout(dropout)(p2)
# print(p2.shape)
c3 = conv_block(p2, n_filters * 4, 3, batch_norm)
# print(c3.shape)
p3 = MaxPooling2D(pool_size=(2, 1), strides=(2, 1))(c3)
p3 = Dropout(dropout)(p3)
# print(p3.shape)
c4 = conv_block(p3, n_filters * 8, 3, batch_norm)
p4 = MaxPooling2D(pool_size=(4, 4), strides=(4, 5))(c4)
p4 = Dropout(dropout)(p4)
c5 = conv_block(p4, n_filters * 16, 3, batch_norm)
u6 = Conv2DTranspose(n_filters * 8, kernel_size=(4, 4), strides=(4, 5), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = conv_block(u6, n_filters * 8, 3, batch_norm)
c6 = Dropout(dropout)(c6)
u7 = Conv2DTranspose(n_filters * 4, kernel_size=(3, 3), strides=(2, 1), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = conv_block(u7, n_filters * 4, 3, batch_norm)
c7 = Dropout(dropout)(c7)
u8 = Conv2DTranspose(n_filters * 2, kernel_size=(3, 3), strides=(3, 3), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = conv_block(u8, n_filters * 2, 3, batch_norm)
c8 = Dropout(dropout)(c8)
u9 = Conv2DTranspose(n_filters, kernel_size=(3, 3), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1])
c9 = conv_block(u9, n_filters, 3, batch_norm)
c9 = Dropout(dropout)(c9)
c10 = Conv2DTranspose(n_filters, kernel_size=(1, 6), strides=(1, 1), padding='valid')(c9)
outputs = Conv2D(4, kernel_size=(1, 1), activation='softmax')(c10)
model = Model(inputs=input_img, outputs=outputs)
return model
def get_bunet(input_img,
n_filters=16,
dropout=1.0,
batchnorm=True,
training=True):
# contracting path
c1 = conv2d_block(input_img,
n_filters=n_filters * 1,
kernel_size=3,
batchnorm=batchnorm)
p1 = MaxPooling2D((2, 2))(c1)
p1 = Dropout(dropout)(p1, training=training)
c2 = conv2d_block(p1,
n_filters=n_filters * 2,
kernel_size=3,
batchnorm=batchnorm)
p2 = MaxPooling2D((2, 2))(c2)
p2 = Dropout(dropout)(p2, training=training)
c3 = conv2d_block(p2,
n_filters=n_filters * 4,
kernel_size=3,
batchnorm=batchnorm)
p3 = MaxPooling2D((2, 2))(c3)
p3 = Dropout(dropout)(p3, training=training)
c4 = conv2d_block(p3,
n_filters=n_filters * 8,
kernel_size=3,
batchnorm=batchnorm)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
p4 = Dropout(dropout)(p4, training=training)
c5 = conv2d_block(p4,
n_filters=n_filters * 16,
kernel_size=3,
batchnorm=batchnorm)
p5 = MaxPooling2D(pool_size=(2, 2))(c5)
p5 = Dropout(dropout)(p5, training=training)
c6 = conv2d_block(p5,
n_filters=n_filters * 32,
kernel_size=3,
batchnorm=batchnorm)
p6 = MaxPooling2D(pool_size=(2, 2))(c6)
p6 = Dropout(dropout)(p6, training=training)
c7 = conv2d_block(p6,
n_filters=n_filters * 64,
kernel_size=3,
batchnorm=batchnorm)
# expansive path
u7 = Conv2DTranspose(n_filters * 32, (3, 3),
strides=(2, 2),
padding='same')(c7)
u7 = concatenate([u7, c6])
u7 = Dropout(dropout)(u7, training=training)
c8 = conv2d_block(u7,
n_filters=n_filters * 32,
kernel_size=3,
batchnorm=batchnorm)
u8 = Conv2DTranspose(n_filters * 16, (3, 3),
strides=(2, 2),
padding='same')(c8)
u8 = concatenate([u8, c5])
u8 = Dropout(dropout)(u8, training=training)
c9 = conv2d_block(u8,
n_filters=n_filters * 16,
kernel_size=3,
batchnorm=batchnorm)
u9 = Conv2DTranspose(n_filters * 8, (3, 3), strides=(2, 2),
padding='same')(c9)
u9 = concatenate([u9, c4])
u9 = Dropout(dropout)(u9, training=training)
c10 = conv2d_block(u9,
n_filters=n_filters * 8,
kernel_size=3,
batchnorm=batchnorm)
u10 = Conv2DTranspose(n_filters * 4, (3, 3),
strides=(2, 2),
padding='same')(c10)
u10 = concatenate([u10, c3])
u10 = Dropout(dropout)(u10, training=training)
c11 = conv2d_block(u10,
n_filters=n_filters * 4,
kernel_size=3,
batchnorm=batchnorm)
u11 = Conv2DTranspose(n_filters * 2, (3, 3),
strides=(2, 2),
padding='same')(c11)
u11 = concatenate([u11, c2])
u11 = Dropout(dropout)(u11, training=training)
c12 = conv2d_block(u11,
n_filters=n_filters * 2,
kernel_size=3,
batchnorm=batchnorm)
u12 = Conv2DTranspose(n_filters * 1, (3, 3),
strides=(2, 2),
padding='same')(c12)
u12 = concatenate([u12, c1])
u12 = Dropout(dropout)(u12, training=training)
c13 = conv2d_block(u12,
n_filters=n_filters * 1,
kernel_size=3,
batchnorm=batchnorm)
prediction = Conv2D(1, (1, 1))(c13)
variances = Conv2D(1, (1, 1), activation="softplus")(c13)
outputs = concatenate([prediction, variances])
model = Model(inputs=input_img, outputs=[outputs])
return model
def FCN(basenet='vgg16', trainable_base=False,
num_output=21, input_shape=(None, None, 3),
weights='imagenet'):
"""Instantiate the FCN8s architecture with keras.
# Arguments
basenet: type of basene {'vgg16'}
trainable_base: Bool whether the basenet weights are trainable
num_output: number of classes
input_shape: input image shape
weights: pre-trained weights to load (None for training from scratch)
# Returns
A Keras model instance
"""
_handle_data_format()
basenet = _get_basenet(basenet)
# input
input = Input(shape=input_shape)
# Get skip_layers=[drop7, pool4, pool3] from the base net: VGG16
skip_layers = basenet(skip_architecture=True)(input)
drop7 = skip_layers[0]
score_fr = Conv2D(filters=num_output, kernel_size=(1, 1),
padding='valid',
name='score_fr')(drop7)
upscore2 = Conv2DTranspose(filters=num_output, kernel_size=(4, 4),
strides=(2, 2), padding='valid', use_bias=False,
data_format=K.image_data_format(),
name='upscore2')(score_fr)
# scale pool4 skip for compatibility
pool4 = skip_layers[1]
scale_pool4 = Lambda(lambda x: x * 0.01, name='scale_pool4')(pool4)
score_pool4 = Conv2D(filters=num_output, kernel_size=(1, 1),
padding='valid', name='score_pool4')(scale_pool4)
score_pool4c = _crop(upscore2, offset=(5, 5),
name='score_pool4c')(score_pool4)
fuse_pool4 = add([upscore2, score_pool4c])
upscore_pool4 = Conv2DTranspose(filters=num_output, kernel_size=(4, 4),
strides=(2, 2), padding='valid',
use_bias=False,
data_format=K.image_data_format(),
name='upscore_pool4')(fuse_pool4)
# scale pool3 skip for compatibility
pool3 = skip_layers[2]
scale_pool3 = Lambda(lambda x: x * 0.0001, name='scale_pool3')(pool3)
score_pool3 = Conv2D(filters=num_output, kernel_size=(1, 1),
padding='valid', name='score_pool3')(scale_pool3)
score_pool3c = _crop(upscore_pool4, offset=(9, 9),
name='score_pool3c')(score_pool3)
fuse_pool3 = add([upscore_pool4, score_pool3c])
# score
upscore8 = Conv2DTranspose(filters=num_output, kernel_size=(16, 16),
strides=(8, 8), padding='valid',
use_bias=False,
data_format=K.image_data_format(),
name='upscore8')(fuse_pool3)
score = _crop(input, offset=(31, 31), name='score')(upscore8)
# model
model = Model(input, score, name='fcn_vgg16')
# load weights
if weights == 'imagenet':
weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels.h5',
basenet.WEIGHTS_PATH,
cache_subdir='models')
layer_names = load_weights(model, weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
# Freezing basenet weights
if not trainable_base:
for layer in model.layers:
if layer.name in layer_names:
layer.trainable = False
return model
s_autoencoder.summary()
s_autoencoder.fit(s_training_data,
s_training_data,
epochs=10,
validation_data=(s_validation_data, s_validation_data))
s_autoencoder.evaluate(s_test_data, s_test_data)
# CNN model, low params
input_layer_cnn = Input(shape=(24, 24, 1))
conv1 = Conv2D(2, (4, 4), padding="same", strides=2)(input_layer_cnn)
mxp1 = MaxPooling2D((2, 2))(conv1)
encoding_layer_cnn = Conv2D(1, (4, 4), padding='same', strides=2)(mxp1)
conv3 = Conv2DTranspose(2, (4, 4), padding='same',
strides=2)(encoding_layer_cnn)
us1 = UpSampling2D((2, 2))(conv3)
decoding_layer_cnn = Conv2DTranspose(1, (4, 4), padding='same', strides=2)(us1)
encoder_cnn = Model(input_layer_cnn, encoding_layer_cnn)
autoencoder_cnn = Model(input_layer_cnn, decoding_layer_cnn)
autoencoder_cnn.compile(optimizer='rmsprop', loss='mean_squared_error')
autoencoder_cnn.summary()
autoencoder_cnn.fit(training_data_cnn,
training_data_cnn,
epochs=100,
validation_data=(validation_data_cnn, validation_data_cnn),
callbacks=[early_stopping])
c4 = Color2D(s4).do()
t4 = Texture2D(s4).do()
m4 = concatenate([c4, t4])
m4 = concatenate([m4, p3])
m4 = Conv2D(nn, (3, 3),
activation=act,
kernel_initializer='he_normal',
padding='same')(m4)
m4 = Dropout(don)(m4)
m4 = Conv2D(nn, (3, 3),
activation=act,
kernel_initializer='he_normal',
padding='same')(m4)
u1 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(m4)
u1 = concatenate([u1, m3])
m5 = Conv2D(32, (3, 3),
activation=act,
kernel_initializer='he_normal',
padding='same')(u1)
m5 = Dropout(don)(m5)
m5 = Conv2D(32, (3, 3),
activation=act,
kernel_initializer='he_normal',
padding='same')(m5)
u2 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(m5)
u2 = concatenate([u2, m2])
m6 = Conv2D(32, (3, 3),
activation=act,
kernel_initializer='he_normal',
padding='same')(s)
c1 = Conv2D(16, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c4 = Conv2DTranspose(16, (2, 2), strides=(4, 4), padding='same')(p2)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c4)
model = Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[mean_iou])
model.summary()
results = model.fit(X_train,
Y_train,
validation_split=0.2,
batch_size=16,
epochs=100)
g = Sequential()
g.add(
Dense(1024,
kernel_initializer=RandomNormal(mean=0, stddev=0.02),
bias_initializer=RandomNormal(mean=0, stddev=0.02),
input_dim=latent_dim))
g.add(BatchNormalization(momentum=0.8))
g.add(Activation('relu'))
g.add(Dense(7 * 7 * 128))
g.add(BatchNormalization(momentum=0.8))
g.add(Activation('relu'))
g.add(Reshape((7, 7, 128)))
g.add(
Conv2DTranspose(64,
kernel_size=(4, 4),
kernel_initializer=RandomNormal(mean=0, stddev=0.02),
bias_initializer=RandomNormal(mean=0, stddev=0.02),
strides=(2, 2),
padding='same'))
g.add(BatchNormalization(momentum=0.8))
g.add(Activation('relu'))
g.add(
Conv2DTranspose(1,
kernel_size=(4, 4),
kernel_initializer=RandomNormal(mean=0, stddev=0.02),
bias_initializer=RandomNormal(mean=0, stddev=0.02),
strides=(2, 2),
padding='same',
activation='tanh'))
g.summary()
def FastSegNet(input_shape, n_labels, kernel=3, pool_size=(2, 2), output_mode='softmax'):
"""
SegNet perpixel image classifier
"""
# encoder
inputs = Input(shape=input_shape)
conv_1 = Convolution2D(32, (kernel, kernel), padding='same')(inputs)
conv_1 = BatchNormalization()(conv_1)
conv_1 = Activation('relu')(conv_1)
pool_1 = MaxPool2D(pool_size, name='Pool_1')(conv_1)
conv_2 = Convolution2D(64, (kernel, kernel), padding='same')(pool_1)
conv_2 = BatchNormalization()(conv_2)
conv_2 = Activation('relu')(conv_2)
pool_2 = MaxPool2D(pool_size, name='Pool_2')(conv_2)
conv_3 = Convolution2D(128, (kernel, kernel), padding='same')(pool_2)
conv_3 = BatchNormalization()(conv_3)
conv_3 = Activation('relu')(conv_3)
pool_3 = MaxPool2D(pool_size, name='Pool_3')(conv_3)
conv_4 = Convolution2D(256, (kernel, kernel), padding='same')(pool_3)
conv_4 = BatchNormalization()(conv_4)
conv_4 = Activation('relu')(conv_4)
pool_4 = MaxPool2D(pool_size, name='Pool_4')(conv_4)
conv_5 = Convolution2D(256, (kernel, kernel), padding='same')(pool_4)
conv_5 = BatchNormalization()(conv_5)
conv_5 = Activation('relu')(conv_5)
pool_5 = MaxPool2D(pool_size, name='Pool_5')(conv_5)
# decoder
unpool_1 = UpSampling2D(pool_size, name='Un-Pool_1')(pool_5)
deconv_1 = Conv2DTranspose(256, (kernel, kernel), padding='same')(unpool_1)
deconv_1 = BatchNormalization()(deconv_1)
deconv_1 = Activation('relu')(deconv_1)
concat_1 = Concatenate()([deconv_1, pool_4])
unpool_2 = UpSampling2D(pool_size, name='Un-Pool_2')(concat_1)
deconv_2 = Conv2DTranspose(256, (kernel, kernel), padding='same')(unpool_2)
deconv_2 = BatchNormalization()(deconv_2)
deconv_2 = Activation('relu')(deconv_2)
concat_2 = Concatenate()([deconv_2, pool_3])
unpool_3 = UpSampling2D(pool_size, name='Un-Pool_3')(concat_2)
deconv_3 = Conv2DTranspose(128, (kernel, kernel), padding='same')(unpool_3)
deconv_3 = BatchNormalization()(deconv_3)
deconv_3 = Activation('relu')(deconv_3)
concat_3 = Concatenate()([deconv_3, pool_2])
unpool_4 = UpSampling2D(pool_size, name='Un-Pool_4')(concat_3)
deconv_4 = Conv2DTranspose(64, (kernel, kernel), padding='same')(unpool_4)
deconv_4 = BatchNormalization()(deconv_4)
deconv_4 = Activation('relu')(deconv_4)
concat_4 = Concatenate()([deconv_4, pool_1])
unpool_5 = UpSampling2D(pool_size, name='Un-Pool_5')(concat_4)
deconv_5 = Conv2DTranspose(32, (kernel, kernel), padding='same')(unpool_5)
deconv_5 = BatchNormalization()(deconv_5)
deconv_5 = Activation('relu')(deconv_5)
deconv_6 = Conv2DTranspose(n_labels, (1, 1), padding='valid')(deconv_5)
deconv_6 = BatchNormalization()(deconv_6)
deconv_6 = Reshape((input_shape[0] * input_shape[1], n_labels), input_shape=(input_shape[0], input_shape[1], n_labels))(deconv_6)
outputs = Activation(output_mode)(deconv_6)
return Model(inputs=inputs, outputs=outputs, name='FastSegNet')
def deconv2d(x, depth, kernel_size, stride=1, padding="SAME", kernel_init=kernel_init_default, bias_init=0):
output = Conv2DTranspose(depth, kernel_size=kernel_size, strides=stride, padding=padding,
kernel_initializer=kernel_init,
bias_initializer=Constant(bias_init))(x)
return output
def unet_model(n_classes=255,
im_sz=128,
n_channels=7,
n_filters_start=32,
growth_factor=2,
upconv=True,
class_weights=cls_wgt):
droprate = 0.11
n_filters = n_filters_start
inputs = Input((128, 128, 7))
#inputs = BatchNormalization()(inputs)
conv1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(inputs)
conv1 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv1)
conv1 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv1)
conv1 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
#pool1 = Dropout(droprate)(pool1)
n_filters *= growth_factor
pool1 = BatchNormalization()(pool1)
conv2 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(pool1)
conv2 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv2)
conv2 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv2)
conv2 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
pool2 = Dropout(droprate)(pool2)
n_filters *= growth_factor
pool2 = BatchNormalization()(pool2)
conv3 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(pool2)
conv3 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv3)
conv3 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv3)
conv3 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
pool3 = Dropout(droprate)(pool3)
n_filters *= growth_factor
pool3 = BatchNormalization()(pool3)
conv4_0 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(pool3)
conv4_0 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_0)
conv4_0 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_0)
conv4_0 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_0)
pool4_1 = MaxPooling2D(pool_size=(2, 2))(conv4_0)
pool4_1 = Dropout(droprate)(pool4_1)
n_filters *= growth_factor
pool4_1 = BatchNormalization()(pool4_1)
conv4_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(pool4_1)
conv4_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_1)
conv4_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_1)
conv4_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_1)
pool4_2 = MaxPooling2D(pool_size=(2, 2))(conv4_1)
pool4_2 = Dropout(droprate)(pool4_2)
n_filters *= growth_factor
conv5 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(pool4_2)
conv5 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv5)
conv5 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv5)
conv5 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv5)
n_filters //= growth_factor
if upconv:
up6_1 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv5), conv4_1
])
else:
up6_1 = concatenate([UpSampling2D(size=(2, 2))(conv5), conv4_1])
up6_1 = BatchNormalization()(up6_1)
conv6_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(up6_1)
conv6_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_1)
conv6_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_1)
conv6_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_1)
conv6_1 = Dropout(droprate)(conv6_1)
n_filters //= growth_factor
if upconv:
up6_2 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv6_1), conv4_0
])
else:
up6_2 = concatenate([UpSampling2D(size=(2, 2))(conv6_1), conv4_0])
up6_2 = BatchNormalization()(up6_2)
conv6_2 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(up6_2)
conv6_2 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_2)
conv6_2 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_2)
conv6_2 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_2)
conv6_2 = Dropout(droprate)(conv6_2)
n_filters //= growth_factor
if upconv:
up7 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv6_2), conv3
])
else:
up7 = concatenate([UpSampling2D(size=(2, 2))(conv6_2), conv3])
up7 = BatchNormalization()(up7)
conv7 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(up7)
conv7 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv7)
conv7 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv7)
conv7 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv7)
conv7 = Dropout(droprate)(conv7)
n_filters //= growth_factor
if upconv:
up8 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv7), conv2
])
else:
up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2])
up8 = BatchNormalization()(up8)
conv8 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(up8)
conv8 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv8)
conv8 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv8)
conv8 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv8)
conv8 = Dropout(droprate)(conv8)
n_filters //= growth_factor
if upconv:
up9 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv8), conv1
])
else:
up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1])
conv9 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(up9)
conv9 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv9)
conv9 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv9)
conv9 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv9)
conv10 = Conv2D(n_classes, (1, 1), activation='sigmoid')(conv9)
model = Model(inputs=inputs, outputs=conv10)
def weighted_binary_crossentropy(y_true, y_pred):
class_loglosses = K.mean(K.binary_crossentropy(y_true, y_pred),
axis=[0, 1, 2])
return K.sum(class_loglosses * K.constant(class_weights))
model.compile(optimizer=Adam(lr=0.1),
loss=weighted_binary_crossentropy,
metrics=[mean_iou])
return model
def unet_define_model() -> Model:
inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
s = Lambda(lambda x: x / 255)(inputs)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(s)
c1 = Dropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Dropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p2)
c3 = Dropout(0.2)(c3)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p3)
c4 = Dropout(0.2)(c4)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p4)
c5 = Dropout(0.3)(c5)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u6)
c6 = Dropout(0.2)(c6)
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u7)
c7 = Dropout(0.2)(c7)
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u8)
c8 = Dropout(0.1)(c8)
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u9)
c9 = Dropout(0.1)(c9)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[dice_score])
return model
c2 = Conv2D(16, (3, 3), activation='relu', padding='same')(p1) c2 = Conv2D(16, (3, 3), activation='relu', padding='same')(c2) p2 = MaxPooling2D((2, 2))(c2) c3 = Conv2D(32, (3, 3), activation='relu', padding='same')(p2) c3 = Conv2D(32, (3, 3), activation='relu', padding='same')(c3) p3 = MaxPooling2D((2, 2))(c3) c4 = Conv2D(64, (3, 3), activation='relu', padding='same')(p3) c4 = Conv2D(64, (3, 3), activation='relu', padding='same')(c4) p4 = MaxPooling2D(pool_size=(2, 2))(c4) c5 = Conv2D(128, (3, 3), activation='relu', padding='same')(p4) c5 = Conv2D(128, (3, 3), activation='relu', padding='same')(c5) u6 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c5) u6 = concatenate([u6, c4]) c6 = Conv2D(64, (3, 3), activation='relu', padding='same')(u6) c6 = Conv2D(64, (3, 3), activation='relu', padding='same')(c6) u7 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c6) u7 = concatenate([u7, c3]) c7 = Conv2D(32, (3, 3), activation='relu', padding='same')(u7) c7 = Conv2D(32, (3, 3), activation='relu', padding='same')(c7) u8 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c7) u8 = concatenate([u8, c2]) c8 = Conv2D(16, (3, 3), activation='relu', padding='same')(u8) c8 = Conv2D(16, (3, 3), activation='relu', padding='same')(c8) u9 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same')(c8)
def unet(width, height, channels, pretrained_weights = None):
inputs = Input((width, height, channels))
s = Lambda(lambda x: x / 255) (inputs)
c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (s)
c1 = Dropout(0.1) (c1)
c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c1)
p1 = MaxPooling2D((2, 2)) (c1)
c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (p1)
c2 = Dropout(0.1) (c2)
c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c2)
p2 = MaxPooling2D((2, 2)) (c2)
c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (p2)
c3 = Dropout(0.2) (c3)
c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c3)
p3 = MaxPooling2D((2, 2)) (c3)
c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (p3)
c4 = Dropout(0.2) (c4)
c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c4)
p4 = MaxPooling2D(pool_size=(2, 2)) (c4)
c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (p4)
c5 = Dropout(0.3) (c5)
c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same') (c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (u6)
c6 = Dropout(0.2) (c6)
c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (u7)
c7 = Dropout(0.2) (c7)
c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (u8)
c8 = Dropout(0.1) (c8)
c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (u9)
c9 = Dropout(0.1) (c9)
c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid') (c9)
model = Model(inputs=[inputs], outputs=[outputs])
#model.compile(optimizer=Adam(lr = 1e-4), loss='binary_crossentropy', metrics=[dice_coef,'accuracy'])
if(pretrained_weights):
model.load_weights(pretrained_weights)
model.summary()
return model
