def init_and_train_model(train_df, train_path, val_df, val_path, model_type, BACKBONE, AUGMENTATIONS, batch_size, epoch_num):
preprocess_input = sm.backbones.get_preprocessing(BACKBONE)
if(model_type == "Linknet"):
model = sm.Linknet(BACKBONE,input_shape = (256, 256, 3), encoder_weights='imagenet', encoder_freeze=True)
train_gen = keras_generator_with_augs(train_df, batch_size, train_path, AUGMENTATIONS)
val_gen = keras_generator(val_df, batch_size, val_path)
elif(model_type == "Unet"):
model = sm.Unet(BACKBONE,input_shape = (256, 256, 3), encoder_weights='imagenet', encoder_freeze=True)
train_gen = keras_generator_with_augs(train_df, batch_size, train_path, AUGMENTATIONS)
val_gen = keras_generator(val_df, batch_size, val_path)
elif(model_type == "FPN"):
model = sm.FPN(BACKBONE,input_shape = (256, 256, 3), encoder_weights='imagenet', encoder_freeze=True)
train_gen = keras_generator_with_augs(train_df, batch_size, train_path, AUGMENTATIONS)
val_gen = keras_generator(val_df, batch_size, val_path)
elif(model_type == "PSPNet"):
model = sm.PSPNet(BACKBONE,input_shape = (240, 240, 3), encoder_weights='imagenet', encoder_freeze=True)
train_gen = keras_generator_with_augs(train_df, batch_size, train_path, AUGMENTATIONS, PSPNet=True)
val_gen = keras_generator(val_df, batch_size, val_path, PSPNet = True)
model.compile(
'Adam',
loss=sm.losses.dice_loss,
metrics=[sm.metrics.dice_score],
)
best_w = keras.callbacks.ModelCheckpoint(model_type + '_' + BACKBONE + '_best.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
last_w = keras.callbacks.ModelCheckpoint(model_type + '_' + BACKBONE + '_last.h5',
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1)
callbacks = [best_w, last_w]
history = model.fit_generator(train_gen,
steps_per_epoch=50,
epochs=epoch_num,
verbose=1,
callbacks=callbacks,
validation_data=val_gen,
validation_steps=50,
class_weight=None,
max_queue_size=1,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0)
return model, history
def get_sm_model():
CLASSES = ['sofa']
sm.set_framework('tf.keras')
model = sm.Linknet(BACKBONE, classes=len(CLASSES), encoder_weights=None)
return model
def inference(model_name, image_folder_path):
# wrap our image inside the Dataset wrapper used for training,
# TODO: remove this and add custom pipeline for preprocessing.
trial_dataset = Dataset(
image_folder_path,
image_folder_path,
classes=CLASSES,
augmentation=get_validation_augmentation(),
preprocessing=get_preprocessing(preprocess_input),
)
print(model_name)
if model_name=="unet":
print("Loading Models. This might take some time...")
modelUnet = sm.Unet(BACKBONE, classes=n_classes, activation=activation)
model_c = config.STYLES["unet"]
model_path = os.path.join(f"{config.MODEL_PATH}",f"{model_c}.h5")
modelUnet.load_weights(model_path)
print("Loaded Unet.")
model = modelUnet
elif model_name=="featurepyramidnetwork":
modelFPN = sm.FPN(BACKBONE, classes=n_classes, activation=activation)
model_c = config.STYLES["featurepyramidnetwork"]
model_path = os.path.join(f"{config.MODEL_PATH}",f"{model_c}.h5")
modelFPN.load_weights(model_path)
print("Loaded FPN.")
model = modelFPN
elif model_name=="linknet":
modelLinknet = sm.Linknet(BACKBONE, classes=n_classes, activation=activation)
model_c = config.STYLES["linknet"]
model_path = os.path.join(f"{config.MODEL_PATH}",f"{model_c}.h5")
modelLinknet.load_weights(model_path)
print("Loaded Linknet.")
model = modelLinknet
# model.load_weights(model_path)
# trial folder must have only one image. hence the [0]
image, gt_mask = trial_dataset[0]
image = np.expand_dims(image, axis=0)
pr_mask = model.predict(image).round()
#print(pr_mask.shape)
#print(pr_mask[0].shape)
# make image back to normal
image=denormalize(image.squeeze())
gt_mask=gt_mask[..., 0].squeeze()
pr_mask=pr_mask[..., 0].squeeze()
# DEBUG:
# visualize(
# image=image,
# gt_mask=gt_mask,
# pr_mask=pr_mask,
# )
del model
gc.collect()
return pr_mask,gt_mask
def create_model():
"""
Function used to create model based on parameters specified in settings.py
"""
if model_type == 'myresunet':
model = myresunet.create_model()
elif model_type == 'unet':
model = sm.Unet(backbone_name=backbone,
input_shape=(image_height, image_width, 3),
classes=num_classes,
activation='softmax',
encoder_weights='imagenet',
encoder_freeze=False,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(decoder_scaler * 256, decoder_scaler * 128, decoder_scaler * 64,
decoder_scaler * 32, decoder_scaler * 16),
decoder_use_batchnorm=True)
elif model_type == 'fpn':
model = sm.FPN(backbone_name=backbone,
input_shape=(image_height, image_width, 3),
classes=num_classes,
activation='softmax',
encoder_weights='imagenet',
encoder_freeze=False,
encoder_features='default',
pyramid_block_filters=decoder_scaler * 256,
pyramid_use_batchnorm=True,
pyramid_aggregation='concat',
pyramid_dropout=None)
elif model_type == 'linknet':
model = sm.Linknet(backbone_name=backbone,
input_shape=(image_height, image_width, 3),
classes=num_classes,
activation='softmax',
encoder_weights='imagenet',
encoder_freeze=False,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(None, None, None, None, decoder_scaler * 16),
decoder_use_batchnorm=True)
elif model_type == 'pspnet':
model = sm.PSPNet(backbone_name=backbone,
input_shape=(image_height, image_width, 3),
classes=num_classes,
activation='softmax',
encoder_weights='imagenet',
encoder_freeze=False,
downsample_factor=8,
psp_conv_filters=decoder_scaler * 512,
psp_pooling_type='avg',
psp_use_batchnorm=True,
psp_dropout=None)
else:
print('Invalid segmentation model type')
exit(0)
return model
def create_model(border=False, trainable_encoder=False):
if model_type == 'unet':
model = sm.Unet(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=2 if border else 1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(256, 128, 64, 32, 16),
decoder_use_batchnorm=True)
elif model_type == 'fpn':
model = sm.FPN(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=2 if border else 1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
encoder_features='default',
pyramid_block_filters=256,
pyramid_use_batchnorm=True,
pyramid_aggregation='concat',
pyramid_dropout=None)
elif model_type == 'linknet':
model = sm.Linknet(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=2 if border else 1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(None, None, None, None, 16),
decoder_use_batchnorm=True)
elif model_type == 'pspnet':
model = sm.PSPNet(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=2 if border else 1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
downsample_factor=8,
psp_conv_filters=512,
psp_pooling_type='avg',
psp_use_batchnorm=True,
psp_dropout=None)
else:
print('Invalid segmentation model type')
exit(0)
return model
def get_backboned_model(model, backbone, freeze=True):
if model == 'Unet':
base_model = sm.Unet(backbone_name=backbone,
encoder_weights='imagenet',
classes=1,
activation='sigmoid',
freeze_encoder=freeze)
elif model == 'FPN':
base_model = sm.FPN(backbone_name=backbone,
encoder_weights='imagenet',
classes=1,
activation='sigmoid',
freeze_encoder=freeze)
elif model == 'Linknet':
base_model = sm.Linknet(backbone_name=backbone,
encoder_weights='imagenet',
classes=1,
activation='sigmoid',
freeze_encoder=freeze)
elif model == 'PSPNet':
base_model = sm.PSPNet(backbone_name=backbone,
encoder_weights='imagenet',
classes=1,
activation='sigmoid',
freeze_encoder=freeze)
else:
print('Model not identified! Unet is selected')
base_model = sm.Unet(backbone_name=backbone,
encoder_weights='imagenet',
classes=1,
activation='sigmoid',
freeze_encoder=freeze)
inp = Input(shape=(96, 96, 1))
l1 = Conv2D(3, (1, 1))(inp) # map N channels data to 3 channels
out = base_model(l1)
model = Model(inp, out, name=base_model.name)
# print(model.summary())
return model
def get_model_definition():
backbone = 'mobilenet'
n_classes = 2
lr = 0.001
activation = 'softmax'
pre_process_input = sm.get_preprocessing(backbone)
optimizer = keras.optimizers.Adam(lr)
metrics = [
sm.metrics.FScore(threshold=0.5),
]
model = sm.Linknet(backbone,
classes=n_classes,
activation=activation,
encoder_freeze=True)
if n_classes == 1:
loss = sm.losses.BinaryFocalLoss()
else:
loss = sm.losses.CategoricalFocalLoss()
model.compile(optimizer, loss, metrics)
return model, pre_process_input
def _build(self):
if self.type == 'Unet':
model = sm.Unet(backbone_name=self.backbone,
classes=self.n_class,
activation=self.activate,
encoder_weights=self.encoder_weights)
elif self.type == 'Linknet':
model = sm.Linknet(backbone_name=self.backbone,
classes=self.n_class,
activation=self.activate,
encoder_weights=self.encoder_weights)
elif self.type == 'FPN':
model = sm.FPN(backbone_name=self.backbone,
classes=self.n_class,
activation=self.activate,
encoder_weights=self.encoder_weights)
else:
raise ValueError('Model type {} not support now.'.format(self.type))
return model
RESULTADOS_finetuning = []
# Define the K-fold Cross Validator
kfold = KFold(n_splits=num_folds, shuffle=True, random_state=1)
# K-fold Cross Validation model evaluation
fold_no = 1
for train, test in kfold.split(imagens, mascaras_medico):
print("######### KFOLD ", fold_no, "#########")
###### CRIA O MODELO
BACKBONE = 'resnet34'
preprocess_input = sm.get_preprocessing(BACKBONE)
# define model
model = sm.Linknet(BACKBONE,
encoder_weights='imagenet',
encoder_freeze=True)
model.compile(
'Adam',
loss=sm.losses.bce_jaccard_loss,
metrics=[sm.metrics.iou_score],
)
###### dividir treino e validação, lembrando que como são 5 folds, tem 80% pra trein e 20% para test. Então dos 80% de treino pego 25% para validação. E ai mantenho a mesma proporção de antes do kfold (60% treino, 20% teste e 20% validação)
x_train, x_val, y_train, y_val = train_test_split(imagens[train],
mascaras_medico[train],
test_size=0.25,
random_state=11)
x_train = np.asarray(x_train)
y_train = (np.asarray(y_train) > threshold_otsu(np.asarray(y_train)))
with tf.device('/gpu:3'):
dataset = Dataset(x_train_path, y_train_path, classes=['non-polyp', 'polyp'], augmentation=get_training_augmentation())
BATCH_SIZE = 8
CLASSES = ['non-polyp', 'polyp']
LR = 0.0001
EPOCHS = 25
IMAGE_ORDERING = 'channels_last'
n_classes = 2
# SOTA
BACKBONE = 'resnet34'
# define model
model = sm.Unet(BACKBONE, encoder_weights='imagenet')
model = sm.Linknet(BACKBONE, encoder_weights='imagenet')
model = sm.FPN(BACKBONE, encoder_weights='imagenet')
model = sm.PSPNet(BACKBONE, encoder_weights='imagenet')
model = fcn_8.fcn_8(2)
optim = tf.keras.optimizers.Adam(LR)
# Segmentation models losses can be combined together by '+' and scaled by integer or float factor
# set class weights for dice_loss (car: 1.; pedestrian: 2.; background: 0.5;)
dice_loss = sm.losses.DiceLoss(class_weights=np.array([0.5, 1]))
focal_loss = sm.losses.BinaryFocalLoss()
# if n_classes == 1 else sm.losses.CategoricalFocalLoss()
total_loss = dice_loss + (1 * focal_loss)
# actulally total_loss can be imported directly from library, above example just show you how to manipulate with losses
def Setup(self):
'''
User function: Setup all the parameters
Args:
None
Returns:
None
'''
# define network parameters
self.system_dict["local"]["n_classes"] = 1 if len(
self.system_dict["params"]["classes_to_train"]) == 1 else (
len(self.system_dict["params"]["classes_to_train"]) +
1) # case for binary and multiclass segmentation
activation = 'sigmoid' if self.system_dict["local"][
"n_classes"] == 1 else 'softmax'
#create model
if (self.system_dict["params"]["model"] == "Unet"):
self.system_dict["local"]["model"] = sm.Unet(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
elif (self.system_dict["params"]["model"] == "FPN"):
self.system_dict["local"]["model"] = sm.FPN(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
elif (self.system_dict["params"]["model"] == "Linknet"):
self.system_dict["local"]["model"] = sm.Linknet(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
elif (self.system_dict["params"]["model"] == "PSPNet"):
self.system_dict["local"]["model"] = sm.PSPNet(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
# define optomizer
optim = keras.optimizers.Adam(0.0001)
# Segmentation models losses can be combined together by '+' and scaled by integer or float factor
dice_loss = sm.losses.DiceLoss()
focal_loss = sm.losses.BinaryFocalLoss() if self.system_dict["local"][
"n_classes"] == 1 else sm.losses.CategoricalFocalLoss()
total_loss = dice_loss + (1 * focal_loss)
# actulally total_loss can be imported directly from library, above example just show you how to manipulate with losses
# total_loss = sm.losses.binary_focal_dice_loss # or sm.losses.categorical_focal_dice_loss
metrics = [
sm.metrics.IOUScore(threshold=0.5),
sm.metrics.FScore(threshold=0.5)
]
# compile keras model with defined optimozer, loss and metrics
self.system_dict["local"]["model"].compile(optim, total_loss, metrics)
self.system_dict["local"]["model"].load_weights(
self.system_dict["params"]["path_to_model"])
def create_model(double_size=True, slide_augmentation=True, trainable_encoder=True, n=32, dropout=0.2):
if model_type == 'my_res_unet':
model = my_res_unet(n=n, batch_norm=True, dropout=dropout, slide_augmentation=slide_augmentation)
else:
image_size = 256 if double_size else 128
if model_type == 'unet':
model = sm.Unet(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(16*n, 8*n, 4*n, 2*n, n),
decoder_use_batchnorm=True)
elif model_type == 'fpn':
model = sm.FPN(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
encoder_features='default',
pyramid_block_filters=256,
pyramid_use_batchnorm=True,
pyramid_dropout=None,
final_interpolation='bilinear')
elif model_type == 'linknet':
model = sm.Linknet(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(None, None, None, None, 16),
decoder_use_batchnorm=True)
elif model_type == 'pspnet':
image_size = 240 if double_size else 120
model = sm.PSPNet(backbone_name=backbone,
input_shape=(image_size, image_size, 3),
classes=1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=not trainable_encoder,
downsample_factor=8,
psp_conv_filters=512,
psp_pooling_type='avg',
psp_use_batchnorm=True,
psp_dropout=None,
final_interpolation='bilinear')
else:
print('Invalid segmentation model type')
exit(0)
if not slide_augmentation:
x = keras.layers.Input(shape=(101, 101, 1), name='input')
if model_type == 'pspnet':
y = keras.layers.ZeroPadding2D(((9, 10), (9, 10)), name='zero_pad_input')(x)
else:
y = keras.layers.ZeroPadding2D(((13, 14), (13, 14)), name='zero_pad_input')(x)
y = keras.layers.Cropping2D()(y)
else:
if model_type == 'pspnet':
x = keras.layers.Input(shape=(120, 120, 1), name='input')
else:
x = keras.layers.Input(shape=(128, 128, 1), name='input')
y = x
if double_size:
y = keras.layers.UpSampling2D(size=(2, 2), interpolation='bilinear')(y)
y = keras.layers.concatenate([y, y, y], name='channel_x3')
y = model(y)
if double_size:
y = keras.layers.AvgPool2D(pool_size=(2, 2))(y)
model = keras.models.Model(x, y)
return model
print("Unet execution time is: ", execution_time_unet)
model1.save('unet_res34_backbone_50epochs.hdf5')
# convert the history.history dict to a pandas DataFrame:
hist1_df = pd.DataFrame(history1.history)
hist1_csv_file = 'history_unet_50epochs.csv'
with open(hist1_csv_file, mode='w') as f:
hist1_df.to_csv(f)
############################################################
###Model 2
#Using the same backbone as unet
# define model (Change to unet or Linknet based on the need )
model2 = sm.Linknet(BACKBONE1,
encoder_weights='imagenet',
classes=n_classes,
activation=activation)
# compile keras model with defined optimozer, loss and metrics
model2.compile(optim, total_loss, metrics=metrics)
print(model2.summary())
start2 = datetime.now()
history2 = model2.fit(X_train1,
y_train_cat,
batch_size=8,
epochs=50,
verbose=1,
validation_data=(X_test1, y_test_cat))
ModelCheckpoint(os.path.join(args.model_path,
'centernet_loss_{epoch}.h5'),
monitor='loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min'),
ReduceLROnPlateau(monitor='val_loss',
factor=0.25,
patience=3,
verbose=1,
mode='min')
]
model = sm.Linknet('resnet18',
input_shape=(args.image_height, args.image_width, 3),
classes=train_generator.classes + 2,
activation='sigmoid')
model.summary()
loss = centernet_loss(train_generator.num_classes, args.alpha_pos,
args.alpha_neg, args.near_center_coef,
args.negatives_weight, args.regression_weight)
opt = Nadam(lr=args.lr)
model.compile(opt,
loss=loss,
metrics=[
center_pos_loss(train_generator.num_classes),
center_neg_loss(train_generator.num_classes),
reg_loss(train_generator.num_classes)
def get_model(model,
BACKBONE,
opt,
loss,
metric,
nclass=None,
freeze_encoder=False,
batchnormalization=True,
dropout=None):
h, w = None, None
if nclass is not None:
nclass = nclass
else:
nclass = n_classes
if model == 'fpn':
model = sm.FPN(BACKBONE,
classes=nclass,
input_shape=(h, w, 3),
activation='sigmoid',
encoder_freeze=freeze_encoder,
pyramid_use_batchnorm=batchnormalization,
pyramid_dropout=dropout)
model.compile(optimizer=opt, loss=loss, metrics=metric)
elif model == 'unet':
model = sm.Unet(BACKBONE,
classes=nclass,
input_shape=(h, w, 3),
activation='sigmoid')
model.compile(optimizer=opt, loss=loss, metrics=metric)
elif model == 'psp':
model = sm.PSPNet(BACKBONE,
classes=nclass,
input_shape=(h, w, 3),
activation='sigmoid')
model.compile(optimizer=opt, loss=loss, metrics=metric)
elif model == 'linknet':
model = sm.Linknet(BACKBONE,
classes=nclass,
input_shape=(h, w, 3),
activation='sigmoid')
model.compile(optimizer=opt, loss=loss, metrics=metric)
elif model == 'xnet':
model = smx.Xnet(BACKBONE,
classes=nclass,
input_shape=(h, w, 3),
activation='sigmoid')
model.compile(optimizer=opt, loss=loss, metrics=metric)
elif model == 'jpu':
model = JPU_DeepLab(h, w, nclass)
model.compile(optimizer=opt, loss=loss, metrics=metric)
elif model == 'deeplab':
model = Deeplabv3(weights=None,
input_shape=(h, w, 3),
classes=4,
backbone='xception',
alpha=1.,
activation='sigmoid')
model.compile(optimizer=opt, loss=loss, metrics=metric)
else:
raise ValueError('Unknown network ' + model)
return model
def RunTest(
params,
model_name_template='models_3/{model}_{backbone}_{optimizer}_{augmented_image_size}-{padded_image_size}-{nn_image_size}_lrf{lrf}_{metric}_{CC}_f{test_fold_no}_{phash}'
):
# # Params
# In[ ]:
DEV_MODE_RANGE = 0 # off
# In[ ]:
# In[ ]:
def params_dict():
return {
x[0]: x[1]
for x in vars(params).items() if not x[0].startswith('__')
}
def params_str():
return '\n'.join([
repr(x[0]) + ' : ' + repr(x[1]) + ','
for x in vars(params).items() if not x[0].startswith('__')
])
def params_hash(shrink_to=6):
import hashlib
import json
return hashlib.sha1(
json.dumps(params_dict(),
sort_keys=True).encode()).hexdigest()[:shrink_to]
def params_save(fn, verbose=True):
params_fn = fn + '.param.txt'
with open(params_fn, 'w+') as f:
s = params_str()
hash = params_hash(shrink_to=1000)
s = '{\n' + s + '\n}\nhash: ' + hash[:6] + ' ' + hash[6:]
f.write(s)
if verbose:
print('params: ' + s + '\nsaved to ' + params_fn)
# # Imports
# In[ ]:
import sys
#sys.path.append(r'D:\Programming\3rd_party\keras')
# In[ ]:
import sys
from imp import reload
import numpy as np
import keras
import datetime
import time
from keras.models import Model, load_model
from keras.layers import Input, Dropout, BatchNormalization, Activation, Add
from keras.layers.core import Lambda
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, CSVLogger
from keras import backend as K
import tensorflow as tf
# # Load data
# In[ ]:
import load_data
load_data = reload(load_data)
import keras_unet_divrikwicky_model
keras_unet_divrikwicky_model = reload(keras_unet_divrikwicky_model)
# In[ ]:
train_df = load_data.LoadData(train_data=True,
DEV_MODE_RANGE=DEV_MODE_RANGE,
to_gray=False)
# In[ ]:
train_df.images[0].shape
# In[ ]:
train_images, train_masks, validate_images, validate_masks = load_data.SplitTrainData(
train_df, params.test_fold_no)
train_images.shape, train_masks.shape, validate_images.shape, validate_masks.shape
# # Reproducability setup:
# In[ ]:
import random as rn
import os
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(params.seed)
rn.seed(params.seed)
#session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
tf.set_random_seed(params.seed)
#sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
sess = tf.Session(graph=tf.get_default_graph())
K.set_session(sess)
# # IOU metric
# In[ ]:
thresholds = np.array(
[0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95])
def iou(img_true, img_pred):
assert (img_true.shape[-1] == 1) and (len(img_true.shape) == 3) or (
img_true.shape[-1] != 1) and (len(img_true.shape) == 2)
i = np.sum((img_true * img_pred) > 0)
u = np.sum((img_true + img_pred) > 0)
if u == 0:
return 1
return i / u
def iou_metric(img_true, img_pred):
img_pred = img_pred > 0.5 # added by sgx 20180728
if img_true.sum() == img_pred.sum() == 0:
scores = 1
else:
scores = (thresholds <= iou(img_true, img_pred)).mean()
return scores
def iou_metric_batch(y_true_in, y_pred_in):
batch_size = len(y_true_in)
metric = []
for batch in range(batch_size):
value = iou_metric(y_true_in[batch], y_pred_in[batch])
metric.append(value)
#print("metric = ",metric)
return np.mean(metric)
# adapter for Keras
def my_iou_metric(label, pred):
metric_value = tf.py_func(iou_metric_batch, [label, pred], tf.float64)
return metric_value
# # Data generator
# In[ ]:
mean_val = np.mean(train_images.apply(np.mean))
mean_std = np.mean(train_images.apply(np.std))
mean_val, mean_std
#####################################
def FillCoordConvNumpy(imgs):
print(imgs.shape)
assert len(imgs.shape) == 4
assert imgs.shape[3] == 3
n = imgs.shape[2]
hor_img = np.linspace(-1., 1., n).reshape((1, 1, n, 1))
n = imgs.shape[1]
ver_img = np.linspace(-1., 1., n).reshape((1, n, 1, 1))
imgs[:, :, :, 0:1] = hor_img
imgs[:, :, :, 2:3] = ver_img
def FillCoordConvList(imgs):
print(imgs.shape)
assert len(imgs[0].shape) == 3
assert imgs[0].shape[2] == 3
for img in imgs:
n = img.shape[1]
hor_img = np.linspace(-1., 1., n).reshape((1, n, 1))
n = img.shape[0]
ver_img = np.linspace(-1., 1., n).reshape((n, 1, 1))
img[:, :, 0:1] = hor_img
img[:, :, 2:3] = ver_img
if params.coord_conv:
FillCoordConvList(train_images)
FillCoordConvList(validate_images)
print(train_images[0][0, 0, 0], train_images[0][0, 0, 2])
assert train_images[0][0, 0, 0] == -1.
assert train_images[0][0, 0, 2] == 1.
######################################
from my_augs import AlbuDataGenerator
# # model
# In[ ]:
sys.path.append('../3rd_party/segmentation_models')
import segmentation_models
segmentation_models = reload(segmentation_models)
from segmentation_models.utils import set_trainable
# In[ ]:
if not hasattr(params, 'model_params'):
params.model_params = {}
if params.load_model_from:
model = load_model(params.load_model_from,
custom_objects={'my_iou_metric': my_iou_metric})
print('MODEL LOADED from: ' + params.load_model_from)
else:
model = None
if params.model == 'FNN':
model = segmentation_models.FPN(
backbone_name=params.backbone,
input_shape=(None, None, params.channels),
encoder_weights=params.initial_weightns,
freeze_encoder=True,
dropout=params.dropout,
**params.model_params)
if params.model == 'FNNdrop':
model = segmentation_models.FPNdrop(
backbone_name=params.backbone,
input_shape=(None, None, params.channels),
encoder_weights=params.initial_weightns,
freeze_encoder=True,
dropout=params.dropout,
**params.model_params)
if params.model == 'Unet':
model = segmentation_models.Unet(
backbone_name=params.backbone,
input_shape=(None, None, params.channels),
encoder_weights=params.initial_weightns,
freeze_encoder=True,
**params.model_params)
if params.model == 'Linknet':
model = segmentation_models.Linknet(
backbone_name=params.backbone,
input_shape=(None, None, params.channels),
encoder_weights=params.initial_weightns,
freeze_encoder=True,
**params.model_params)
if params.model == 'divrikwicky':
model = keras_unet_divrikwicky_model.CreateModel(
params.nn_image_size, **params.model_params)
params.backbone = ''
assert model
for l in model.layers:
if isinstance(
l, segmentation_models.fpn.layers.UpSampling2D) or isinstance(
l, keras.layers.UpSampling2D):
print(l)
if hasattr(l, 'interpolation'):
print(l.interpolation)
if hasattr(params, 'model_params'
) and 'interpolation' in params.model_params:
l.interpolation = params.model_params['interpolation']
else:
print('qq')
if hasattr(params,
'kernel_constraint_norm') and params.kernel_constraint_norm:
for l in model.layers:
if hasattr(l, 'kernel_constraint'):
print('kernel_constraint for ', l, ' is set to ',
params.kernel_constraint_norm)
l.kernel_constraint = keras.constraints.get(
keras.constraints.max_norm(params.kernel_constraint_norm))
# In[ ]:
model_out_file = model_name_template.format(
lrf=params.ReduceLROnPlateau['factor'],
metric=params.monitor_metric[0],
CC='CC' if params.coord_conv else '',
**vars(params)) + '_f{test_fold_no}_{phash}'.format(
test_fold_no=params.test_fold_no, phash=params_hash())
now = datetime.datetime.now()
print('model: ' + model_out_file + ' started at ' +
now.strftime("%Y.%m.%d %H:%M:%S"))
assert not os.path.exists(model_out_file + '.model')
params_save(model_out_file, verbose=True)
log_out_file = model_out_file + '.log.csv'
# In[ ]:
#model = load_model(model1_file, ) #, 'lavazs_loss': lavazs_loss
# # Train
# In[ ]:
optimizer = params.optimizer
if optimizer == 'adam':
optimizer = keras.optimizers.adam(**params.optimizer_params)
elif optimizer == 'sgd':
optimizer = keras.optimizers.sgd(**params.optimizer_params)
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=["acc", my_iou_metric]) #, my_iou_metric
# In[ ]:
if params.coord_conv:
mean = ((0, mean_val, 0), (1, mean_std, 1))
else:
mean = (mean_val, mean_std)
train_gen = AlbuDataGenerator(train_images,
train_masks,
batch_size=params.batch_size,
nn_image_size=params.nn_image_size,
mode=params.train_augmentation_mode,
shuffle=True,
params=params,
mean=mean)
val_gen = AlbuDataGenerator(validate_images,
validate_masks,
batch_size=params.test_batch_size,
nn_image_size=params.nn_image_size,
mode=params.test_augmentation_mode,
shuffle=False,
params=params,
mean=mean)
# In[ ]:
sys.path.append('../3rd_party/keras-tqdm')
from keras_tqdm import TQDMCallback, TQDMNotebookCallback
# In[ ]:
start_t = time.clock()
if params.epochs_warmup:
history = model.fit_generator(
train_gen,
validation_data=None,
epochs=params.epochs_warmup,
callbacks=[TQDMNotebookCallback(leave_inner=True)],
validation_steps=None,
workers=5,
use_multiprocessing=False,
verbose=0)
set_trainable(model)
batches_per_epoch = len(train_images) // params.batch_size
print("batches per epoch: ", batches_per_epoch)
test_epochs = 30
steps = test_epochs * batches_per_epoch
val_period = steps // 1000
print("steps: ", steps, " val_period", val_period)
lr_sheduler = EvalLrTest(log_out_file,
val_gen,
val_period=val_period,
steps=steps)
history = model.fit_generator(
train_gen,
validation_data=None,
epochs=params.epochs,
initial_epoch=params.epochs_warmup,
callbacks=[TQDMNotebookCallback(leave_inner=True), lr_sheduler],
validation_steps=None,
workers=5,
use_multiprocessing=False,
verbose=0)
# In[ ]:
print(params_str())
print('done: ' + model_out_file)
print('elapsed: {}s ({}s/iter)'.format(
time.clock() - start_t, (time.clock() - start_t) / len(history.epoch)))
return model
# models pre load for faster execution.
print("Loading Models. This might take some time...")
modelUnet = sm.Unet(BACKBONE, classes=n_classes, activation=activation)
model_c = config.STYLES["unet"]
model_path = os.path.join(f"{config.MODEL_PATH}", f"{model_c}.h5")
modelUnet.load_weights(model_path)
print("Loaded Unet.")
modelFPN = sm.FPN(BACKBONE, classes=n_classes, activation=activation)
model_c = config.STYLES["featurepyramidnetwork"]
model_path = os.path.join(f"{config.MODEL_PATH}", f"{model_c}.h5")
modelFPN.load_weights(model_path)
print("Loaded FPN.")
modelLinknet = sm.Linknet(BACKBONE, classes=n_classes, activation=activation)
model_c = config.STYLES["linknet"]
model_path = os.path.join(f"{config.MODEL_PATH}", f"{model_c}.h5")
modelLinknet.load_weights(model_path)
print("Loaded Linknet.")
# below was the part of the pipline is used for training and preprocessing
# TODO: Replace this pipeline with custom for faster inference.
# helper function for data visualization
def visualize(**images):
"""PLot images in one row."""
n = len(images)
plt.figure(figsize=(16, 5))
for i, (name, image) in enumerate(images.items()):
plt.subplot(1, n, i + 1)
target_index.append(index)
else:
continue
target_index = np.asarray(target_index)
X_tst = X_tst[target_index]
Y_tst = Y_tst[target_index]
print("there are {} images with target mask in testset".format(len(Y_tst)))
# model construction (keras + sm)
if model_id == 0:
model = sm.Unet(classes=1,
activation='sigmoid',
encoder_weights='imagenet')
elif model_id == 1:
model = sm.Linknet(classes=1,
activation='sigmoid',
encoder_weights='imagenet')
elif model_id == 2:
model = sm.FPN(classes=1, activation='sigmoid', encoder_weights='imagenet')
elif model_id == 3:
model = sm.PSPNet(classes=1,
activation='sigmoid',
encoder_weights='imagenet') # input size must be 384x384
data_gen_args = dict(rotation_range=360,
width_shift_range=0.15,
height_shift_range=0.15,
zoom_range=0.15,
brightness_range=[0.7, 1.3],
horizontal_flip=True,
vertical_flip=False,
def train(self,
model_name='unet',
backbone='resnet50',
fine_tune=False,
model_path=None,
opt='adam',
lr=0.001,
shape=(256, 256)):
os.makedirs('saved_models/segmentation', exist_ok=True)
if fine_tune: lr = lr / 10
opt_dict = {'adam': Adam(lr), 'sgd': SGD(lr), 'adadelta': Adadelta(lr)}
if fine_tune and model_path:
new_name = model_path.strip('.h5') + '_fine-tune_{}.h5'.format(opt)
model = keras.models.load_model(model_path, compile=False)
model.compile(
optimizer=opt_dict[opt.lower()],
loss=sm.losses.bce_jaccard_loss,
metrics=['acc', sm.metrics.iou_score, sm.metrics.f1_score])
model.summary()
model.fit_generator(
generator=self.data_loader.generator(is_train=True,
shape=shape,
shrink=self.shrink),
steps_per_epoch=self.data_loader.train_steps,
validation_data=self.data_loader.generator(is_train=False,
shape=shape,
shrink=self.shrink),
validation_steps=self.data_loader.val_steps,
verbose=1,
initial_epoch=0,
epochs=300,
callbacks=[
keras.callbacks.TensorBoard('logs'),
keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
patience=7,
verbose=1),
keras.callbacks.EarlyStopping(monitor='val_loss',
patience=38,
verbose=1),
keras.callbacks.ModelCheckpoint(
monitor='val_loss',
verbose=1,
save_weights_only=False,
save_best_only=True,
filepath='saved_models/segmentation/' + new_name)
])
else:
if model_name.lower() == 'unet':
model = sm.Unet(backbone,
encoder_weights='imagenet',
activation='sigmoid',
classes=1,
input_shape=(shape[0], shape[1], 3),
decoder_use_batchnorm=True)
elif model_name.lower() == 'pspnet':
model = sm.PSPNet(backbone,
encoder_weights='imagenet',
activation='sigmoid',
classes=1,
input_shape=(shape[0], shape[1], 3))
elif model_name.lower() == 'fpn':
model = sm.FPN(backbone,
encoder_weights='imagenet',
activation='sigmoid',
classes=1,
input_shape=(shape[0], shape[1], 3))
elif model_name.lower() == 'linknet':
model = sm.Linknet(backbone,
encoder_weights='imagenet',
activation='sigmoid',
classes=1,
input_shape=(shape[0], shape[1], 3))
else:
raise NotImplementedError
model.compile(
optimizer=opt_dict[opt.lower()],
loss=sm.losses.bce_jaccard_loss,
metrics=['acc', sm.metrics.iou_score, sm.metrics.f1_score])
model.summary()
name_list = [
model_name, backbone, opt, 'init-training',
'none' if shape is None else str(shape[0]) + 'x' +
str(shape[1])
]
if self.shrink is None:
name_list.append('without-shrink')
new_name = '_'.join(name_list) + '.h5'
model.fit_generator(
generator=self.data_loader.generator(is_train=True,
shape=shape,
shrink=self.shrink),
steps_per_epoch=self.data_loader.train_steps,
validation_data=self.data_loader.generator(is_train=False,
shape=shape,
shrink=self.shrink),
validation_steps=self.data_loader.val_steps,
verbose=1,
initial_epoch=0,
epochs=300,
callbacks=[
keras.callbacks.TensorBoard('logs'),
keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
patience=7,
verbose=1),
keras.callbacks.EarlyStopping(monitor='val_loss',
patience=38,
verbose=1),
keras.callbacks.ModelCheckpoint(
monitor='val_loss',
verbose=1,
save_weights_only=False,
save_best_only=True,
filepath='saved_models/segmentation/' + new_name)
])
def create_model(self):
return sm.Linknet(self._backbone,
activation="softmax",
classes=self._data.get_n_classes(),
encoder_weights=self._encoder_weights,
input_shape=self._input_shape)
def Setup(self):
'''
User function: Setup all the parameters
Args:
None
Returns:
None
'''
preprocess_input = sm.get_preprocessing(
self.system_dict["params"]["backbone"])
# define network parameters
self.system_dict["local"]["n_classes"] = 1 if len(
self.system_dict["dataset"]["train"]
["classes_to_train"]) == 1 else (
len(self.system_dict["dataset"]["train"]["classes_to_train"]) +
1) # case for binary and multiclass segmentation
activation = 'sigmoid' if self.system_dict["local"][
"n_classes"] == 1 else 'softmax'
#create model
if (self.system_dict["params"]["model"] == "Unet"):
self.system_dict["local"]["model"] = sm.Unet(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
elif (self.system_dict["params"]["model"] == "FPN"):
self.system_dict["local"]["model"] = sm.FPN(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
elif (self.system_dict["params"]["model"] == "Linknet"):
self.system_dict["local"]["model"] = sm.Linknet(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
elif (self.system_dict["params"]["model"] == "PSPNet"):
self.system_dict["local"]["model"] = sm.PSPNet(
self.system_dict["params"]["backbone"],
classes=self.system_dict["local"]["n_classes"],
activation=activation)
# define optomizer
optim = keras.optimizers.Adam(self.system_dict["params"]["lr"])
# Segmentation models losses can be combined together by '+' and scaled by integer or float factor
dice_loss = sm.losses.DiceLoss()
focal_loss = sm.losses.BinaryFocalLoss() if self.system_dict["local"][
"n_classes"] == 1 else sm.losses.CategoricalFocalLoss()
total_loss = dice_loss + (1 * focal_loss)
# actulally total_loss can be imported directly from library, above example just show you how to manipulate with losses
# total_loss = sm.losses.binary_focal_dice_loss # or sm.losses.categorical_focal_dice_loss
metrics = [
sm.metrics.IOUScore(threshold=0.5),
sm.metrics.FScore(threshold=0.5)
]
# compile keras model with defined optimozer, loss and metrics
self.system_dict["local"]["model"].compile(optim, total_loss, metrics)
# Dataset for train images
train_dataset = Dataset(
self.system_dict["dataset"]["train"]["img_dir"],
self.system_dict["dataset"]["train"]["mask_dir"],
self.system_dict["dataset"]["train"]["classes_dict"],
classes_to_train=self.system_dict["dataset"]["train"]
["classes_to_train"],
augmentation=get_training_augmentation(),
preprocessing=get_preprocessing(preprocess_input),
)
if (self.system_dict["params"]["image_shape"][0] % 32 != 0):
self.system_dict["params"]["image_shape"][0] += (
32 - self.system_dict["params"]["image_shape"][0] % 32)
if (self.system_dict["params"]["image_shape"][1] % 32 != 0):
self.system_dict["params"]["image_shape"][1] += (
32 - self.system_dict["params"]["image_shape"][1] % 32)
# Dataset for validation images
if (self.system_dict["dataset"]["val"]["status"]):
valid_dataset = Dataset(
self.system_dict["dataset"]["val"]["img_dir"],
self.system_dict["dataset"]["val"]["mask_dir"],
self.system_dict["dataset"]["train"]["classes_dict"],
classes_to_train=self.system_dict["dataset"]["train"]
["classes_to_train"],
augmentation=get_validation_augmentation(
self.system_dict["params"]["image_shape"][0],
self.system_dict["params"]["image_shape"][1]),
preprocessing=get_preprocessing(preprocess_input),
)
else:
valid_dataset = Dataset(
self.system_dict["dataset"]["train"]["img_dir"],
self.system_dict["dataset"]["train"]["mask_dir"],
self.system_dict["dataset"]["train"]["classes_dict"],
classes_to_train=self.system_dict["dataset"]["train"]
["classes_to_train"],
augmentation=get_validation_augmentation(
self.system_dict["params"]["image_shape"][0],
self.system_dict["params"]["image_shape"][1]),
preprocessing=get_preprocessing(preprocess_input),
)
self.system_dict["local"]["train_dataloader"] = Dataloder(
train_dataset,
batch_size=self.system_dict["params"]["batch_size"],
shuffle=True)
self.system_dict["local"]["valid_dataloader"] = Dataloder(
valid_dataset, batch_size=1, shuffle=False)
input_shape=input_layer,
classes=config.nb_classes,
activation=config.activation,
encoder_weights=config.encoder_weights,
psp_dropout=config.dropout)
elif 'fpn' in config.network:
model = sm.FPN(backbone_name=config.BACKBONE,
input_shape=input_layer,
classes=config.nb_classes,
activation=config.activation,
encoder_weights=config.encoder_weights,
pyramid_dropout=config.dropout)
elif 'linknet' in config.network:
model = sm.Linknet(backbone_name=config.BACKBONE,
input_shape=input_layer,
classes=config.nb_classes,
activation=config.activation,
encoder_weights=config.encoder_weights)
elif 'deeplabv3plus' in config.network:
model = Deeplabv3(weights=config.encoder_weights,
input_shape=input_layer,
classes=config.nb_classes,
backbone=config.BACKBONE,
activation=config.activation)
else:
print("Error:")
print(model.summary())
print("Train by : {}_{}".format(config.network, config.BACKBONE))
#
