def simple(
img_path='/media/dengpingfan/leone/dpfan/gepeng/Dataset/3Dataset/img',
gt_path='/media/dengpingfan/leone/dpfan/gepeng/Dataset/3Dataset/gt',
batchSize=7,
target_size=(256, 256),
epoch=24,
lr=0.03,
steps_per_epoch=663,
model_save_path='./models/FPN/fpn_camouflage_baseline.h5'):
seed = 1
data_gen_args = dict(horizontal_flip=True, fill_mode='nearest')
img_datagen = ImageDataGenerator(**data_gen_args)
data_gen_args['rescale'] = 1. / 255
mask_datagen = ImageDataGenerator(**data_gen_args)
img_gen = img_datagen.flow_from_directory(img_path,
batch_size=batchSize,
target_size=target_size,
shuffle=True,
class_mode=None,
seed=seed)
mask_gen = mask_datagen.flow_from_directory(gt_path,
color_mode='grayscale',
batch_size=batchSize,
target_size=target_size,
shuffle=True,
class_mode=None,
seed=seed)
train_gen = zip(img_gen, mask_gen)
# model = Xnet(backbone_name='vgg16', encoder_weights='imagenet', decoder_block_type='transpose')
model = FPN(backbone_name='resnet50', classes=1, activation='sigmoid')
print(model.summary())
opt = optimizers.SGD(lr=lr, momentum=0.9, decay=0.0001)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['binary_accuracy'])
save_best = callbacks.ModelCheckpoint(filepath=model_save_path,
monitor='loss',
save_best_only=True,
verbose=1)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
patience=30,
verbose=2,
mode='min')
callbacks_list = [save_best, early_stopping]
model.fit_generator(train_gen,
steps_per_epoch=steps_per_epoch,
epochs=epoch,
verbose=1,
callbacks=callbacks_list)
class unet():
def __init__(self,
architecture,
backbone,
image_net,
single_labels,
colors,
address,
annotation_file,
image_folder,
annotation_folder,
bodyparts,
BATCH_SIZE=5,
train_flag=1,
annotation_assistance=0,
lr=0.001,
loss="Weighted Categorical_cross_entropy",
markersize=13):
self.lr_drop = 25
self.loss_function = loss
self.markerSize = int(markersize)
self.BATCH_SIZE = int(BATCH_SIZE) # the higher the better
self.annotation_file = os.path.join(address, annotation_file)
self.bodyparts = bodyparts
self.colors = colors
self.error = 0
self.single_labels = single_labels
self.architecture = architecture
self.backbone = backbone
self.image_net = image_net
if self.image_net == 'None':
self.image_net = None
self.annotation_assistance = annotation_assistance
self.learning_rate = float(lr)
self.train_flag = train_flag
self.num_bodyparts = len(self.bodyparts)
self.image_folder = image_folder
self.annotation_folder = annotation_folder
self.address = address
try:
file = open(self.address + os.sep + 'annotation_options.txt')
self.options = file.readlines()
except:
wx.MessageBox(
'Error in reading the annotation_options, please check the existence or choose'
' annotation_options\n '
'annotation_options reading error ', 'Error!',
wx.OK | wx.ICON_ERROR)
return
try:
configuration_file = self.address + os.sep + 'file_configuration.txt'
file = open(configuration_file)
self.pref = file.readlines()
except:
wx.MessageBox(
'Error in reading the configuration file, please check the address or existence'
'configuration file \n '
'configuration file_error', 'Error!', wx.OK | wx.ICON_ERROR)
self.error = 1
return
self.scorer = self.options[1][:-1]
self.number_videos = len(self.pref) - 2
for i in range(0, self.number_videos):
ind = self.pref[2 + i].rfind(os.sep)
if self.pref[2 + i][ind + 1:-1] in self.annotation_file:
self.name_video_list = self.pref[2 + i][ind + 1:-1]
break
try:
self.dataFrame = pd.read_pickle(self.annotation_file)
except:
wx.MessageBox(
'Annotations reading error\n '
'Annotation not found', 'Error!', wx.OK | wx.ICON_ERROR)
self.error = 1
return
self.annotated = np.where(
np.bitwise_and((np.isnan(self.dataFrame.iloc[:,
0].values) == False),
self.dataFrame.iloc[:, 0].values > 0) == True)[0]
if train_flag == 1:
self.train()
if self.error != 1:
self.test()
def weighted_categorical_crossentropy(self, weights):
weights = K.variable(weights)
def loss(y_true, y_pred):
# scale predictions so that the class probas of each sample sum to 1
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
# clip to prevent NaN's and Inf's
y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())
# calc
loss = y_true * K.log(y_pred) * weights
loss = -K.sum(loss, -1)
return loss
return loss
def train(self):
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
# config = tf.ConfigProto(gpu_options=gpu_options)
session_config = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True, per_process_gpu_memory_fraction=0.6))
session = tf.Session(config=session_config)
addresss = self.address
warnings.filterwarnings('ignore',
category=UserWarning,
module='skimage')
seed = 42
np.random.seed(10)
print('Getting and resizing train images and masks ... ')
sys.stdout.flush()
counter = 0
self.IMG_WIDTH = 288 # for faster computing on kaggle
self.IMG_HEIGHT = 288 # for faster computing on kaggle
counter = 0
files_original_name = list()
#self.num_bodyparts =1
if len(self.annotated) == 0:
wx.MessageBox(
'Did you save your annotation?\n '
'No annotation found in your file, please save and re-run',
'Error!', wx.OK | wx.ICON_ERROR)
self.error = 1
return
for i in range(0, len(self.annotated)):
files_original_name.append(self.dataFrame[
self.dataFrame.columns[0]]._stat_axis[self.annotated[i]][7:])
img = imread(self.image_folder + os.sep + files_original_name[0])
IMG_CHANNELS = len(np.shape(img))
self.IMG_CHANNELS = IMG_CHANNELS
self.file_name_for_prediction_confidence = files_original_name
X_train = np.zeros((len(
self.annotated), self.IMG_HEIGHT, self.IMG_WIDTH, IMG_CHANNELS),
dtype=np.uint8)
Y_train = np.zeros((len(self.annotated), self.IMG_HEIGHT,
self.IMG_WIDTH, self.num_bodyparts + 1),
dtype=np.int)
New_train = np.zeros(
(len(self.annotated), self.IMG_HEIGHT, self.IMG_WIDTH),
dtype=np.int)
for l in range(0, len(self.annotated)):
img = imread(self.image_folder + os.sep + files_original_name[l])
# mask_ = np.zeros((np.shape(img)[0],np.shape(img)[1],self.num_bodyparts))
mask_ = np.zeros(
(np.shape(img)[0], np.shape(img)[1], self.num_bodyparts))
img = resize(img, (self.IMG_HEIGHT, self.IMG_WIDTH),
mode='constant',
preserve_range=True)
X_train[counter] = img
for j in range(0, self.num_bodyparts):
mask_single_label = np.zeros((mask_.shape[0], mask_.shape[1]))
#if annotation was assisted, x is negative
points = np.asarray([
self.dataFrame[self.dataFrame.columns[j * 2]].values[
self.annotated[l]],
self.dataFrame[self.dataFrame.columns[j * 2 + 1]].values[
self.annotated[l]]
],
dtype=float)
points = np.abs(points)
if np.isnan(points[0]):
continue
cv2.circle(mask_single_label, (int(round(
(points[0] * (2**4)))), int(round(points[1] * (2**4)))),
int(round(self.markerSize * (2**4))),
(255, 255, 255),
thickness=-1,
shift=4)
mask_[:, :, j] = mask_single_label
mask_ = resize(mask_, (self.IMG_HEIGHT, self.IMG_WIDTH),
mode='constant',
preserve_range=True)
a, mask_ = cv2.threshold(mask_, 200, 255, cv2.THRESH_BINARY)
mask_ = mask_ / 255.0
if len(np.shape(mask_)) == 2:
mask_new = np.zeros(
(np.shape(mask_)[0], np.shape(mask_)[1], 1))
mask_new[:, :, 0] = mask_
mask_ = mask_new
for j in range(0, self.num_bodyparts):
New_train[counter] = New_train[counter] + mask_[:, :,
j] * (j + 1)
# alternative method to build the ground truth
# temp = temp + 1
# temp[temp == 0] = 1
# temp[temp > 1] = 0
# Y_train[counter, :, :,1:] = mask_
# Y_train[counter,:,:,0] = temp
counter += 1
#
try:
Y_train = tf.keras.utils.to_categorical(
New_train, num_classes=self.num_bodyparts + 1)
except:
wx.MessageBox(
'two or more labels are overlapping!\n '
'Check annotation or re-perform the labeling operation',
'Error!', wx.OK | wx.ICON_ERROR)
self.error = 1
return
counter = 0
from segmentation_models import get_preprocessing
self.processer = get_preprocessing(self.backbone)
X_train = self.processer(X_train)
print('Done!')
from tensorflow.keras.preprocessing import image
# Creating the training Image and Mask generator
image_datagen = image.ImageDataGenerator(shear_range=0.5,
rotation_range=50,
zoom_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
fill_mode='reflect')
mask_datagen = image.ImageDataGenerator(shear_range=0.5,
rotation_range=50,
zoom_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
fill_mode='reflect')
# Keep the same seed for image and mask generators so they fit together
image_datagen.fit(X_train[:int(X_train.shape[0] * 0.9)],
augment=True,
seed=seed)
mask_datagen.fit(Y_train[:int(Y_train.shape[0] * 0.9)],
augment=True,
seed=seed)
x = image_datagen.flow(X_train[:int(X_train.shape[0] * 0.9)],
batch_size=self.BATCH_SIZE,
shuffle=True,
seed=seed)
y = mask_datagen.flow(Y_train[:int(Y_train.shape[0] * 0.9)],
batch_size=self.BATCH_SIZE,
shuffle=True,
seed=seed)
# Creating the validation Image and Mask generator
image_datagen_val = image.ImageDataGenerator()
mask_datagen_val = image.ImageDataGenerator()
image_datagen_val.fit(X_train[int(X_train.shape[0] * 0.9):],
augment=True,
seed=seed)
mask_datagen_val.fit(Y_train[int(Y_train.shape[0] * 0.9):],
augment=True,
seed=seed)
x_val = image_datagen_val.flow(X_train[int(X_train.shape[0] * 0.9):],
batch_size=self.BATCH_SIZE,
shuffle=True,
seed=seed)
y_val = mask_datagen_val.flow(Y_train[int(Y_train.shape[0] * 0.9):],
batch_size=self.BATCH_SIZE,
shuffle=True,
seed=seed)
train_generator = zip(x, y)
val_generator = zip(x_val, y_val)
from segmentation_models import Unet, PSPNet, Linknet, FPN
from segmentation_models.losses import CategoricalFocalLoss
from segmentation_models.utils import set_trainable
import segmentation_models
from tensorflow.keras.optimizers import RMSprop, SGD
if self.architecture == 'Linknet':
self.model = Linknet(self.backbone,
classes=self.num_bodyparts + 1,
activation='softmax',
encoder_weights=self.image_net,
input_shape=(self.IMG_WIDTH, self.IMG_HEIGHT,
self.IMG_CHANNELS))
elif self.architecture == 'unet':
self.model = Unet(self.backbone,
classes=self.num_bodyparts + 1,
activation='softmax',
encoder_weights=self.image_net,
input_shape=(self.IMG_WIDTH, self.IMG_HEIGHT,
self.IMG_CHANNELS))
elif self.architecture == 'PSPnet':
self.model = PSPNet(self.backbone,
classes=self.num_bodyparts + 1,
activation='softmax',
encoder_weights=self.image_net,
input_shape=(self.IMG_WIDTH, self.IMG_HEIGHT,
self.IMG_CHANNELS))
elif self.architecture == 'FPN':
self.model = FPN(self.backbone,
classes=self.num_bodyparts + 1,
activation='softmax',
encoder_weights=self.image_net,
input_shape=(self.IMG_WIDTH, self.IMG_HEIGHT,
self.IMG_CHANNELS))
weights = np.zeros((1, self.num_bodyparts + 1), dtype=float)
weight = 1.0 / self.num_bodyparts
num_zeros = 1
# while (weight * 100 < 1):
# weight = weight * 100
# num_zeros += 1
#
# weight = int(weight * 100) / np.power(100, num_zeros)
weights[0, 1:] = weight
weights[0, 0] = 0.01 * len(self.bodyparts)
while weights[0, 0] > weights[0, 1]:
weights[0, 0] = weights[0, 0] / 10
num_zeros += 1
for i in range(1, len(self.bodyparts) + 1):
weights[0, i] = weights[0, i] - 10**-(num_zeros + 1)
if self.loss_function == "Weighted Categorical_cross_entropy":
loss = self.weighted_categorical_crossentropy(weights)
else:
loss = segmentation_models.losses.DiceLoss(class_weights=weights)
metric = segmentation_models.metrics.IOUScore(class_weights=weights,
per_image=True)
self.model.compile(optimizer=RMSprop(lr=self.learning_rate),
loss=loss,
metrics=[metric])
earlystopper = EarlyStopping(patience=6, verbose=1)
#
checkpointer = ModelCheckpoint(os.path.join(self.address, 'Unet.h5'),
verbose=1,
save_best_only=True)
reduce_lr = keras.callbacks.LearningRateScheduler(self.lr_scheduler)
#
# model.fit_generator(train_generator, validation_data=val_generator, validation_steps=10, steps_per_epoch=50,
# epochs=2, callbacks=[earlystopper, checkpointer],verbose=1)
# model.load_weights(self.address + 'Temp_weights.h5')
# set_trainable(model)
#
self.model.fit_generator(
train_generator,
validation_data=val_generator,
steps_per_epoch=20,
validation_steps=5,
epochs=100,
callbacks=[earlystopper, checkpointer, reduce_lr],
verbose=1)
def test(self):
import segmentation_models
from segmentation_models import Unet
session_config = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True, per_process_gpu_memory_fraction=0.6))
session = tf.Session(config=session_config)
weights = np.zeros((1, self.num_bodyparts + 1), dtype=float)
weight = 1.0 / self.num_bodyparts
num_zeros = 1
# while (weight * 100 < 1):
# weight = weight * 100
# num_zeros += 1
#
# weight = int(weight * 100) / np.power(100, num_zeros)
weights[0, 1:] = weight
#weights[0, 0] = 1 - np.sum(weights[0, 1:])
weights[0, 0] = 0.01 * len(self.bodyparts)
while weights[0, 0] > weights[0, 1]:
weights[0, 0] = weights[0, 0] / 10
num_zeros += 1
for i in range(1, len(weights)):
weights[i] = weights[i] - np.power(10, num_zeros + 1)
weights = weights[0]
if self.loss_function == "Weighted Categorical_cross_entropy":
loss = self.weighted_categorical_crossentropy(weights)
else:
loss = segmentation_models.losses.DiceLoss(class_weights=weights)
metric = segmentation_models.metrics.IOUScore(class_weights=weights,
per_image=True)
model = load_model(os.path.join(self.address, 'Unet.h5'),
custom_objects={
'loss': loss,
'dice_loss':
segmentation_models.losses.DiceLoss,
'iou_score': metric
})
OUTPUT = os.path.join(self.address,
self.name_video_list + '_prediction')
try:
os.mkdir(OUTPUT)
except:
pass
files = os.listdir(self.image_folder)
files_original_name = list()
self.annotated = np.where(
np.bitwise_and((np.isnan(self.dataFrame.iloc[:,
0].values) == False),
self.dataFrame.iloc[:, 0].values > 0) == True)[0]
# if len(self.annotated)==0:
# wx.MessageBox('Did you save your annotation?\n '
# 'No annotation found in your file, please save and re-run'
# , 'Error!', wx.OK | wx.ICON_ERROR)
# self.error = 1
# return
for i in range(0, len(self.annotated)):
files_original_name.append(files[self.annotated[i]])
#files = np.unique(files_original_name)
img = imread(self.image_folder + os.sep + files[0])
IMG_CHANNELS = len(np.shape(img))
self.IMG_CHANNELS = IMG_CHANNELS
self.IMG_WIDTH = 288
self.IMG_HEIGHT = 288
counter = 0
if self.annotation_assistance == 0:
self.X_test = np.zeros(
(len(files) - len(self.annotated), self.IMG_HEIGHT,
self.IMG_WIDTH, self.IMG_CHANNELS),
dtype=np.uint8)
self.testing_index = list()
for l in range(0, len(files)):
if l not in self.annotated:
self.testing_index.append(l)
img = imread(self.image_folder + os.sep +
files[l])[:, :, :self.IMG_CHANNELS]
img = resize(img, (self.IMG_HEIGHT, self.IMG_WIDTH),
mode='constant',
preserve_range=True)
self.X_test[counter] = img
counter += 1
self.testing_index = np.asarray(self.testing_index)
else:
if os.path.isfile(
os.path.join(
os.path.dirname(self.annotation_file),
self.name_video_list + '_index_annotation.txt')):
self.pref_ann = open(
os.path.join(
os.path.dirname(self.annotation_file),
self.name_video_list + '_index_annotation_auto.txt'),
'r')
temporary = self.pref_ann.readlines()
for i in range(0, len(temporary)):
temporary[i] = temporary[i][:-1]
temporary = np.asarray(temporary)
self.frame_selected_for_annotation = temporary.astype(int)
self.frame_selected_for_annotation = np.sort(
self.frame_selected_for_annotation)
if len(self.frame_selected_for_annotation) == 0:
filename = self.ask()
if filename == "":
return
else:
num_frame_automatic = int(filename)
if num_frame_automatic > len(files):
wx.MessageBox(
'Error! Number of frames must be lower than the total number of frames\n '
'Input error', 'Error!', wx.OK | wx.ICON_ERROR)
my_list = list(
range(0, len(files))
) # list of integers from 1 to end # adjust this boundaries to fit your needs
random.shuffle(my_list)
self.frame_selected_for_annotation = my_list[
0:num_frame_automatic]
output = open(
os.path.join(
os.path.dirname(self.annotation_file),
self.name_video_list +
'_index_annotation_auto.txt'), 'w')
for fra in self.frame_selected_for_annotation:
output.writelines(str(fra))
output.writelines('\n')
output.close()
self.X_test = np.zeros(
(len(self.frame_selected_for_annotation), self.IMG_HEIGHT,
self.IMG_WIDTH, self.IMG_CHANNELS),
dtype=np.uint8)
for l in range(0, len(files)):
if l in self.frame_selected_for_annotation:
img = imread(self.image_folder + os.sep +
files[l])[:, :, :self.IMG_CHANNELS]
img = resize(img, (self.IMG_HEIGHT, self.IMG_WIDTH),
mode='constant',
preserve_range=True)
self.X_test[counter] = img
counter += 1
#X_test = X_test / 255.0
seed = 42
# Creating the training Image and Mask generator
# image_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)
# # Keep the same seed for image and mask generators so they fit together
# image_datagen.fit(self.X_test, augment=False,seed=seed)
# x = image_datagen.flow(self.X_test, batch_size=1, shuffle=False, seed=seed)
from segmentation_models import get_preprocessing
self.processer = get_preprocessing(self.backbone)
self.X_test = self.processer(self.X_test)
x = self.X_test
preds_test = model.predict(x, verbose=0)
# Threshold predictions
# preds_train_t = (preds_train > 0.5).astype(np.uint8)
# K.clear_session()
# preds_val_t = (preds_val > 0.5).astype(np.uint8)
#preds_test_t = (preds_test > 0.5).astype(np.uint8)
preds_test_t = preds_test
# Create list of upsampled test masks
preds_test_upsampled = []
address_single_labels = os.path.join(
self.address, self.name_video_list + '_Single_labels')
self.annotated = np.where(
np.bitwise_and((np.isnan(self.dataFrame.iloc[:,
0].values) == False),
self.dataFrame.iloc[:, 0].values > 0) == True)[0]
try:
os.mkdir(address_single_labels)
except:
pass
if self.single_labels == 'Yes':
if self.annotation_assistance == 1:
self.testing_index = self.frame_selected_for_annotation
# PREDICT AND SAVE SINGLE LABELS IMAGES
for i in range(0, len(preds_test)):
img = imread(self.image_folder + os.sep +
files[self.testing_index[i]])
sizes_test = np.shape(img)[:-1]
for j in range(0, self.num_bodyparts + 1):
preds_test_upsampled = resize(np.squeeze(
preds_test_t[i, :, :,
j]), (sizes_test[0], sizes_test[1]),
mode='constant',
preserve_range=True)
preds_test_upsampled = (preds_test_upsampled *
255).astype(int)
cv2.imwrite(
address_single_labels + os.sep + ("{:02d}".format(j)) +
files[self.testing_index[i]], preds_test_upsampled)
#let us create a dataframe to store prediction confidence
imlist = os.listdir(self.image_folder)
self.index = np.sort(imlist)
a = np.empty((
len(self.index),
3,
))
self.dataFrame3 = None
a[:] = np.nan
self.scorer = 'user'
for bodypart in self.bodyparts:
index = pd.MultiIndex.from_product(
[[self.scorer], [bodypart], ['x', 'y', 'confidence']],
names=['scorer', 'bodyparts', 'coords'])
frame = pd.DataFrame(a, columns=index, index=imlist)
self.dataFrame3 = pd.concat([self.dataFrame3, frame], axis=1)
num_columns = len(self.dataFrame3.columns)
# if this is a regular testing, all of the images (but the annotated ones) have to be analyzed
if self.annotation_assistance == 0:
out = skvideo.io.FFmpegWriter(os.path.join(
OUTPUT, self.name_video_list + '.avi'),
outputdict={'-b': '300000000'})
for i in range(0, len(preds_test)):
results = np.zeros((self.num_bodyparts * 2))
results_plus_conf = np.zeros((self.num_bodyparts * 3))
#here to update dataframe with annotations
img = imread(self.image_folder + os.sep +
files[self.testing_index[i]])
sizes_test = np.shape(img)[:-1]
for j in range(0, self.num_bodyparts + 1):
if j == 0: continue
preds_test_upsampled = resize(np.squeeze(
preds_test_t[i, :, :,
j]), (sizes_test[0], sizes_test[1]),
mode='constant',
preserve_range=True)
preds_test_upsampled = (preds_test_upsampled * 255).astype(
np.uint8)
results[(j - 1) * 2:(j - 1) * 2 +
2] = self.prediction_to_annotation(
preds_test_upsampled)
results_plus_conf[(j - 1) * 3:(j - 1) * 3 +
3] = self.compute_confidence(
preds_test_upsampled)
self.dataFrame[self.dataFrame.columns[(j - 1) * 2]].values[
self.testing_index[i]] = -results[(j - 1) * 2]
self.dataFrame[self.dataFrame.columns[
(j - 1) * 2 +
1]].values[self.testing_index[i]] = results[(j - 1) * 2
+ 1]
self.dataFrame3[self.dataFrame3.columns[
(j - 1) *
3]].values[self.testing_index[i]] = -results_plus_conf[
(j - 1) * 3]
self.dataFrame3[self.dataFrame3.columns[
(j - 1) * 3 +
1]].values[self.testing_index[i]] = results_plus_conf[
(j - 1) * 3 + 1]
self.dataFrame3[self.dataFrame3.columns[
(j - 1) * 3 +
2]].values[self.testing_index[i]] = results_plus_conf[
(j - 1) * 3 + 2]
self.plot_annotation(img, results,
files[self.testing_index[i]], OUTPUT, out)
else:
#if annotation assistance is requested, we only want to annotate the random frames extracted by the user
for i in range(0, len(preds_test)):
results = np.zeros((self.num_bodyparts * 2))
results_plus_conf = np.zeros((self.num_bodyparts * 3))
# here to update dataframe with annotations
img = imread(self.image_folder +
files[self.frame_selected_for_annotation[i]])
sizes_test = np.shape(img)[:-1]
for j in range(0, self.num_bodyparts + 1):
if j == 0: continue
preds_test_upsampled = resize(np.squeeze(
preds_test_t[i, :, :,
j]), (sizes_test[0], sizes_test[1]),
mode='constant',
preserve_range=True)
preds_test_upsampled = (preds_test_upsampled * 255).astype(
np.uint8)
results[(j - 1) * 2:(j - 1) * 2 +
2] = self.prediction_to_annotation(
preds_test_upsampled)
results_plus_conf[(j - 1) * 3:(j - 1) * 3 +
3] = self.compute_confidence(
preds_test_upsampled)
self.plot_annotation(
img, results,
files[self.frame_selected_for_annotation[i]], OUTPUT,
out)
self.dataFrame[self.dataFrame.columns[(j - 1) * 2]].values[
self.frame_selected_for_annotation[i]] = -results[
(j - 1) * 2]
self.dataFrame[self.dataFrame.columns[
(j - 1) * 2 + 1]].values[
self.frame_selected_for_annotation[i]] = results[
(j - 1) * 2 + 1]
self.dataFrame3[self.dataFrame3.columns[
(j - 1) *
3]].values[self.frame_selected_for_annotation[
i]] = -results_plus_conf[(j - 1) * 3]
self.dataFrame3[self.dataFrame3.columns[
(j - 1) * 3 +
1]].values[self.frame_selected_for_annotation[
i]] = results_plus_conf[(j - 1) * 3 + 1]
self.dataFrame3[self.dataFrame3.columns[
(j - 1) * 3 +
2]].values[self.frame_selected_for_annotation[
i]] = results_plus_conf[(j - 1) * 3 + 2]
self.dataFrame.to_pickle(self.annotation_file)
self.dataFrame.to_csv(os.path.join(self.annotation_file + ".csv"))
out.close()
try:
self.dataFrame3.to_csv(
os.path.join(self.address,
self.annotation_file + "_with_confidence.csv"))
except:
wx.MessageBox(
'Error in writing the results file'
'file not accessible\n '
'Error in writing', 'Error!', wx.OK | wx.ICON_ERROR)
def lr_scheduler(self, epoch):
return self.learning_rate * (0.5**(epoch // self.lr_drop))
def prediction_to_annotation(self, annotation):
# compute_corresponding_annotation_point
# annotation = cv2.cvtColor(annotation, cv2.COLOR_BGR2GRAY)
thresh, annotation = cv2.threshold(annotation, 150, 255,
cv2.THRESH_BINARY)
contour, hierarchy = cv2.findContours(annotation, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
if contour != []:
max_area = 0
i_max = []
for cc in contour:
mom = cv2.moments(cc)
area = mom['m00']
if area > max_area:
max_area = area
i_max = cc
if len(i_max) == 0:
xc = 1
yc = -1
else:
center = cv2.moments(i_max)
xc = center['m10'] / center['m00']
yc = center['m01'] / center['m00']
else:
#maybe one joint is missing, but the other were correctly identified
# self.dataFrame[self.dataFrame.columns[(i - 1) * 2]].values[j] = -1
# self.dataFrame[self.dataFrame.columns[(i - 1) * 2 + 1]].values[j] = -1
xc = 1
yc = -1
return xc, yc
# self.statusbar.SetStatusText("File saved")
# MainFrame.updateZoomPan(self)
#
# copyfile(os.path.join(self.filename + ".csv"), os.path.join(self.filename + "_MANUAL.csv"))
# self.dataFrame.to_csv(os.path.join(self.filename + ".csv"))
# copyfile(os.path.join(self.filename + ".csv"), os.path.join(self.filename + "_MANUAL.csv"))
# self.dataFrame.to_pickle(self.filename) # where to save it, usually as a .pkl
# wx.PyCommandEvent(wx.EVT_BUTTON.typeId, self.load.GetId())
def compute_confidence(self, annotation):
# compute_corresponding_annotation_point
# annotation = cv2.cvtColor(annotation, cv2.COLOR_BGR2GRAY)
confidence_image = copy.copy(annotation)
raw_image = copy.copy(annotation)
confidence_image[np.where(confidence_image != 0)] = 1
mask = np.zeros_like(confidence_image)
thresh, annotation = cv2.threshold(annotation, 150, 255,
cv2.THRESH_BINARY)
contour, hierarchy = cv2.findContours(annotation, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
if contour != []:
max_area = 0
i_max = []
for cc in contour:
mom = cv2.moments(cc)
area = mom['m00']
if area > max_area:
max_area = area
i_max = cc
if len(i_max) == 0:
xc = 1
yc = -1
confidence = 0
else:
center = cv2.moments(i_max)
cv2.drawContours(mask, [i_max], -1, (255, 255, 255), -1)
mask[np.where(mask != 0)] = 1
mean_values = np.multiply(confidence_image, mask)
confidence = np.mean(raw_image[np.where(mean_values != 0)])
confidence = confidence / 255.0
xc = center['m10'] / center['m00']
yc = center['m01'] / center['m00']
else:
# maybe one joint is missing, but the other were correctly identified
# self.dataFrame[self.dataFrame.columns[(i - 1) * 2]].values[j] = -1
# self.dataFrame[self.dataFrame.columns[(i - 1) * 2 + 1]].values[j] = -1
xc = 1
yc = -1
confidence = 0
return xc, yc, confidence
def plot_annotation(self, image, points, name, OUTPUT, out):
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
for i in range(0, len(self.bodyparts)):
if not np.isnan(points[i * 2] and points[i * 2 + 1]):
cv2.circle(image, (int(round(
(points[i * 2] *
(2**4)))), int(round(points[i * 2 + 1] * (2**4)))),
int(round(self.markerSize * (2**4))),
self.colors[i] * 255,
thickness=-1,
shift=4)
cv2.imwrite(os.path.join(OUTPUT, name), image)
out.writeFrame(cv2.cvtColor(image, cv2.COLOR_RGB2BGR))
def ask(
self,
parent=None,
message='Attention:\n you did not input the number of frames to automatically detect in previous interface \n Please, fill the following textbox with such number or cancel'
):
default_value = ""
dlg = wx.TextEntryDialog(parent, message, value=default_value)
dlg.ShowModal()
result = dlg.GetValue()
dlg.Destroy()
return result
def train_generatorh5(params):
from hitif_losses import dice_coef_loss_bce
from hitif_losses import double_head_loss
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
# Prepare for splitting the training set
imgs_ind = np.arange(number_of_imgs)
np.random.shuffle(imgs_ind)
# Split 80-20
train_last_id = int(number_of_imgs * 0.80)
# Generators
training_generator = DataGeneratorH5(
source_target_list_IDs=imgs_ind[0:train_last_id].copy(), **params)
validation_generator = DataGeneratorH5(
source_target_list_IDs=imgs_ind[train_last_id:number_of_imgs].copy(),
**params)
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
layers = get_feature_layers(backbone_name, 4)
if backbone_type == 'FPN':
model = FPN(input_shape=(None, None, num_channels),
classes=num_mask_channels,
encoder_weights=encoder_weights,
encoder_freeze=freezeFlag,
backbone_name=backbone_name,
activation=act_fcn,
encoder_features=layers)
elif backbone_type == 'Unet':
model = Unet(input_shape=(None, None, num_channels),
classes=num_mask_channels,
encoder_weights=encoder_weights,
encoder_freeze=freezeFlag,
backbone_name=backbone_name,
activation=act_fcn,
encoder_features=layers)
#model.summary()
#model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy', metrics=['binary_crossentropy','mean_squared_error',dice_coef, dice_coef_batch, dice_coef_loss_bce,focal_loss()])
#model.compile(optimizer=Adam(lr=1e-3), loss=dice_coef_loss_bce, metrics=['binary_crossentropy','mean_squared_error',dice_coef, dice_coef_batch,focal_loss()])
#model.compile(optimizer=Adam(lr=1e-3), loss=loss_fcn, metrics=['binary_crossentropy','mean_squared_error',dice_coef, dice_coef_batch,focal_loss()])
if loss_fcn == 'dice_coef_loss_bce':
model.compile(optimizer=Adam(lr=1e-3), loss=dice_coef_loss_bce)
elif loss_fcn == 'double_head_loss':
model.compile(optimizer=Adam(lr=1e-3), loss=double_head_loss)
else:
model.compile(optimizer=Adam(lr=1e-3), loss=loss_fcn)
# Loading previous weights for restarting
if oldmodelwtsfname is not None:
if os.path.isfile(oldmodelwtsfname) and reloadFlag:
print('-' * 30)
print('Loading previous weights ...')
weights = np.load(oldmodelwtsfname, allow_pickle=True)
model.set_weights(weights)
#model.load_weights(oldmodelwtsfname)
checkpoint_path = get_checkpoint_path(log_dir_name)
print("checkpoint_path:", checkpoint_path)
model_checkpoint = ModelCheckpoint(checkpoint_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=True)
custom_checkpoint = Checkpoints(checkpoint_path,
monitor='val_loss',
verbose=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=25,
min_lr=1e-6,
verbose=1,
restore_best_weights=True)
model_es = EarlyStopping(monitor='val_loss',
min_delta=1e-7,
patience=15,
verbose=1,
mode='auto')
csv_logger = CSVLogger(csvfname, append=True)
#my_callbacks = [reduce_lr, model_es, csv_logger]
#my_callbacks = [model_checkpoint, reduce_lr, model_es, csv_logger]
my_callbacks = [custom_checkpoint, reduce_lr, model_es, csv_logger]
print('-' * 30)
print('Fitting model encoder freeze...')
print('-' * 30)
if freezeFlag and num_coldstart_epochs > 0:
model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=True,
workers=num_gen_workers,
epochs=num_coldstart_epochs,
callbacks=my_callbacks,
verbose=2)
# release all layers for training
set_trainable(model) # set all layers trainable and recompile model
#model.summary()
print('-' * 30)
print('Fitting full model...')
print('-' * 30)
## Retrain after the cold-start
model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=True,
workers=num_gen_workers,
epochs=num_finetuning_epochs,
callbacks=my_callbacks,
verbose=2)
## <<FIXME>>: GZ will work on it.
# Find the last best epoch model weights and then symlink it to the modelwtsfname
# Note that the symlink will have issues on NON-Linux OS so it is better to copy.
'''
save_name = 'weights/' + args.backbone + '.h5'
callbacks_list = [
ModelCheckpoint(save_name,
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
save_weights_only=True),
ReduceLROnPlateau(monitor='loss', factor=0.2, patience=2, min_lr=1e-5)
]
model.compile(optimizer=Adam(1e-4),
loss=custom_loss,
metrics=[dice_coef, jaccard_coef, mean_iou])
history = model.fit_generator(generator,
steps_per_epoch=3000,
epochs=10,
verbose=1,
callbacks=callbacks_list)
model_json = model.to_json()
json_file = open('models/' + args.backbone + '.json', 'w')
json_file.write(model_json)
json_file.close()
print('Model saved!')
K.clear_session()
print('Cache cleared')
#model.compile('Adam', loss=cce_jaccard_loss, metrics=[jaccard_score])
model.summary()
#from keras.utils import plot_model
#plot_model(model, to_file='model.png')
modelCheckpoint = keras.callbacks.ModelCheckpoint(filepath=model_checkpoint_prefix+'_weights.{epoch:02d}-{val_loss:.4f}.hdf5',
monitor='val_loss',
verbose=0, save_best_only=False, save_weights_only=False,
mode='auto', period=1)
reduceLROnPlateau = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=7, verbose=1,
mode='auto', min_delta=0.001, cooldown=0, min_lr=10e-7)
model.fit_generator(generator=train_generator, steps_per_epoch=None, epochs=20, verbose=1,
callbacks=[reduceLROnPlateau, modelCheckpoint],
validation_data=val_generator, validation_steps=None, class_weight=None, max_queue_size=10,
workers=12, use_multiprocessing=True, shuffle=True, initial_epoch=0)
os._exit(0)
from segmentation_models import Unet
from segmentation_models.utils import set_trainable
'''
Some times, it is useful to train only randomly initialized decoder in order not to damage weights of properly trained encoder with huge gradients during first steps of training.
In this case, all you need is just pass freeze_encoder = True argument while initializing the model.
'''
model = Unet(backbone_name='resnet34', classes=len(class_ids), encoder_weights='imagenet', freeze_encoder=True)
model.compile('Adam', loss=cce_jaccard_loss, metrics=[jaccard_score])