def do_predict_dir(dicts_models, all_files, num_class=30, gpu_num=1):
if isinstance(all_files, str): # csv file
all_files, all_labels = my_data.get_images_labels(
filename_csv_or_pd=all_files)
batch_size_test = 64 # 虽然可以128，但是my_images_generator 将list转换np 耗费CPU
# 不是写死几个模型，提高灵活性
prob_list = [] ##每一个模型的概率
for i, model in enumerate(dicts_models):
if ('model' not in model1)(model['model'] is None):
print('prepare to load model:', model['model_file'])
model1 = keras.models.load_model(model['model_file'],
compile=False)
print('load model:', model['model_file'], ' complete')
if gpu_num > 1:
print('convert Multi-GPU model:', model['model_file'])
model1 = keras.utils.multi_gpu_model(model1, gpus=gpu_num)
print('convert Multi-GPU model:', model['model_file'],
' complete')
else:
model1 = model['model']
prob_list.append(np.empty((0, num_class)))
j = 0 # 样本数可能很多，每计算100个，一个数出提示
image_size = model['image_size']
for x in my_images_generator.my_Generator_test(
all_files,
image_shape=(image_size, image_size, 3),
batch_size=batch_size_test):
probabilities = model1.predict_on_batch(x)
if prob_list[i].size == 0:
prob_list[i] = probabilities
else:
prob_list[i] = np.vstack((prob_list[i], probabilities))
j += 1
print('batch:', j)
sum_models_weights = 0
for i, prob1 in enumerate(prob_list):
if 'model_weight' not in dicts_models[i]:
model_weight = 1
else:
model_weight = dicts_models[i]['model_weight']
sum_models_weights += model_weight
if i == 0:
prob_total = prob1 * model_weight
else:
prob_total += prob1 * model_weight
prob_total /= sum_models_weights
return prob_total
def compute_probs(models, file_csv, batch_size=32, cuda_visible_devices='0'):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
all_files, all_labels = my_data.get_images_labels(file_csv)
#region using every model to predict every image, generate list_probs
list_probs = []
for i, model in enumerate(models):
if ('model' not in model) or (model['model'] is None):
print('prepare to load model:', model['model_file'])
model1 = keras.models.load_model(model['model_file'],
compile=False)
print('load model:', model['model_file'], ' complete')
else:
model1 = model['model']
batch_no = 0 # batch number
tmp_probs = None
image_size = model['image_size']
generator_test = my_images_generator_2d.My_images_generator_2d_test(
all_files,
batch_size=batch_size,
image_shape=(image_size, image_size, 3))
for x in generator_test.gen():
probabilities = model1.predict_on_batch(x)
if tmp_probs is None:
tmp_probs = probabilities
else:
tmp_probs = np.vstack((tmp_probs, probabilities))
batch_no += 1
print('batch:', batch_no)
list_probs.append(tmp_probs)
#endregion
#region weighted average every model's result, generate list_probs_weighted
list_probs_weighted = None
model_weight_sum = 0
for i, probs1 in enumerate(list_probs):
model_weight = models[i]['model_weight']
model_weight_sum += model_weight
if i == 0:
list_probs_weighted = probs1 * model_weight
else:
list_probs_weighted = list_probs_weighted + probs1 * model_weight
list_probs_weighted = list_probs_weighted / model_weight_sum
#endregion
# import keras.backend as K
# K.clear_session() # release GPU memory
return list_probs, list_probs_weighted
def save_multi_label_csv(file_csv, list_probs, csv_results, list_threshold):
num_classes = len(list_threshold)
all_files, all_labels = my_data.get_images_labels(file_csv)
if csv_results != '':
if os.path.exists(csv_results):
os.remove(csv_results)
with open(csv_results, 'w', newline='') as csvfile:
csv_writer = csv.writer(csvfile,
quotechar='"',
quoting=csv.QUOTE_ALL,
delimiter=',')
list_title = ['images', 'labels', 'bigclasses']
for i in range(num_classes):
list_title.append('class' + str(i))
csv_writer.writerow(list_title)
for i in range(len(all_files)):
file, label = all_files[i], all_labels[i]
probs1 = list_probs[i]
# 0_1_1
pred_classes = ''
for j, prob1 in enumerate(probs1):
if prob1 > list_threshold[j]:
pred_classes += '1_'
else:
pred_classes += '0_'
pred_classes = pred_classes[:-1]
# 4_8_28
'''
pred_classes = '_'
for class_i in range(num_classes):
if probs1[class_i] > list_threshold[class_i]:
pred_classes = pred_classes + str(class_i) + str('_')
'''
list_row = [file, label, pred_classes]
for k in range(num_classes):
list_row.append(round(probs1[k], 3))
csv_writer.writerow(list_row)
print('csv file ok')
'model_weight': 1
}
dicts_models.append(dict_model1)
dict_model1 = {
'model_file': os.path.join(model_dir, 'Xception-004-0.984.hdf5'),
'input_shape': (299, 299, 3),
'model_weight': 1
}
dicts_models.append(dict_model1)
filename_csv = os.path.join(dir_dest, 'LaserSpot_predict_dir.csv')
if GEN_CSV:
os.makedirs(os.path.dirname(filename_csv), exist_ok=True)
write_csv_dir_nolabel(filename_csv, dir_preprocess)
df = pd.read_csv(filename_csv)
all_files, all_labels = get_images_labels(filename_csv_or_pd=df)
prob_total, y_pred_total, prob_list, pred_list = \
do_predict(dicts_models, filename_csv, argmax=True)
import pickle
os.makedirs(os.path.dirname(pkl_prob), exist_ok=True)
with open(pkl_prob, 'wb') as file:
pickle.dump(prob_total, file)
if COMPUTE_DIR_FILES:
op_files_multiclass(filename_csv,
prob_total,
dir_preprocess=dir_preprocess,
dir_dest=dir_dest,
def train_task(filename_csv_train, filename_csv_valid, epoch_fine_tuning=20):
str_subclass_no = '0.3'
TRAIN_TYPE = 'DLP_SubClass' + str_subclass_no
IMG_AUG_ROTATE_MODE = 1 #1:do flip,roate, 2:do flip 3: only translate_percent
MODEL_SAVE_DIR = '/tmp2/models_subclass/' + TRAIN_TYPE
#region read csv set weight_class_start, split train validation set
df = pd.read_csv(filename_csv_train)
NUM_CLASSES = df['labels'].nunique(dropna=True)
# len(df.loc[(df['labels'] == 0)])
weight_class_start = np.array([1, 9.5]) #23648, 2020
weight_class_end = np.array([1, 9.5])
balance_ratio = 0.93
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid,
shuffle=False)
#endregion
#region load pre-trained modal
model_name = 'ResNet448'
IMAGE_SIZE = 448
# model_file = '/home/ubuntu/dlp/deploy_models_2019/SubClass0_3/ResNet448-008-train0.7611_val0.840.hdf5'
model_file = '/home/ubuntu/dlp/deploy_models/DR0_DR1/ResNet448-007-train0.8310_val0.883.hdf5'
model1 = keras.models.load_model(model_file, compile=False)
model1.summary()
BATCH_SIZE_TRAIN = 32
BATCH_SIZE_VALID = 64
#endregion
#region save model dir and checkpointer
if not os.path.exists(MODEL_SAVE_DIR):
os.makedirs(MODEL_SAVE_DIR)
model_save_filepath = os.path.join(
MODEL_SAVE_DIR, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer = ModelCheckpoint(model_save_filepath,
verbose=1,
save_weights_only=False,
save_best_only=False)
#endregion
image_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)
#region train header layers
model_train_top = my_transfer_learning.convert_model_transfer(
model1, change_top=False, clsss_num=NUM_CLASSES)
#endregion
#region change learn rate(only fine tuning )
def scheduler(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print('set learning rate by lr.txt')
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
except Exception:
print('read lr-rate file error')
print('set learning rate automatically')
# 内置dictionary数据类型是无序的
dict_lr_rate = collections.OrderedDict()
dict_lr_rate['0'] = 1e-3 # 0.00001
dict_lr_rate['2'] = 2e-4
dict_lr_rate['4'] = 1e-4
dict_lr_rate['6'] = 3e-5
dict_lr_rate['8'] = 1e-5
dict_lr_rate['10'] = 1e-6 # 0.000001
dict_lr_rate['15'] = 6e-7
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
change_lr = keras.callbacks.LearningRateScheduler(scheduler)
# endregion
#region fine tuning all layers
model_fine_tune = my_transfer_learning.convert_trainable_all(
model_train_top)
if GPU_NUM > 1:
print('convert fine tuning model to Multiple GPU...')
model1 = ModelMGPU(model_fine_tune, GPU_NUM)
print('convert fine tuning model to Multiple GPU OK')
else:
model1 = model_fine_tune
op_adam_fine_tune = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model1.compile(loss='categorical_crossentropy',
optimizer=op_adam_fine_tune,
metrics=['acc'],
weighted_metrics=['acc'])
#region data generator
from imgaug import augmenters as iaa
sometimes = lambda aug: iaa.Sometimes(0.96, aug)
# sometimes1 = lambda aug: iaa.Sometimes(0.96, aug)
imgaug_train = iaa.Sequential([
# iaa.Crop(px=(0, 16)), # crop images from each side by 0 to 16px (randomly chosen)
# sometimes(iaa.CropAndPad(
# percent=(-0.04, 0.04),
# pad_mode=ia.ALL,
# pad_cval=(0, 255)
# )),
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.Flipud(0.2), # horizontally flip 50% of the images
# iaa.GaussianBlur(sigma=(0, 3.0)), # blur images with a sigma of 0 to 3.0,
# iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
# shuortcut for CropAndPad
# improve or worsen the contrast If PCH is set to true, the process happens channel-wise with possibly different S.
# sometimes1(iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5), ),
# change brightness of images (by -5 to 5 of original value)
# sometimes1(iaa.Add((-6, 6), per_channel=0.5),),
sometimes(
iaa.Affine(
# scale={"x": (0.92, 1.08), "y": (0.92, 1.08)},
# scale images to 80-120% of their size, individually per axis
# Translation Shifts the pixels of the image by the specified amounts in the x and y directions
translate_percent={
"x": (-0.02, 0.02),
"y": (-0.02, 0.02)
},
# translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -10 to +10 degrees
# shear=(-16, 16), # shear by -16 to +16 degrees
# order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
])
my_gen_train = My_images_weight_generator(
files=train_files,
labels=train_labels,
image_shape=image_shape,
weight_class_start=weight_class_start,
weight_class_end=weight_class_end,
balance_ratio=balance_ratio,
num_class=NUM_CLASSES,
imgaug_seq=imgaug_train,
batch_size=BATCH_SIZE_TRAIN)
my_gen_valid = My_images_generator(files=valid_files,
labels=valid_labels,
image_shape=image_shape,
num_output=NUM_CLASSES,
batch_size=BATCH_SIZE_VALID)
#endregion
history_fine_tuning = model1.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) /
BATCH_SIZE_TRAIN), #number of training batch
epochs=epoch_fine_tuning,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / BATCH_SIZE_VALID),
callbacks=[checkpointer, change_lr])
#endregion
K.clear_session() #release GPU memory
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import pandas as pd
from LIBS.DataPreprocess import my_data
nb_classes = 2
train_type = 'Gradable'
data_version = 'V1'
filename_csv_test = os.path.join(
os.path.abspath('..'), 'datafiles', train_type,
'split_patid_test_{}.csv'.format(data_version))
df = pd.read_csv(filename_csv_test)
files, labels = my_data.get_images_labels(filename_csv_or_pd=df)
model_file = '/home/ubuntu/dlp/deploy_models_english_dr/Gradable/MobileNetV2-005-0.946.hdf5'
input_shape = (224, 224, 3)
save_tsne_image = "/tmp5/t_sne_2020_5_21/Gradable_tsne.png"
from LIBS.Neural_Networks.TSNE.my_tsne_helper import compute_features, gen_tse_features, draw_tsne
features = compute_features(model_file, files, input_shape=input_shape)
X_tsne = gen_tse_features(features)
# save_npy_file = "/tmp5/probs_test1.npy"
# import numpy as np
# np.save(save_npy_file, X_tsne)
# X_tsne = np.load(save_npy_file)
def train_task(filename_csv_train, filename_csv_valid, sub_class_no, model_no,
imagenet=True, epoch=None):
str_subclass_no = str(sub_class_no)
TRAIN_TYPE = 'DLP_SubClass' + str_subclass_no
IMG_AUG_ROTATE_MODE = 1 #1:do flip,roate, 2:do flip 3: only translate_percent
model_save_dir = MODEL_SAVE_DIR + TRAIN_TYPE
#region read csv set weight_class_start, split train validation set
df = pd.read_csv(filename_csv_train)
NUM_CLASSES = df['labels'].nunique(dropna=True)
# len(df.loc[(df['labels'] == 0)])
if sub_class_no == -1: # fundus images, ocular surface ,others
#191443,25278,568463
# 153332,25321,719762
weight_class_start = np.array([1.6, 9, 1])
weight_class_end = np.array([1.6, 9, 1])
weight_class_start = np.array([2.6, 11, 1])
weight_class_end = np.array([2.6, 11, 1])
balance_ratio = 0.93
if sub_class_no == 0.1: #Tessellated fundus 4567, 1324
weight_class_start = np.array([1, 2.4])
weight_class_end = np.array([1, 2.4])
balance_ratio = 0.93
if sub_class_no == 0.2: # Big Optic Cup 4567,6639
weight_class_start = np.array([1, 0.5])
weight_class_end = np.array([1, 0.5])
balance_ratio = 0.93
if sub_class_no == 1: #DR2,3 10284,2490, single label:9715, 2636
#8412,2241
weight_class_start = np.array([1, 2.4])
weight_class_end = np.array([1, 2.4])
#12949,3129, 8412,2241
# add test dataset 11156,2664
weight_class_start = np.array([1, 2.9])
weight_class_end = np.array([1, 2.9])
balance_ratio = 0.93
if sub_class_no == 2: #Data CRVO 2636,1527, single label:2548,1391
weight_class_start = np.array([1, 1.4])
weight_class_end = np.array([1, 1.4])
balance_ratio = 0.93
if sub_class_no == 5: #543, 683, single label:665,604
weight_class_start = np.array([1, 1])
weight_class_end = np.array([1, 1])
balance_ratio = 0.93
if sub_class_no == 10: # 5953 1449
weight_class_start = np.array([1, 2.6])
weight_class_end = np.array([1, 2.6])
balance_ratio = 0.93
if sub_class_no == 15: # single label:1523,136
weight_class_start = np.array([1, 6])
weight_class_end = np.array([1, 6])
balance_ratio = 0.93
if sub_class_no == 29: # Blur 16253 1814, single label:20882,1099, 12949,3129,
#train:17580, 1234, total:20709,1432
weight_class_start = np.array([1, 11])
weight_class_end = np.array([1, 11])
balance_ratio = 0.93
train_files, train_labels = my_data.get_images_labels(filename_csv_train, shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid, shuffle=False)
#endregion
#region load pre-trained modal
#region define and compile model
if model_no == 1:
model_name = 'Xception'
image_size = 299
image_shape = (image_size, image_size, 3)
print('loading model...')
if imagenet:
model1 = keras.applications.xception.Xception(include_top=True, weights='imagenet',
input_shape=image_shape)
else:
model_file = '/home/ubuntu/dlp/deploy_models_new/bigclasses_multilabels/class_weights5_0.2_0.7/Multi_label_Xception-015-train0.9671_val0.945.hdf5'
model1 = keras.models.load_model(model_file, compile=False)
print('loading model OK!')
if model_no == 2:
model_name = 'InceptionResNetV2'
image_size = 299
image_shape = (image_size, image_size, 3)
print('loading model...')
if imagenet:
model1 = keras.applications.inception_resnet_v2.InceptionResNetV2(include_top=True, weights='imagenet',
input_shape=image_shape)
else:
model_file = '/home/ubuntu/dlp/deploy_models_new/bigclasses_multilabels/class_weights5_0.2_0.7/Multi_label_InceptionResNetV2-006-train0.9674_val0.951.hdf5'
model1 = keras.models.load_model(model_file, compile=False)
print('loading model OK!')
if model_no == 3:
model_name = 'Inception_V3'
image_size = 299
image_shape = (image_size, image_size, 3)
print('loading model...')
model1 = keras.applications.inception_v3.InceptionV3(weights='imagenet', include_top=True,
input_shape=image_shape)
print('loading model OK!')
if model_no == 4:
model_name = 'MobilenetV2'
image_size = 224
image_shape = (image_size, image_size, 3)
print('loading model...')
from keras.applications import MobileNetV2
if imagenet:
model1 = MobileNetV2(include_top=True, input_shape=image_shape, weights='imagenet')
else:
model1 = MobileNetV2(include_top=True, input_shape=image_shape, weights=None, classes=NUM_CLASSES)
print('loading model OK!')
if model_no == 5:
model_name = 'ResNet448'
image_size = 448
image_shape = (image_size, image_size, 3)
from LIBS.CNN_Models import my_models
model1, IMAGE_SIZE, BATCH_SIZE_TRAIN = my_models.get_models(model_name, NUM_CLASSES)
print('loading model OK!')
model1 = my_transfer_learning.convert_model_transfer(model1, change_top=True, freeze_feature_extractor=False, clsss_num=NUM_CLASSES)
if GPU_NUM > 1:
print('convert model multiple GPU...')
# bug change to lambda layer, how to generate CAM?
# model1 = keras.utils.multi_gpu_model(model1, gpus=GPU_NUM)
model1 = ModelMGPU(model1, GPU_NUM)
print('convert model multiple GPU OK!')
op_adam = keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model1.compile(loss='categorical_crossentropy',
optimizer=op_adam, metrics=['acc'])
# endregion
BATCH_SIZE_TRAIN = 32
BATCH_SIZE_VALID = 32
#endregion
#region save model dir and checkpointer
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
model_save_filepath = os.path.join(model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer = ModelCheckpoint(model_save_filepath, verbose=1,
save_weights_only=False, save_best_only=False)
#endregion
def scheduler(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print('set learning rate by lr.txt')
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
except Exception:
print('read lr-rate file error')
print('set learning rate automatically')
# 内置dictionary数据类型是无序的
dict_lr_rate = collections.OrderedDict()
dict_lr_rate['0'] = 1e-3
dict_lr_rate['2'] = 3e-4
dict_lr_rate['3'] = 1e-4
dict_lr_rate['5'] = 1e-5
dict_lr_rate['7'] = 2e-6
dict_lr_rate['9'] = 1e-6 # 0.000001
dict_lr_rate['15'] = 6e-7
#only for ocular surface
dict_lr_rate['0'] = 1e-3
dict_lr_rate['1'] = 1e-4
dict_lr_rate['2'] = 1e-5
dict_lr_rate['3'] = 1e-6
dict_lr_rate['15'] = 6e-7
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
change_lr = keras.callbacks.LearningRateScheduler(scheduler)
#region train
if epoch is None:
if len(df) > 10000:
epoch = 20
elif len(df) > 5000:
epoch = 25
elif len(df) > 2000:
epoch = 30
else:
epoch = 40
#region data generator
from imgaug import augmenters as iaa
sometimes = lambda aug: iaa.Sometimes(0.96, aug)
# sometimes1 = lambda aug: iaa.Sometimes(0.96, aug)
imgaug_train_seq = iaa.Sequential([
# iaa.Crop(px=(0, 16)), # crop images from each side by 0 to 16px (randomly chosen)
# sometimes(iaa.CropAndPad(
# percent=(-0.04, 0.04),
# pad_mode=ia.ALL,
# pad_cval=(0, 255)
# )),
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.Flipud(0.2), # horizontally flip 50% of the images
# iaa.GaussianBlur(sigma=(0, 3.0)), # blur images with a sigma of 0 to 3.0,
# iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
# shuortcut for CropAndPad
# improve or worsen the contrast If PCH is set to true, the process happens channel-wise with possibly different S.
# sometimes1(iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5), ),
# change brightness of images (by -5 to 5 of original value)
# sometimes1(iaa.Add((-6, 6), per_channel=0.5),),
sometimes(iaa.Affine(
# scale={"x": (0.92, 1.08), "y": (0.92, 1.08)},
# scale images to 80-120% of their size, individually per axis
# Translation Shifts the pixels of the image by the specified amounts in the x and y directions
translate_percent={"x": (-0.02, 0.02), "y": (-0.02, 0.02)},
# translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -10 to +10 degrees
# shear=(-16, 16), # shear by -16 to +16 degrees
# order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
])
my_gen_train = My_images_weight_generator(files=train_files, labels=train_labels, image_shape=image_shape,
weight_class_start=weight_class_start, weight_class_end=weight_class_end, balance_ratio=balance_ratio,
num_class=NUM_CLASSES, imgaug_seq=imgaug_train_seq, batch_size=BATCH_SIZE_TRAIN)
my_gen_valid = My_images_generator(files=valid_files, labels=valid_labels, image_shape=image_shape,
num_output=NUM_CLASSES, batch_size=BATCH_SIZE_VALID)
#endregion
history_train = model1.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) / BATCH_SIZE_TRAIN), #number of training batch
epochs=epoch,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / BATCH_SIZE_VALID),
callbacks=[checkpointer, change_lr]
)
def train_task_one_step(model_file,
filename_csv_train,
filename_csv_valid,
filename_csv_test=None,
num_classes=None,
pre_define_model=None,
add_top=False,
change_top=False,
input_shape=(299, 299, 3),
imgaug_train_seq=None,
optimizer="adam",
lookahead=False,
epoch=None,
dict_lr=None,
batch_size_train=32,
batch_size_valid=64,
label_smoothing=0,
class_weight=None,
weight_class_start=None,
weight_class_end=None,
balance_ratio=None,
sensitivity=None,
specificity=None,
devices=None,
verbose=1,
model_save_dir='/tmp',
model_name='model1',
config_file_realtime='config_file_realtime.json',
use_multiprocessing=True,
workers=5,
plt_history_image_file=None):
#region read csv, split train validation set
if num_classes is None:
df = pd.read_csv(filename_csv_train)
num_classes = df['labels'].nunique(dropna=True)
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid)
#endregion
#region data sequence
if weight_class_start is not None:
generator_train = My_images_weight_generator_2d(
train_files,
train_labels,
num_output=num_classes,
batch_size=batch_size_train,
weight_class_start=weight_class_start,
weight_class_end=weight_class_end,
balance_ratio=balance_ratio,
imgaug_seq=imgaug_train_seq,
label_smoothing=label_smoothing,
image_shape=input_shape,
)
else:
generator_train = My_images_generator_2d(
train_files,
train_labels,
num_output=num_classes,
batch_size=batch_size_train,
image_shape=input_shape,
imgaug_seq=imgaug_train_seq,
label_smoothing=label_smoothing)
list1 = [None] + list(
input_shape) # the last batch of an epoch may be smaller.
shape_x = tuple(list1) #(batch_size_train, 299, 299, 3)
shape_y = (None, num_classes) #(batch_size, num_classes)
data_train = tf.data.Dataset.from_generator(generator=generator_train.gen,
output_types=(tf.float32,
tf.float32),
output_shapes=(shape_x,
shape_y))
generator_valid = My_images_generator_2d(valid_files,
valid_labels,
num_output=num_classes,
batch_size=batch_size_valid,
image_shape=input_shape)
data_valid = tf.data.Dataset.from_generator(generator=generator_valid.gen,
output_types=(tf.float32,
tf.float32),
output_shapes=(shape_x,
shape_y))
if filename_csv_test is not None:
test_files, test_labels = my_data.get_images_labels(filename_csv_test)
#endregion
#region callbacks
os.makedirs(model_save_dir, exist_ok=True)
save_filepath_finetuning = os.path.join(
model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_finetuning = keras.callbacks.ModelCheckpoint(
save_filepath_finetuning,
verbose=1,
save_weights_only=False,
save_best_only=False)
class My_callback(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['epoch_compute_cf_train'] == 1:
compute_cf_train = True
else:
compute_cf_train = False
if data['epoch_compute_cf_valid'] == 1:
compute_cf_valid = True
else:
compute_cf_valid = False
if data['epoch_compute_cf_test'] == 1:
compute_cf_test = True
else:
compute_cf_test = False
except:
print('read realtime helper file error!')
compute_cf_train = True
compute_cf_valid = True
compute_cf_test = True
if compute_cf_train:
print('calculate confusion matrix of training dataset...')
generator_cf_train = My_images_generator_2d(
train_files,
train_labels,
num_output=num_classes,
batch_size=batch_size_train,
image_shape=input_shape)
i = 0
for x_train, y_train in generator_cf_train.gen():
probabilities = self.model.predict(x_train)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(train_files) / batch_size_train):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, num_classes)]
confusion_matrix_train = sk_confusion_matrix(train_labels,
y_preds,
labels=labels)
print(confusion_matrix_train)
if compute_cf_valid:
print('calculate confusion matrix of validation dataset...')
generator_cf_valid = My_images_generator_2d(
valid_files,
valid_labels,
num_output=num_classes,
batch_size=batch_size_valid,
image_shape=input_shape)
i = 0
for x_valid, y_valid in generator_cf_valid.gen():
probabilities = self.model.predict(x_valid)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(valid_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, num_classes)]
confusion_matrix_valid = sk_confusion_matrix(valid_labels,
y_preds,
labels=labels)
print(confusion_matrix_valid)
if compute_cf_test:
print('calculate confusion matrix of test dataset...')
generator_cf_test = My_images_generator_2d(
test_files,
test_labels,
num_output=num_classes,
batch_size=batch_size_valid,
image_shape=input_shape)
i = 0
for x_test, y_test in generator_cf_test.gen():
probabilities = self.model.predict(x_test)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(test_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, num_classes)]
confusion_matrix_test = sk_confusion_matrix(test_labels,
y_preds,
labels=labels)
print(confusion_matrix_test)
my_callback = My_callback()
if epoch is None:
if len(df) > 10000:
epoch = 20
elif len(df) > 5000:
epoch = 25
elif len(df) > 2000:
epoch = 30
else:
epoch = 40
def scheduler_finetuning(epoch):
if optimizer == 'adabound':
return K.get_value(model1.optimizer.lr)
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['lr_rate'] > 0:
lr_rate = data['lr_rate']
print(
"epoch：%d, current learn rate: %f by realtime helper file"
% (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
except Exception:
print('read realtime helper file error!')
if dict_lr is not None:
for (k, v) in dict_lr.items():
if epoch >= int(k):
lr_rate = v
print(
"epoch：%d, set learn rate: %f according to pre-defined policy."
% (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(
scheduler_finetuning)
#endregion
strategy = tf.distribute.MirroredStrategy(devices=devices)
print("Number of devices: {} used".format(strategy.num_replicas_in_sync))
with strategy.scope():
if pre_define_model is None:
print('loading model...')
model1 = keras.models.load_model(model_file, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(
model1,
num_output=num_classes,
activation_function='softmax')
if change_top:
model1 = my_transfer_learning.convert_model_transfer(
model1,
clsss_num=num_classes,
change_top=change_top,
activation_function='SoftMax',
freeze_feature_extractor=False)
model1 = my_transfer_learning.convert_trainable_all(model1)
else:
print('creating model...')
if pre_define_model.lower() == 'inceptionv3':
model1 = keras.applications.InceptionV3(include_top=False)
elif pre_define_model.lower() == 'xception':
model1 = keras.applications.Xception(include_top=False)
elif pre_define_model.lower() == 'inceptionresnetv2':
model1 = keras.applications.InceptionResNetV2(
include_top=False)
else:
raise Exception('predefine model error!')
model1 = my_transfer_learning.add_top(
model1, num_output=num_classes, activation_function='softmax')
print('creating model complete!')
assert optimizer in ['adam', 'SGD',
'adabound'], 'optimizer type error'
if optimizer == 'adam':
op_finetuning = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
if optimizer == 'SGD':
op_finetuning = keras.optimizers.sgd(lr=1e-3,
momentum=0.9,
nesterov=True)
if optimizer == 'adabound':
op_finetuning = AdaBound(lr=1e-03,
final_lr=0.1,
gamma=1e-03,
amsbound=False)
if sensitivity is not None and specificity is not None:
sensivity = keras.metrics.SensitivityAtSpecificity(
specificity=sensitivity)
specificity = keras.metrics.SpecificityAtSensitivity(
sensitivity=specificity)
my_metric = ['acc', sensivity, specificity]
else:
my_metric = ['acc']
# loss1 = tf.keras.losses.CategoricalCrossentropy(from_logits=False, label_smoothing=label_smoothing)
loss1 = 'categorical_crossentropy'
model1.compile(loss=loss1, optimizer=op_finetuning, metrics=my_metric)
if lookahead:
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model1)
history = model1.fit(
data_train,
steps_per_epoch=math.ceil(len(train_files) / batch_size_train),
epochs=epoch,
verbose=verbose,
validation_data=data_valid,
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[
checkpointer_finetuning, change_lr_finetuning, my_callback
],
class_weight=class_weight,
use_multiprocessing=use_multiprocessing,
workers=workers)
if plt_history_image_file is not None:
import matplotlib.pyplot as plt
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
plt.savefig(plt_history_image_file)
plt.close()
K.clear_session()
def train_task(filename_csv_train,
filename_csv_valid,
num_classes,
model_name,
model1,
image_size,
add_top=False,
change_top=True,
freeze_layes_num=None,
class_weights=None,
inter_class_ratio=None,
positive_weight_ratio=4,
exclusion_loss_ratio=0,
imgaug_train_seq=None,
epoch_finetuning=15,
dict_lr_finetuning=None,
batch_size_train=32,
batch_size_valid=32,
smooth_factor=0,
model_save_dir='/tmp'):
#region load csv files
print('loading csv data')
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid)
# region get the number of labels in every class
df = pd.read_csv(filename_csv_train)
# NUM_CLASSES = df['labels'].nunique(dropna=True) #only work for single label
LIST_CLASS_SAMPLES_NUM = [0 for _ in range(num_classes)]
for _, row in df.iterrows():
labels = str(row["labels"]) # single label may be int type
list_labels = labels.split('_')
for label in list_labels:
if label == '':
continue
assert label.isdigit(), 'Error label!'
LIST_CLASS_SAMPLES_NUM[int(label)] += 1
# endregion
# region multiple labels convert to simgle label(smallest class num) in order to dynamic resampling
train_labels_single = [0 for _ in range(len(train_labels))]
for i, labels in enumerate(train_labels):
labels = str(labels) # single label may be int type
list_labels = labels.split('_')
label_current = None
for label in list_labels:
if label == '':
continue
if label_current is None: # it is the first label
label_current = label
elif LIST_CLASS_SAMPLES_NUM[int(label)] < LIST_CLASS_SAMPLES_NUM[
int(label_current)]:
label_current = label # current label's class contain smaller number of images than before
train_labels_single[i] = label_current
# endregion
print('loading csv data complete!')
#endregion
#region data generator
image_shape = (image_size, image_size, 3)
file_weight_power = os.path.join(sys.path[0], 'weight_power.txt')
my_gen_train = My_gen_weight_multi_labels(
files=train_files,
labels=train_labels,
labels_single=train_labels_single,
image_shape=image_shape,
file_weight_power=file_weight_power,
list_class_samples_num=LIST_CLASS_SAMPLES_NUM,
num_class=num_classes,
smooth_factor=smooth_factor,
imgaug_seq=imgaug_train_seq,
batch_size=batch_size_train)
my_gen_valid = My_images_generator(files=valid_files,
labels=valid_labels,
image_shape=image_shape,
multi_labels=True,
num_output=num_classes,
batch_size=batch_size_valid)
#endregion
#region custom loss function
if class_weights is None:
if inter_class_ratio is not None:
class_positive_weights = op_class_weight(
LIST_CLASS_SAMPLES_NUM, weight_power=inter_class_ratio)
class_positive_weights = np.array(class_positive_weights)
# class_samples_weights[0] /= (3) #class0 Normal
class_positive_weights *= positive_weight_ratio
else:
class_positive_weights = [1 for _ in range(num_classes)]
class_positive_weights = np.array(class_positive_weights)
print(np.round(class_positive_weights, 2))
class_weights = []
for class_weight1 in class_positive_weights:
class_weights.append([1,
class_weight1]) # sigmoid : 0:1, 1:weight1
class_weights = np.array(class_weights)
custom_loss_function = get_weighted_binary_crossentropy(
class_weights, exclusion_loss_ratio)
#endregion
# region loading and converting model
if isinstance(model1, str):
print('loading model...')
model1 = keras.models.load_model(model1, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(model1,
num_output=num_classes,
activation_function='Sigmoid')
model_train_top = my_transfer_learning.convert_model_transfer(
model1,
change_top=change_top,
clsss_num=num_classes,
activation_function='Sigmoid',
freeze_feature_extractor=True,
freeze_layes_num=freeze_layes_num)
model_train_top.summary()
if GPU_NUM > 1:
print('convert model to Multiple GPU...')
model_train_top = ModelMGPU(model_train_top, GPU_NUM)
print('convert model to Multiple GPU OK')
op_adam_train_top = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
model_train_top.compile(loss=custom_loss_function,
optimizer=op_adam_train_top,
metrics=['acc'])
from LIBS.CNN_Models.optimization.lookahead import Lookahead
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_train_top)
#endregion
#region fine tuning all layers
model_fine_tune = my_transfer_learning.convert_trainable_all(
model_train_top)
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
save_filepath_finetuning = os.path.join(
model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_finetuning = ModelCheckpoint(save_filepath_finetuning,
verbose=1,
save_weights_only=False,
save_best_only=False)
def scheduler_fine_tuning(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') # 删除换行符
lr_rate = float(line)
print("epoch：%d, current learn rate by lr.txt: %f" %
(epoch, lr_rate))
K.set_value(model_fine_tune.optimizer.lr, lr_rate)
except Exception:
if dict_lr_finetuning is not None:
dict_lr_rate = dict_lr_finetuning
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate automatically: %f" %
(epoch, lr_rate))
K.set_value(model_fine_tune.optimizer.lr, lr_rate)
return K.get_value(model_fine_tune.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(
scheduler_fine_tuning)
history_finetuning = model_fine_tune.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) /
batch_size_train), #number of training batch
epochs=epoch_finetuning,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_finetuning, change_lr_finetuning])
#endregion
K.clear_session() #release GPU memory
def do_predict_batch(dicts_models,
all_files,
batch_size_test=64,
argmax=False,
cuda_visible_devices="",
gpu_num=1):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
if isinstance(all_files, str): #csv file
all_files, all_labels = my_data.get_images_labels(
filename_csv_or_pd=all_files)
assert len(all_files) > 0, 'No Data'
prob_lists = [
] #each element contain all probabilities multiple batch, np.vstack
preds_list = []
for dict_model in dicts_models:
if ('model' not in dict_model) or (dict_model['model'] is None):
print('prepare load model:', dict_model['model_file'])
model1 = keras.models.load_model(dict_model['model_file'],
compile=False)
print('load model:', dict_model['model_file'], ' complete')
if gpu_num > 1:
print('convert Multi-GPU model:', dict_model['model_file'])
model1 = keras.utils.multi_gpu_model(model1, gpus=gpu_num)
print('convert Multi-GPU model:', dict_model['model_file'],
' complete')
dict_model['model'] = model1
else:
model1 = dict_model['model'] # avoid loading models multiple times
if 'image_size' in dict_model:
image_size = dict_model['image_size']
elif model1.input_shape[2] is not None:
image_size = model1.input_shape[2]
else:
image_size = 299
j = 0 # batch
for x in my_images_generator.my_Generator_test(
all_files,
image_shape=(image_size, image_size, 3),
batch_size=batch_size_test):
probabilities = model1.predict_on_batch(x)
if j == 0: #'probs' not in locals().keys():
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
j += 1
print('batch:', j)
prob_lists.append(probs)
if argmax:
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
preds_list.append(y_preds)
sum_models_weights = 0
for i, prob1 in enumerate(prob_lists):
if 'model_weight' not in dicts_models[i]:
model_weight = 1
else:
model_weight = dicts_models[i]['model_weight']
if i == 0:
prob_total = prob1 * model_weight
else:
prob_total += prob1 * model_weight
sum_models_weights += model_weight
prob_total /= sum_models_weights
if argmax:
y_pred_total = prob_total.argmax(axis=-1)
return prob_total, y_pred_total, prob_lists, preds_list
else:
return prob_total, prob_lists
def train_task_two_steps(model1, filename_csv_train, FILENAME_CSV_VALID, filename_csv_test=None,
input_shape=(299, 299, 3), imgaug_train_seq=None,
add_top=False, change_top=True, freeze_layes_num=None,
optimizer="adam", lookahead=True,
epoch_traintop=0, epoch_finetuning=0,
dict_lr_traintop=None, dict_lr_finetuning=None,
batch_size_train=32, batch_size_valid=64,
label_smoothing=0, class_weight=None,
weight_class_start=None, weight_class_end=None, balance_ratio=None,
model_save_dir='/tmp', model_name='model1',
gpu_num=1, verbose=1,
config_file_realtime='config_file_realtime.json'):
#region read csv, split train validation set
df = pd.read_csv(filename_csv_train)
NUM_CLASSES = df['labels'].nunique(dropna=True)
train_files, train_labels = my_data.get_images_labels(filename_csv_train, shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(FILENAME_CSV_VALID)
#endregion
#region load , convert train_top model
if isinstance(model1, str):
print('loading model...')
model1 = keras.models.load_model(model1, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(model1, num_output=NUM_CLASSES, activation_function='SoftMax')
model_traintop = my_transfer_learning.convert_model_transfer(model1, clsss_num=NUM_CLASSES,
change_top=change_top, activation_function='SoftMax',
freeze_feature_extractor=True, freeze_layes_num=freeze_layes_num)
if gpu_num > 1:
print('convert base model to Multiple GPU...')
model_traintop = ModelMGPU(model_traintop, gpu_num)
print('convert base top model to Multiple GPU OK')
assert optimizer in ['adam', 'SGD', 'adabound'], 'optimizer type error'
#endregion
#region data generator
if weight_class_start is not None:
my_gen_train = My_images_weight_generator(files=train_files, labels=train_labels, image_shape=input_shape,
weight_class_start=weight_class_start, weight_class_end=weight_class_end, balance_ratio=balance_ratio,
num_class=NUM_CLASSES, imgaug_seq=imgaug_train_seq, batch_size=batch_size_train,
label_smoothing=label_smoothing)
else:
my_gen_train = My_images_generator(files=train_files, labels=train_labels, image_shape=input_shape,
imgaug_seq=imgaug_train_seq, num_output=NUM_CLASSES, batch_size=batch_size_train)
my_gen_valid = My_images_generator(files=valid_files, labels=valid_labels,
image_shape=input_shape, num_output=NUM_CLASSES, batch_size=batch_size_valid)
if filename_csv_test is not None:
test_files, test_labels = my_data.get_images_labels(filename_csv_test)
my_gen_test = My_images_generator(files=test_files, labels=test_labels,
image_shape=input_shape, num_output=NUM_CLASSES, batch_size=batch_size_valid)
#endregion
# region save model dir and checkpointer
os.makedirs(model_save_dir, exist_ok=True)
save_filepath_traintop = os.path.join(model_save_dir, model_name + "-traintop-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_traintop = ModelCheckpoint(save_filepath_traintop,
verbose=1, save_weights_only=False, save_best_only=False)
save_filepath_finetuning = os.path.join(model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_finetuning = ModelCheckpoint(save_filepath_finetuning,
verbose=1, save_weights_only=False, save_best_only=False)
# endregion
#region computer validation confusion matrix
class My_callback(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['epoch_compute_cf_train'] == 1:
compute_cf_train = True
else:
compute_cf_train = False
if data['epoch_compute_cf_valid'] == 1:
compute_cf_valid = True
else:
compute_cf_valid = False
if data['epoch_compute_cf_test'] == 1:
compute_cf_test = True
else:
compute_cf_test = False
except:
print('read realtime helper file error!')
compute_cf_train = True
compute_cf_valid = True
compute_cf_test = True
if compute_cf_train:
print('calculate confusion matrix of training dataset...')
# do not use img augmentation
my_gen_train_test = My_images_generator(files=train_files, labels=train_labels,
image_shape=input_shape, num_output=NUM_CLASSES,
batch_size=batch_size_train)
i = 0
for x_train, y_train in my_gen_train_test.gen():
probabilities = self.model.predict(x_train)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(train_files) / batch_size_train):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, NUM_CLASSES)]
confusion_matrix_train = sk_confusion_matrix(train_labels, y_preds, labels=labels)
print(confusion_matrix_train)
if compute_cf_valid:
print('calculate confusion matrix of validation dataset...')
i = 0
for x_valid, y_valid in my_gen_valid.gen():
probabilities = self.model.predict(x_valid)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(valid_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, NUM_CLASSES)]
confusion_matrix_valid = sk_confusion_matrix(valid_labels, y_preds, labels=labels)
print(confusion_matrix_valid)
if compute_cf_test:
print('calculate confusion matrix of test dataset...')
i = 0
for x_test, y_test in my_gen_test.gen():
probabilities = self.model.predict(x_test)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(test_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, NUM_CLASSES)]
confusion_matrix_test = sk_confusion_matrix(test_labels, y_preds, labels=labels)
print(confusion_matrix_test)
#endregion
my_callback = My_callback()
#endregion
#region train header layers
if epoch_traintop > 0:
if optimizer == 'adam':
op_train_top = keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if optimizer == 'SGD':
op_train_top = keras.optimizers.sgd(lr=1e-3, momentum=0.9, nesterov=True)
if optimizer == 'adabound':
op_train_top = AdaBound(lr=1e-03, final_lr=0.1, gamma=1e-03, amsbound=False)
model_traintop.compile(loss='categorical_crossentropy',
optimizer=op_train_top, metrics=['acc'])
# optimizer = op_train_top, metrics = ['acc', sensitivity, specificity])
if lookahead:
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_traintop)
if epoch_traintop is None:
if len(df) > 10000:
epoch_traintop = 5
elif len(df) > 5000:
epoch_traintop = 8
elif len(df) > 2000:
epoch_traintop = 10
else:
epoch_traintop = 15
def scheduler_traintop(epoch):
if optimizer == 'adabound':
return K.get_value(model_traintop.optimizer.lr)
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['lr_rate'] > 0:
lr_rate = data['lr_rate']
print("epoch：%d, current learn rate: %f by realtime helper file" % (epoch, lr_rate))
K.set_value(model_traintop.optimizer.lr, lr_rate)
return K.get_value(model_traintop.optimizer.lr)
except Exception:
print('read realtime helper file error!')
if dict_lr_traintop is not None:
for (k, v) in dict_lr_traintop.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, set learn rate: %f according to pre-defined policy." % (epoch, lr_rate))
K.set_value(model_traintop.optimizer.lr, lr_rate)
return K.get_value(model_traintop.optimizer.lr)
change_lr_traintop = keras.callbacks.LearningRateScheduler(scheduler_traintop)
history_top = model_traintop.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) / batch_size_train), #number of training batch
epochs=epoch_traintop,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_traintop, change_lr_traintop, my_callback],
class_weight=class_weight)
#endregion
#region fine tuning all layers
if epoch_finetuning > 0:
model_finetuning = my_transfer_learning.convert_trainable_all(model_traintop)
if optimizer == 'adam':
op_finetuning = keras.optimizers.Adam(lr=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if optimizer == 'SGD':
op_finetuning = keras.optimizers.sgd(lr=1e-5, momentum=0.9, nesterov=True)
if optimizer == 'adabound':
op_finetuning = AdaBound(lr=1e-04, final_lr=0.01, gamma=1e-03, amsbound=False)
model_finetuning.compile(loss='categorical_crossentropy',
optimizer=op_finetuning, metrics=['acc'])
# optimizer = op_finetuning, metrics = ['acc', sensitivity, specificity])
if lookahead:
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_finetuning)
if epoch_finetuning is None:
if len(df) > 10000:
epoch_finetuning = 20
elif len(df) > 5000:
epoch_finetuning = 25
elif len(df) > 2000:
epoch_finetuning = 30
else:
epoch_finetuning = 40
def scheduler_finetuning(epoch):
if optimizer == 'adabound':
return K.get_value(model_finetuning.optimizer.lr)
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['lr_rate'] > 0:
lr_rate = data['lr_rate']
print("epoch：%d, current learn rate: %f by realtime helper file" % (epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
return K.get_value(model_finetuning.optimizer.lr)
except Exception:
print('read realtime helper file error!')
if dict_lr_finetuning is not None:
for (k, v) in dict_lr_finetuning.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, set learn rate: %f according to pre-defined policy." % (epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
return K.get_value(model_finetuning.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(scheduler_finetuning)
history_finetuning = model_finetuning.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) / batch_size_train), # number of training batch
epochs=epoch_finetuning,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_finetuning, change_lr_finetuning, my_callback],
class_weight=class_weight
)
#endregion
K.clear_session() #release GPU memory
def train_task_two_steps(model1,
filename_csv_train,
FILENAME_CSV_VALID,
filename_csv_test=None,
image_size=299,
imgaug_train_seq=None,
num_output=1,
add_top=False,
change_top=True,
freeze_layes_num=None,
epoch_traintop=None,
epoch_finetuning=None,
dict_lr_traintop=None,
dict_lr_finetuning=None,
batch_size_train=32,
batch_size_valid=64,
model_save_dir='/tmp',
model_name='model1',
compute_cf_train=False,
compute_cf_valid=False,
compute_cf_test=False,
gpu_num=1,
verbose=1):
#region read csv, split train validation set
df = pd.read_csv(filename_csv_train
) #get number of train samples, set epoch automaticaly
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(FILENAME_CSV_VALID)
#endregion
#region load , convert and compile train_top model
if isinstance(model1, str):
print('loading model...')
model1 = keras.models.load_model(model1, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(model1,
num_output=num_output,
activation_function='SoftMax')
model_traintop = my_transfer_learning.convert_model_transfer(
model1,
clsss_num=num_output,
change_top=change_top,
activation_function='Regression',
freeze_feature_extractor=True,
freeze_layes_num=freeze_layes_num)
if gpu_num > 1:
print('convert base model to Multiple GPU...')
model_traintop = ModelMGPU(model_traintop, gpu_num)
print('convert base top model to Multiple GPU OK')
#endregion
#region data generator
image_shape = (image_size, image_size, 3)
my_gen_train = My_images_generator(files=train_files,
labels=train_labels,
regression=True,
imgaug_seq=imgaug_train_seq,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_train)
my_gen_valid = My_images_generator(files=valid_files,
labels=valid_labels,
regression=True,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_valid)
if compute_cf_test:
test_files, test_labels = my_data.get_images_labels(filename_csv_test)
my_gen_test = My_images_generator(files=test_files,
labels=test_labels,
regression=True,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_valid)
#endregion
# region save model dir and checkpointer
os.makedirs(model_save_dir, exist_ok=True)
save_filepath_traintop = os.path.join(
model_save_dir, model_name +
"-{epoch:03d}--{val_loss:.1f}---{val_mean_absolute_error:.1f}---{mean_squared_error:.1f}-{mean_absolute_error:.1f}.hdf5"
)
checkpointer_traintop = ModelCheckpoint(save_filepath_traintop,
save_weights_only=False,
save_best_only=False)
save_filepath_finetuning = os.path.join(
model_save_dir, model_name +
"-{epoch:03d}--{val_loss:.1f}---{val_mean_absolute_error:.1f}---{mean_squared_error:.1f}-{mean_absolute_error:.1f}.hdf5"
)
checkpointer_finetuning = ModelCheckpoint(save_filepath_finetuning,
save_weights_only=False,
save_best_only=False)
# endregion
#region computer validation confusion matrix
class My_callback(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
if compute_cf_train:
print('calculate confusion matrix of training dataset...')
my_gen_train_no_imgaug = My_images_generator(
files=train_files,
labels=train_labels,
regression=True,
imgaug_seq=imgaug_train_seq,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_train)
i = 0
for x_train, y_train in my_gen_train_no_imgaug.gen():
probabilities = self.model.predict(x_train)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(train_files) / batch_size_train):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
# endregion
if compute_cf_valid:
print('calculate confusion matrix of validation dataset...')
#use the same my_gen_valid
i = 0
for x_valid, y_valid in my_gen_valid.gen():
probabilities = self.model.predict(x_valid)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(valid_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
if compute_cf_test:
print('calculate confusion matrix of test dataset...')
i = 0
for x_test, y_test in my_gen_test.gen():
probabilities = self.model.predict(x_test)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(test_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
#endregion
my_callback = My_callback()
#endregion
#region train header layers
if epoch_traintop > 0:
op_adam_train_top = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
op_adam_train_top.compile(loss='mse',
optimizer=op_adam_train_top,
metrics=['mae', 'mse'])
from LIBS.CNN_Models.optimization.lookahead import Lookahead
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_traintop)
if epoch_traintop is None:
if len(df) > 10000:
epoch_traintop = 5
elif len(df) > 5000:
epoch_traintop = 8
elif len(df) > 2000:
epoch_traintop = 10
else:
epoch_traintop = 15
def scheduler_traintop(epoch):
if dict_lr_traintop is not None:
dict_lr = dict_lr_traintop
for (k, v) in dict_lr.items():
if epoch >= int(k):
lr_rate = v
print(
"epoch：%d, set learn rate: %f according to pre-defined policy."
% (epoch, lr_rate))
K.set_value(model_traintop.optimizer.lr, lr_rate)
return K.get_value(model_traintop.optimizer.lr)
change_lr_traintop = keras.callbacks.LearningRateScheduler(
scheduler_traintop)
history_top = model_traintop.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(
len(train_files) /
batch_size_train), #number of training batch
epochs=epoch_traintop,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_traintop, change_lr_traintop, my_callback])
#endregion
#region fine tuning all layers
if epoch_finetuning > 0:
model_finetuning = my_transfer_learning.convert_trainable_all(
model_traintop)
op_adam_finetuning = keras.optimizers.Adam(lr=1e-5,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
model_finetuning.compile(loss='mse',
optimizer=op_adam_finetuning,
metrics=['mae', 'mse'])
from LIBS.CNN_Models.optimization.lookahead import Lookahead
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_finetuning)
if epoch_finetuning is None:
if len(df) > 10000:
epoch_finetuning = 20
elif len(df) > 5000:
epoch_finetuning = 25
elif len(df) > 2000:
epoch_finetuning = 30
else:
epoch_finetuning = 40
def scheduler_finetuning(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print("epoch：%d, set learning rate: %f by lr.txt" %
(epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
except Exception:
if dict_lr_finetuning is not None:
dict_lr = dict_lr_finetuning
for (k, v) in dict_lr.items():
if epoch >= int(k):
lr_rate = v
print(
"epoch：%d, set learn rate: %f according to pre-defined policy."
% (epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
return K.get_value(model_finetuning.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(
scheduler_finetuning)
history_finetuning = model_finetuning.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(
len(train_files) /
batch_size_train), # number of training batch
epochs=epoch_finetuning,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[
checkpointer_finetuning, change_lr_finetuning, my_callback
])
#endregion
K.clear_session() #release GPU memory
def op_files_multiclass(all_files,
prob_total,
dir_preprocess,
dir_original='',
dir_dest='/tmp',
keep_subdir=False):
if isinstance(all_files, str): #csv file
all_files, all_labels = my_data.get_images_labels(
filename_csv_or_pd=all_files)
for i in range(len(all_files)):
#region get top class, and prob of top class
prob_list = prob_total[i].tolist()
top_class_n = heapq.nlargest(5, range(len(prob_total[i])),
prob_total[i].take)
#top_n[0] Softmax argmax class no
prob_top_class0 = round(prob_list[top_class_n[0]] * 100, 0)
prob_top_class0 = int(prob_top_class0)
#endregion
img_file_source = all_files[i]
filename = os.path.basename(img_file_source)
if '#' in filename: #obtain filename after #,
filename = filename.split('#')[-1]
filename_dest = 'prob' + str(prob_top_class0) + '#' + filename
if keep_subdir:
basedir = os.path.dirname(img_file_source)
if not dir_preprocess.endswith('/'):
dir_preprocess += '/'
if not basedir.endswith('/'):
basedir += '/'
img_file_dest = os.path.join(dir_dest,
basedir.replace(dir_preprocess, ''),
str(top_class_n[0]), filename_dest)
else:
img_file_dest = os.path.join(dir_dest, str(top_class_n[0]),
filename_dest)
os.makedirs(os.path.dirname(img_file_dest), exist_ok=True)
# copy original files instead of preprocessed images
if dir_original != '':
if dir_original.endswith('/'):
dir_original = dir_original[:-1]
if dir_preprocess.endswith('/'):
dir_preprocess = dir_preprocess[:-1]
img_file_source = img_file_source.replace(dir_preprocess,
dir_original)
if not os.path.exists(img_file_source):
raise RuntimeError(img_file_source + ' not found!')
shutil.copy(img_file_source, img_file_dest)
print(img_file_dest)
def do_predict(dicts_models, files, devices=None,
batch_size_test=64, argmax=False,
use_multiprocessing=True, workers=4):
if isinstance(files, str): #csv file
files, _ = my_data.get_images_labels(filename_csv_or_pd=files)
assert len(files) > 0, 'No Data'
prob_lists = [] #each element contain all probabilities multiple batch, np.vstack
preds_list = []
strategy = tf.distribute.MirroredStrategy(devices=devices)
with strategy.scope():
for dict_model in dicts_models:
if ('model' not in dict_model) or (dict_model['model'] is None):
print('prepare to load model:', dict_model['model_file'])
model1 = keras.models.load_model(dict_model['model_file'], compile=False)
print('load model:', dict_model['model_file'], ' complete')
dict_model['model'] = model1
else:
model1 = dict_model['model'] # avoid loading models multiple times
if 'input_shape' in dict_model:
input_shape = dict_model['input_shape']
elif len(model1.input_shape) == 4: # [batch, height, width, channel]
input_shape = model1.input_shape[1:]
else:
input_shape = (299, 299, 3)
from LIBS.Generator.my_images_generator_2d import My_images_generator_2d_test
list1 = [None] + list(input_shape) # the last batch of an epoch may be smaller.
shape_x = tuple(list1) # (batch_size_train, 299, 299, 3)
generator_test = My_images_generator_2d_test(files,
batch_size=batch_size_test, image_shape=input_shape)
data_test = tf.data.Dataset.from_generator(generator=generator_test.gen,
output_types=(tf.float16), output_shapes=(shape_x))
class My_callback(keras.callbacks.Callback):
def on_predict_batch_end(self, batch, logs=None):
print('batch:', batch)
probs = model1.predict(data_test,
callbacks=[My_callback()],
use_multiprocessing=use_multiprocessing, workers=workers)
''' #old version
j = 0 # batch
for x in generator_test.gen():
probabilities = model1.predict_on_batch(x)
if j == 0: #'probs' not in locals().keys():
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
j += 1
print('batch:', j)
'''
prob_lists.append(probs)
if argmax:
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
preds_list.append(y_preds)
sum_models_weights = 0
for i, prob1 in enumerate(prob_lists):
if 'model_weight' not in dicts_models[i]:
model_weight = 1
else:
model_weight = dicts_models[i]['model_weight']
if i == 0:
prob_total = prob1 * model_weight
else:
prob_total += prob1 * model_weight
sum_models_weights += model_weight
prob_total /= sum_models_weights
if argmax:
y_pred_total = prob_total.argmax(axis=-1)
return prob_total, y_pred_total, prob_lists, preds_list
else:
return prob_total, prob_lists
