def train_task(filename_csv_train, filename_csv_valid, epoch_fine_tuning=20):
str_subclass_no = '0.3'
TRAIN_TYPE = 'DLP_SubClass' + str_subclass_no
IMG_AUG_ROTATE_MODE = 1 #1:do flip,roate, 2:do flip 3: only translate_percent
MODEL_SAVE_DIR = '/tmp2/models_subclass/' + TRAIN_TYPE
#region read csv set weight_class_start, split train validation set
df = pd.read_csv(filename_csv_train)
NUM_CLASSES = df['labels'].nunique(dropna=True)
# len(df.loc[(df['labels'] == 0)])
weight_class_start = np.array([1, 9.5]) #23648, 2020
weight_class_end = np.array([1, 9.5])
balance_ratio = 0.93
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid,
shuffle=False)
#endregion
#region load pre-trained modal
model_name = 'ResNet448'
IMAGE_SIZE = 448
# model_file = '/home/ubuntu/dlp/deploy_models_2019/SubClass0_3/ResNet448-008-train0.7611_val0.840.hdf5'
model_file = '/home/ubuntu/dlp/deploy_models/DR0_DR1/ResNet448-007-train0.8310_val0.883.hdf5'
model1 = keras.models.load_model(model_file, compile=False)
model1.summary()
BATCH_SIZE_TRAIN = 32
BATCH_SIZE_VALID = 64
#endregion
#region save model dir and checkpointer
if not os.path.exists(MODEL_SAVE_DIR):
os.makedirs(MODEL_SAVE_DIR)
model_save_filepath = os.path.join(
MODEL_SAVE_DIR, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer = ModelCheckpoint(model_save_filepath,
verbose=1,
save_weights_only=False,
save_best_only=False)
#endregion
image_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)
#region train header layers
model_train_top = my_transfer_learning.convert_model_transfer(
model1, change_top=False, clsss_num=NUM_CLASSES)
#endregion
#region change learn rate(only fine tuning )
def scheduler(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print('set learning rate by lr.txt')
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
except Exception:
print('read lr-rate file error')
print('set learning rate automatically')
# 内置dictionary数据类型是无序的
dict_lr_rate = collections.OrderedDict()
dict_lr_rate['0'] = 1e-3 # 0.00001
dict_lr_rate['2'] = 2e-4
dict_lr_rate['4'] = 1e-4
dict_lr_rate['6'] = 3e-5
dict_lr_rate['8'] = 1e-5
dict_lr_rate['10'] = 1e-6 # 0.000001
dict_lr_rate['15'] = 6e-7
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
change_lr = keras.callbacks.LearningRateScheduler(scheduler)
# endregion
#region fine tuning all layers
model_fine_tune = my_transfer_learning.convert_trainable_all(
model_train_top)
if GPU_NUM > 1:
print('convert fine tuning model to Multiple GPU...')
model1 = ModelMGPU(model_fine_tune, GPU_NUM)
print('convert fine tuning model to Multiple GPU OK')
else:
model1 = model_fine_tune
op_adam_fine_tune = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model1.compile(loss='categorical_crossentropy',
optimizer=op_adam_fine_tune,
metrics=['acc'],
weighted_metrics=['acc'])
#region data generator
from imgaug import augmenters as iaa
sometimes = lambda aug: iaa.Sometimes(0.96, aug)
# sometimes1 = lambda aug: iaa.Sometimes(0.96, aug)
imgaug_train = iaa.Sequential([
# iaa.Crop(px=(0, 16)), # crop images from each side by 0 to 16px (randomly chosen)
# sometimes(iaa.CropAndPad(
# percent=(-0.04, 0.04),
# pad_mode=ia.ALL,
# pad_cval=(0, 255)
# )),
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.Flipud(0.2), # horizontally flip 50% of the images
# iaa.GaussianBlur(sigma=(0, 3.0)), # blur images with a sigma of 0 to 3.0,
# iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
# shuortcut for CropAndPad
# improve or worsen the contrast If PCH is set to true, the process happens channel-wise with possibly different S.
# sometimes1(iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5), ),
# change brightness of images (by -5 to 5 of original value)
# sometimes1(iaa.Add((-6, 6), per_channel=0.5),),
sometimes(
iaa.Affine(
# scale={"x": (0.92, 1.08), "y": (0.92, 1.08)},
# scale images to 80-120% of their size, individually per axis
# Translation Shifts the pixels of the image by the specified amounts in the x and y directions
translate_percent={
"x": (-0.02, 0.02),
"y": (-0.02, 0.02)
},
# translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -10 to +10 degrees
# shear=(-16, 16), # shear by -16 to +16 degrees
# order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
])
my_gen_train = My_images_weight_generator(
files=train_files,
labels=train_labels,
image_shape=image_shape,
weight_class_start=weight_class_start,
weight_class_end=weight_class_end,
balance_ratio=balance_ratio,
num_class=NUM_CLASSES,
imgaug_seq=imgaug_train,
batch_size=BATCH_SIZE_TRAIN)
my_gen_valid = My_images_generator(files=valid_files,
labels=valid_labels,
image_shape=image_shape,
num_output=NUM_CLASSES,
batch_size=BATCH_SIZE_VALID)
#endregion
history_fine_tuning = model1.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) /
BATCH_SIZE_TRAIN), #number of training batch
epochs=epoch_fine_tuning,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / BATCH_SIZE_VALID),
callbacks=[checkpointer, change_lr])
#endregion
K.clear_session() #release GPU memory
def train_task(filename_csv_train, filename_csv_valid, sub_class_no, model_no,
imagenet=True, epoch=None):
str_subclass_no = str(sub_class_no)
TRAIN_TYPE = 'DLP_SubClass' + str_subclass_no
IMG_AUG_ROTATE_MODE = 1 #1:do flip,roate, 2:do flip 3: only translate_percent
model_save_dir = MODEL_SAVE_DIR + TRAIN_TYPE
#region read csv set weight_class_start, split train validation set
df = pd.read_csv(filename_csv_train)
NUM_CLASSES = df['labels'].nunique(dropna=True)
# len(df.loc[(df['labels'] == 0)])
if sub_class_no == -1: # fundus images, ocular surface ,others
#191443,25278,568463
# 153332,25321,719762
weight_class_start = np.array([1.6, 9, 1])
weight_class_end = np.array([1.6, 9, 1])
weight_class_start = np.array([2.6, 11, 1])
weight_class_end = np.array([2.6, 11, 1])
balance_ratio = 0.93
if sub_class_no == 0.1: #Tessellated fundus 4567, 1324
weight_class_start = np.array([1, 2.4])
weight_class_end = np.array([1, 2.4])
balance_ratio = 0.93
if sub_class_no == 0.2: # Big Optic Cup 4567,6639
weight_class_start = np.array([1, 0.5])
weight_class_end = np.array([1, 0.5])
balance_ratio = 0.93
if sub_class_no == 1: #DR2,3 10284,2490, single label:9715, 2636
#8412,2241
weight_class_start = np.array([1, 2.4])
weight_class_end = np.array([1, 2.4])
#12949,3129, 8412,2241
# add test dataset 11156,2664
weight_class_start = np.array([1, 2.9])
weight_class_end = np.array([1, 2.9])
balance_ratio = 0.93
if sub_class_no == 2: #Data CRVO 2636,1527, single label:2548,1391
weight_class_start = np.array([1, 1.4])
weight_class_end = np.array([1, 1.4])
balance_ratio = 0.93
if sub_class_no == 5: #543, 683, single label:665,604
weight_class_start = np.array([1, 1])
weight_class_end = np.array([1, 1])
balance_ratio = 0.93
if sub_class_no == 10: # 5953 1449
weight_class_start = np.array([1, 2.6])
weight_class_end = np.array([1, 2.6])
balance_ratio = 0.93
if sub_class_no == 15: # single label:1523,136
weight_class_start = np.array([1, 6])
weight_class_end = np.array([1, 6])
balance_ratio = 0.93
if sub_class_no == 29: # Blur 16253 1814, single label:20882,1099, 12949,3129,
#train:17580, 1234, total:20709,1432
weight_class_start = np.array([1, 11])
weight_class_end = np.array([1, 11])
balance_ratio = 0.93
train_files, train_labels = my_data.get_images_labels(filename_csv_train, shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid, shuffle=False)
#endregion
#region load pre-trained modal
#region define and compile model
if model_no == 1:
model_name = 'Xception'
image_size = 299
image_shape = (image_size, image_size, 3)
print('loading model...')
if imagenet:
model1 = keras.applications.xception.Xception(include_top=True, weights='imagenet',
input_shape=image_shape)
else:
model_file = '/home/ubuntu/dlp/deploy_models_new/bigclasses_multilabels/class_weights5_0.2_0.7/Multi_label_Xception-015-train0.9671_val0.945.hdf5'
model1 = keras.models.load_model(model_file, compile=False)
print('loading model OK!')
if model_no == 2:
model_name = 'InceptionResNetV2'
image_size = 299
image_shape = (image_size, image_size, 3)
print('loading model...')
if imagenet:
model1 = keras.applications.inception_resnet_v2.InceptionResNetV2(include_top=True, weights='imagenet',
input_shape=image_shape)
else:
model_file = '/home/ubuntu/dlp/deploy_models_new/bigclasses_multilabels/class_weights5_0.2_0.7/Multi_label_InceptionResNetV2-006-train0.9674_val0.951.hdf5'
model1 = keras.models.load_model(model_file, compile=False)
print('loading model OK!')
if model_no == 3:
model_name = 'Inception_V3'
image_size = 299
image_shape = (image_size, image_size, 3)
print('loading model...')
model1 = keras.applications.inception_v3.InceptionV3(weights='imagenet', include_top=True,
input_shape=image_shape)
print('loading model OK!')
if model_no == 4:
model_name = 'MobilenetV2'
image_size = 224
image_shape = (image_size, image_size, 3)
print('loading model...')
from keras.applications import MobileNetV2
if imagenet:
model1 = MobileNetV2(include_top=True, input_shape=image_shape, weights='imagenet')
else:
model1 = MobileNetV2(include_top=True, input_shape=image_shape, weights=None, classes=NUM_CLASSES)
print('loading model OK!')
if model_no == 5:
model_name = 'ResNet448'
image_size = 448
image_shape = (image_size, image_size, 3)
from LIBS.CNN_Models import my_models
model1, IMAGE_SIZE, BATCH_SIZE_TRAIN = my_models.get_models(model_name, NUM_CLASSES)
print('loading model OK!')
model1 = my_transfer_learning.convert_model_transfer(model1, change_top=True, freeze_feature_extractor=False, clsss_num=NUM_CLASSES)
if GPU_NUM > 1:
print('convert model multiple GPU...')
# bug change to lambda layer, how to generate CAM?
# model1 = keras.utils.multi_gpu_model(model1, gpus=GPU_NUM)
model1 = ModelMGPU(model1, GPU_NUM)
print('convert model multiple GPU OK!')
op_adam = keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model1.compile(loss='categorical_crossentropy',
optimizer=op_adam, metrics=['acc'])
# endregion
BATCH_SIZE_TRAIN = 32
BATCH_SIZE_VALID = 32
#endregion
#region save model dir and checkpointer
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
model_save_filepath = os.path.join(model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer = ModelCheckpoint(model_save_filepath, verbose=1,
save_weights_only=False, save_best_only=False)
#endregion
def scheduler(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print('set learning rate by lr.txt')
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
except Exception:
print('read lr-rate file error')
print('set learning rate automatically')
# 内置dictionary数据类型是无序的
dict_lr_rate = collections.OrderedDict()
dict_lr_rate['0'] = 1e-3
dict_lr_rate['2'] = 3e-4
dict_lr_rate['3'] = 1e-4
dict_lr_rate['5'] = 1e-5
dict_lr_rate['7'] = 2e-6
dict_lr_rate['9'] = 1e-6 # 0.000001
dict_lr_rate['15'] = 6e-7
#only for ocular surface
dict_lr_rate['0'] = 1e-3
dict_lr_rate['1'] = 1e-4
dict_lr_rate['2'] = 1e-5
dict_lr_rate['3'] = 1e-6
dict_lr_rate['15'] = 6e-7
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
change_lr = keras.callbacks.LearningRateScheduler(scheduler)
#region train
if epoch is None:
if len(df) > 10000:
epoch = 20
elif len(df) > 5000:
epoch = 25
elif len(df) > 2000:
epoch = 30
else:
epoch = 40
#region data generator
from imgaug import augmenters as iaa
sometimes = lambda aug: iaa.Sometimes(0.96, aug)
# sometimes1 = lambda aug: iaa.Sometimes(0.96, aug)
imgaug_train_seq = iaa.Sequential([
# iaa.Crop(px=(0, 16)), # crop images from each side by 0 to 16px (randomly chosen)
# sometimes(iaa.CropAndPad(
# percent=(-0.04, 0.04),
# pad_mode=ia.ALL,
# pad_cval=(0, 255)
# )),
iaa.Fliplr(0.5), # horizontally flip 50% of the images
iaa.Flipud(0.2), # horizontally flip 50% of the images
# iaa.GaussianBlur(sigma=(0, 3.0)), # blur images with a sigma of 0 to 3.0,
# iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
# sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
# shuortcut for CropAndPad
# improve or worsen the contrast If PCH is set to true, the process happens channel-wise with possibly different S.
# sometimes1(iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5), ),
# change brightness of images (by -5 to 5 of original value)
# sometimes1(iaa.Add((-6, 6), per_channel=0.5),),
sometimes(iaa.Affine(
# scale={"x": (0.92, 1.08), "y": (0.92, 1.08)},
# scale images to 80-120% of their size, individually per axis
# Translation Shifts the pixels of the image by the specified amounts in the x and y directions
translate_percent={"x": (-0.02, 0.02), "y": (-0.02, 0.02)},
# translate by -20 to +20 percent (per axis)
rotate=(-10, 10), # rotate by -10 to +10 degrees
# shear=(-16, 16), # shear by -16 to +16 degrees
# order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
# cval=(0, 255), # if mode is constant, use a cval between 0 and 255
# mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
])
my_gen_train = My_images_weight_generator(files=train_files, labels=train_labels, image_shape=image_shape,
weight_class_start=weight_class_start, weight_class_end=weight_class_end, balance_ratio=balance_ratio,
num_class=NUM_CLASSES, imgaug_seq=imgaug_train_seq, batch_size=BATCH_SIZE_TRAIN)
my_gen_valid = My_images_generator(files=valid_files, labels=valid_labels, image_shape=image_shape,
num_output=NUM_CLASSES, batch_size=BATCH_SIZE_VALID)
#endregion
history_train = model1.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) / BATCH_SIZE_TRAIN), #number of training batch
epochs=epoch,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / BATCH_SIZE_VALID),
callbacks=[checkpointer, change_lr]
)
def train_task(model_name,
model_file,
image_size,
dict_lr_rate=None,
change_top=True,
epoch_top=2,
epoch_finetuning=15,
batch_size_train=32,
batch_size_valid=32):
#region load pre_trained Model, model_name(callback model save will use)
image_shape = (image_size, image_size, 3)
print('load pre-trained model...')
#mobilenetv2
from keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope({
'relu6': keras.layers.ReLU(6.),
'DepthwiseConv2D': keras.layers.DepthwiseConv2D
}):
model1 = keras.models.load_model(model_file, compile=False)
print('load pre-trained model OK')
#endregion
#region callback function savemodel, change learn rate
if not os.path.exists(MODEL_SAVE_DIR):
os.makedirs(MODEL_SAVE_DIR)
model_save_filepath = os.path.join(
MODEL_SAVE_DIR, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer = ModelCheckpoint(model_save_filepath,
verbose=1,
save_weights_only=False,
save_best_only=False)
#change lr-rate only for fine-tuning
if dict_lr_rate is None:
dict_lr_rate = collections.OrderedDict()
dict_lr_rate['0'] = 1e-5 # 0.00001
dict_lr_rate['3'] = 3e-6 # 0.000003
dict_lr_rate['5'] = 1e-6 # 0.000001
dict_lr_rate['9'] = 6e-7
def scheduler(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print('set learning rate by lr.txt')
except Exception:
print('read lr-rate file error')
print('set learning rate automatically')
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate: %f" % (epoch, lr_rate))
K.set_value(model1.optimizer.lr, lr_rate)
return K.get_value(model1.optimizer.lr)
change_lr = keras.callbacks.LearningRateScheduler(scheduler)
# endregion
#region data generator implement dynamic resampling
my_gen = My_images_weight_generator(files=train_files,
labels=train_labels,
weight_class_start=weight_class_start,
weight_class_end=weight_class_end,
balance_ratio=balance_ratio,
num_class=NUM_CLASSES,
img_aug_mode=img_aug_mode,
batch_size=batch_size_train,
image_shape=image_shape)
#endregion
#region cnvert model(top layer) and train top layers(if epoch_top >0)
model_train_top = my_transfer_learning.convert_model_transfer(
model1,
change_top=change_top,
clsss_num=NUM_CLASSES,
activation_function='SoftMax',
freeze_feature_extractor=True)
if GPU_NUM > 1:
print('convert base model to Multiple GPU...')
model1 = ModelMGPU(model_train_top, GPU_NUM)
print('convert base model to Multiple GPU OK')
else:
model1 = model_train_top
if epoch_top > 0:
op_adam_train_top = keras.optimizers.Adam(lr=1e-4,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
model1.compile(loss='categorical_crossentropy',
optimizer=op_adam_train_top,
metrics=['acc'])
history_top = model1.fit_generator(
my_gen.gen(),
steps_per_epoch=math.ceil(
len(train_files) /
batch_size_train), #number of training batch
epochs=epoch_top,
validation_data=my_Generator(valid_files,
valid_labels,
image_shape=image_shape,
batch_size=batch_size_valid,
num_class=NUM_CLASSES,
train_or_valid='valid'),
validation_steps=math.ceil(len(valid_files) / batch_size_valid))
#endregion
#region fine tuning all layers
model_fine_tune = my_transfer_learning.convert_trainable_all(
model_train_top)
if GPU_NUM > 1:
print('convert fine-tuning model to Multiple GPU...')
model_fine_tune = ModelMGPU(model_fine_tune, GPU_NUM)
print('convert fine-tuning model to Multiple GPU OK')
op_adam_fine_tune = keras.optimizers.Adam(lr=1e-5,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model1.compile(loss='categorical_crossentropy',
optimizer=op_adam_fine_tune,
metrics=['acc'])
history_fine_tuning = model_fine_tune.fit_generator(
my_gen.gen(),
steps_per_epoch=math.ceil(len(train_files) /
batch_size_train), #number of training batch
epochs=epoch_finetuning,
validation_data=my_Generator(valid_files,
valid_labels,
image_shape=image_shape,
batch_size=batch_size_valid,
num_class=NUM_CLASSES,
train_or_valid='valid'),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer, change_lr])
#endregion
K.clear_session() #release GPU memory
def train_task(filename_csv_train,
filename_csv_valid,
num_classes,
model_name,
model1,
image_size,
add_top=False,
change_top=True,
freeze_layes_num=None,
class_weights=None,
inter_class_ratio=None,
positive_weight_ratio=4,
exclusion_loss_ratio=0,
imgaug_train_seq=None,
epoch_finetuning=15,
dict_lr_finetuning=None,
batch_size_train=32,
batch_size_valid=32,
smooth_factor=0,
model_save_dir='/tmp'):
#region load csv files
print('loading csv data')
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(filename_csv_valid)
# region get the number of labels in every class
df = pd.read_csv(filename_csv_train)
# NUM_CLASSES = df['labels'].nunique(dropna=True) #only work for single label
LIST_CLASS_SAMPLES_NUM = [0 for _ in range(num_classes)]
for _, row in df.iterrows():
labels = str(row["labels"]) # single label may be int type
list_labels = labels.split('_')
for label in list_labels:
if label == '':
continue
assert label.isdigit(), 'Error label!'
LIST_CLASS_SAMPLES_NUM[int(label)] += 1
# endregion
# region multiple labels convert to simgle label(smallest class num) in order to dynamic resampling
train_labels_single = [0 for _ in range(len(train_labels))]
for i, labels in enumerate(train_labels):
labels = str(labels) # single label may be int type
list_labels = labels.split('_')
label_current = None
for label in list_labels:
if label == '':
continue
if label_current is None: # it is the first label
label_current = label
elif LIST_CLASS_SAMPLES_NUM[int(label)] < LIST_CLASS_SAMPLES_NUM[
int(label_current)]:
label_current = label # current label's class contain smaller number of images than before
train_labels_single[i] = label_current
# endregion
print('loading csv data complete!')
#endregion
#region data generator
image_shape = (image_size, image_size, 3)
file_weight_power = os.path.join(sys.path[0], 'weight_power.txt')
my_gen_train = My_gen_weight_multi_labels(
files=train_files,
labels=train_labels,
labels_single=train_labels_single,
image_shape=image_shape,
file_weight_power=file_weight_power,
list_class_samples_num=LIST_CLASS_SAMPLES_NUM,
num_class=num_classes,
smooth_factor=smooth_factor,
imgaug_seq=imgaug_train_seq,
batch_size=batch_size_train)
my_gen_valid = My_images_generator(files=valid_files,
labels=valid_labels,
image_shape=image_shape,
multi_labels=True,
num_output=num_classes,
batch_size=batch_size_valid)
#endregion
#region custom loss function
if class_weights is None:
if inter_class_ratio is not None:
class_positive_weights = op_class_weight(
LIST_CLASS_SAMPLES_NUM, weight_power=inter_class_ratio)
class_positive_weights = np.array(class_positive_weights)
# class_samples_weights[0] /= (3) #class0 Normal
class_positive_weights *= positive_weight_ratio
else:
class_positive_weights = [1 for _ in range(num_classes)]
class_positive_weights = np.array(class_positive_weights)
print(np.round(class_positive_weights, 2))
class_weights = []
for class_weight1 in class_positive_weights:
class_weights.append([1,
class_weight1]) # sigmoid : 0:1, 1:weight1
class_weights = np.array(class_weights)
custom_loss_function = get_weighted_binary_crossentropy(
class_weights, exclusion_loss_ratio)
#endregion
# region loading and converting model
if isinstance(model1, str):
print('loading model...')
model1 = keras.models.load_model(model1, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(model1,
num_output=num_classes,
activation_function='Sigmoid')
model_train_top = my_transfer_learning.convert_model_transfer(
model1,
change_top=change_top,
clsss_num=num_classes,
activation_function='Sigmoid',
freeze_feature_extractor=True,
freeze_layes_num=freeze_layes_num)
model_train_top.summary()
if GPU_NUM > 1:
print('convert model to Multiple GPU...')
model_train_top = ModelMGPU(model_train_top, GPU_NUM)
print('convert model to Multiple GPU OK')
op_adam_train_top = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
model_train_top.compile(loss=custom_loss_function,
optimizer=op_adam_train_top,
metrics=['acc'])
from LIBS.CNN_Models.optimization.lookahead import Lookahead
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_train_top)
#endregion
#region fine tuning all layers
model_fine_tune = my_transfer_learning.convert_trainable_all(
model_train_top)
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
save_filepath_finetuning = os.path.join(
model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_finetuning = ModelCheckpoint(save_filepath_finetuning,
verbose=1,
save_weights_only=False,
save_best_only=False)
def scheduler_fine_tuning(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') # 删除换行符
lr_rate = float(line)
print("epoch：%d, current learn rate by lr.txt: %f" %
(epoch, lr_rate))
K.set_value(model_fine_tune.optimizer.lr, lr_rate)
except Exception:
if dict_lr_finetuning is not None:
dict_lr_rate = dict_lr_finetuning
for (k, v) in dict_lr_rate.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, current learn rate automatically: %f" %
(epoch, lr_rate))
K.set_value(model_fine_tune.optimizer.lr, lr_rate)
return K.get_value(model_fine_tune.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(
scheduler_fine_tuning)
history_finetuning = model_fine_tune.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) /
batch_size_train), #number of training batch
epochs=epoch_finetuning,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_finetuning, change_lr_finetuning])
#endregion
K.clear_session() #release GPU memory
def train_task_two_steps(model1, filename_csv_train, FILENAME_CSV_VALID, filename_csv_test=None,
input_shape=(299, 299, 3), imgaug_train_seq=None,
add_top=False, change_top=True, freeze_layes_num=None,
optimizer="adam", lookahead=True,
epoch_traintop=0, epoch_finetuning=0,
dict_lr_traintop=None, dict_lr_finetuning=None,
batch_size_train=32, batch_size_valid=64,
label_smoothing=0, class_weight=None,
weight_class_start=None, weight_class_end=None, balance_ratio=None,
model_save_dir='/tmp', model_name='model1',
gpu_num=1, verbose=1,
config_file_realtime='config_file_realtime.json'):
#region read csv, split train validation set
df = pd.read_csv(filename_csv_train)
NUM_CLASSES = df['labels'].nunique(dropna=True)
train_files, train_labels = my_data.get_images_labels(filename_csv_train, shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(FILENAME_CSV_VALID)
#endregion
#region load , convert train_top model
if isinstance(model1, str):
print('loading model...')
model1 = keras.models.load_model(model1, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(model1, num_output=NUM_CLASSES, activation_function='SoftMax')
model_traintop = my_transfer_learning.convert_model_transfer(model1, clsss_num=NUM_CLASSES,
change_top=change_top, activation_function='SoftMax',
freeze_feature_extractor=True, freeze_layes_num=freeze_layes_num)
if gpu_num > 1:
print('convert base model to Multiple GPU...')
model_traintop = ModelMGPU(model_traintop, gpu_num)
print('convert base top model to Multiple GPU OK')
assert optimizer in ['adam', 'SGD', 'adabound'], 'optimizer type error'
#endregion
#region data generator
if weight_class_start is not None:
my_gen_train = My_images_weight_generator(files=train_files, labels=train_labels, image_shape=input_shape,
weight_class_start=weight_class_start, weight_class_end=weight_class_end, balance_ratio=balance_ratio,
num_class=NUM_CLASSES, imgaug_seq=imgaug_train_seq, batch_size=batch_size_train,
label_smoothing=label_smoothing)
else:
my_gen_train = My_images_generator(files=train_files, labels=train_labels, image_shape=input_shape,
imgaug_seq=imgaug_train_seq, num_output=NUM_CLASSES, batch_size=batch_size_train)
my_gen_valid = My_images_generator(files=valid_files, labels=valid_labels,
image_shape=input_shape, num_output=NUM_CLASSES, batch_size=batch_size_valid)
if filename_csv_test is not None:
test_files, test_labels = my_data.get_images_labels(filename_csv_test)
my_gen_test = My_images_generator(files=test_files, labels=test_labels,
image_shape=input_shape, num_output=NUM_CLASSES, batch_size=batch_size_valid)
#endregion
# region save model dir and checkpointer
os.makedirs(model_save_dir, exist_ok=True)
save_filepath_traintop = os.path.join(model_save_dir, model_name + "-traintop-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_traintop = ModelCheckpoint(save_filepath_traintop,
verbose=1, save_weights_only=False, save_best_only=False)
save_filepath_finetuning = os.path.join(model_save_dir, model_name + "-{epoch:03d}-{val_acc:.3f}.hdf5")
checkpointer_finetuning = ModelCheckpoint(save_filepath_finetuning,
verbose=1, save_weights_only=False, save_best_only=False)
# endregion
#region computer validation confusion matrix
class My_callback(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['epoch_compute_cf_train'] == 1:
compute_cf_train = True
else:
compute_cf_train = False
if data['epoch_compute_cf_valid'] == 1:
compute_cf_valid = True
else:
compute_cf_valid = False
if data['epoch_compute_cf_test'] == 1:
compute_cf_test = True
else:
compute_cf_test = False
except:
print('read realtime helper file error!')
compute_cf_train = True
compute_cf_valid = True
compute_cf_test = True
if compute_cf_train:
print('calculate confusion matrix of training dataset...')
# do not use img augmentation
my_gen_train_test = My_images_generator(files=train_files, labels=train_labels,
image_shape=input_shape, num_output=NUM_CLASSES,
batch_size=batch_size_train)
i = 0
for x_train, y_train in my_gen_train_test.gen():
probabilities = self.model.predict(x_train)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(train_files) / batch_size_train):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, NUM_CLASSES)]
confusion_matrix_train = sk_confusion_matrix(train_labels, y_preds, labels=labels)
print(confusion_matrix_train)
if compute_cf_valid:
print('calculate confusion matrix of validation dataset...')
i = 0
for x_valid, y_valid in my_gen_valid.gen():
probabilities = self.model.predict(x_valid)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(valid_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, NUM_CLASSES)]
confusion_matrix_valid = sk_confusion_matrix(valid_labels, y_preds, labels=labels)
print(confusion_matrix_valid)
if compute_cf_test:
print('calculate confusion matrix of test dataset...')
i = 0
for x_test, y_test in my_gen_test.gen():
probabilities = self.model.predict(x_test)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(test_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
labels = [x for x in range(0, NUM_CLASSES)]
confusion_matrix_test = sk_confusion_matrix(test_labels, y_preds, labels=labels)
print(confusion_matrix_test)
#endregion
my_callback = My_callback()
#endregion
#region train header layers
if epoch_traintop > 0:
if optimizer == 'adam':
op_train_top = keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if optimizer == 'SGD':
op_train_top = keras.optimizers.sgd(lr=1e-3, momentum=0.9, nesterov=True)
if optimizer == 'adabound':
op_train_top = AdaBound(lr=1e-03, final_lr=0.1, gamma=1e-03, amsbound=False)
model_traintop.compile(loss='categorical_crossentropy',
optimizer=op_train_top, metrics=['acc'])
# optimizer = op_train_top, metrics = ['acc', sensitivity, specificity])
if lookahead:
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_traintop)
if epoch_traintop is None:
if len(df) > 10000:
epoch_traintop = 5
elif len(df) > 5000:
epoch_traintop = 8
elif len(df) > 2000:
epoch_traintop = 10
else:
epoch_traintop = 15
def scheduler_traintop(epoch):
if optimizer == 'adabound':
return K.get_value(model_traintop.optimizer.lr)
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['lr_rate'] > 0:
lr_rate = data['lr_rate']
print("epoch：%d, current learn rate: %f by realtime helper file" % (epoch, lr_rate))
K.set_value(model_traintop.optimizer.lr, lr_rate)
return K.get_value(model_traintop.optimizer.lr)
except Exception:
print('read realtime helper file error!')
if dict_lr_traintop is not None:
for (k, v) in dict_lr_traintop.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, set learn rate: %f according to pre-defined policy." % (epoch, lr_rate))
K.set_value(model_traintop.optimizer.lr, lr_rate)
return K.get_value(model_traintop.optimizer.lr)
change_lr_traintop = keras.callbacks.LearningRateScheduler(scheduler_traintop)
history_top = model_traintop.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) / batch_size_train), #number of training batch
epochs=epoch_traintop,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_traintop, change_lr_traintop, my_callback],
class_weight=class_weight)
#endregion
#region fine tuning all layers
if epoch_finetuning > 0:
model_finetuning = my_transfer_learning.convert_trainable_all(model_traintop)
if optimizer == 'adam':
op_finetuning = keras.optimizers.Adam(lr=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if optimizer == 'SGD':
op_finetuning = keras.optimizers.sgd(lr=1e-5, momentum=0.9, nesterov=True)
if optimizer == 'adabound':
op_finetuning = AdaBound(lr=1e-04, final_lr=0.01, gamma=1e-03, amsbound=False)
model_finetuning.compile(loss='categorical_crossentropy',
optimizer=op_finetuning, metrics=['acc'])
# optimizer = op_finetuning, metrics = ['acc', sensitivity, specificity])
if lookahead:
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_finetuning)
if epoch_finetuning is None:
if len(df) > 10000:
epoch_finetuning = 20
elif len(df) > 5000:
epoch_finetuning = 25
elif len(df) > 2000:
epoch_finetuning = 30
else:
epoch_finetuning = 40
def scheduler_finetuning(epoch):
if optimizer == 'adabound':
return K.get_value(model_finetuning.optimizer.lr)
try:
with open(config_file_realtime, 'r') as json_file:
data = json.load(json_file)
if data['lr_rate'] > 0:
lr_rate = data['lr_rate']
print("epoch：%d, current learn rate: %f by realtime helper file" % (epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
return K.get_value(model_finetuning.optimizer.lr)
except Exception:
print('read realtime helper file error!')
if dict_lr_finetuning is not None:
for (k, v) in dict_lr_finetuning.items():
if epoch >= int(k):
lr_rate = v
print("epoch：%d, set learn rate: %f according to pre-defined policy." % (epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
return K.get_value(model_finetuning.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(scheduler_finetuning)
history_finetuning = model_finetuning.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(len(train_files) / batch_size_train), # number of training batch
epochs=epoch_finetuning,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_finetuning, change_lr_finetuning, my_callback],
class_weight=class_weight
)
#endregion
K.clear_session() #release GPU memory
def train_task_two_steps(model1,
filename_csv_train,
FILENAME_CSV_VALID,
filename_csv_test=None,
image_size=299,
imgaug_train_seq=None,
num_output=1,
add_top=False,
change_top=True,
freeze_layes_num=None,
epoch_traintop=None,
epoch_finetuning=None,
dict_lr_traintop=None,
dict_lr_finetuning=None,
batch_size_train=32,
batch_size_valid=64,
model_save_dir='/tmp',
model_name='model1',
compute_cf_train=False,
compute_cf_valid=False,
compute_cf_test=False,
gpu_num=1,
verbose=1):
#region read csv, split train validation set
df = pd.read_csv(filename_csv_train
) #get number of train samples, set epoch automaticaly
train_files, train_labels = my_data.get_images_labels(filename_csv_train,
shuffle=True)
valid_files, valid_labels = my_data.get_images_labels(FILENAME_CSV_VALID)
#endregion
#region load , convert and compile train_top model
if isinstance(model1, str):
print('loading model...')
model1 = keras.models.load_model(model1, compile=False)
print('loading model complete!')
if add_top:
model1 = my_transfer_learning.add_top(model1,
num_output=num_output,
activation_function='SoftMax')
model_traintop = my_transfer_learning.convert_model_transfer(
model1,
clsss_num=num_output,
change_top=change_top,
activation_function='Regression',
freeze_feature_extractor=True,
freeze_layes_num=freeze_layes_num)
if gpu_num > 1:
print('convert base model to Multiple GPU...')
model_traintop = ModelMGPU(model_traintop, gpu_num)
print('convert base top model to Multiple GPU OK')
#endregion
#region data generator
image_shape = (image_size, image_size, 3)
my_gen_train = My_images_generator(files=train_files,
labels=train_labels,
regression=True,
imgaug_seq=imgaug_train_seq,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_train)
my_gen_valid = My_images_generator(files=valid_files,
labels=valid_labels,
regression=True,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_valid)
if compute_cf_test:
test_files, test_labels = my_data.get_images_labels(filename_csv_test)
my_gen_test = My_images_generator(files=test_files,
labels=test_labels,
regression=True,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_valid)
#endregion
# region save model dir and checkpointer
os.makedirs(model_save_dir, exist_ok=True)
save_filepath_traintop = os.path.join(
model_save_dir, model_name +
"-{epoch:03d}--{val_loss:.1f}---{val_mean_absolute_error:.1f}---{mean_squared_error:.1f}-{mean_absolute_error:.1f}.hdf5"
)
checkpointer_traintop = ModelCheckpoint(save_filepath_traintop,
save_weights_only=False,
save_best_only=False)
save_filepath_finetuning = os.path.join(
model_save_dir, model_name +
"-{epoch:03d}--{val_loss:.1f}---{val_mean_absolute_error:.1f}---{mean_squared_error:.1f}-{mean_absolute_error:.1f}.hdf5"
)
checkpointer_finetuning = ModelCheckpoint(save_filepath_finetuning,
save_weights_only=False,
save_best_only=False)
# endregion
#region computer validation confusion matrix
class My_callback(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
if compute_cf_train:
print('calculate confusion matrix of training dataset...')
my_gen_train_no_imgaug = My_images_generator(
files=train_files,
labels=train_labels,
regression=True,
imgaug_seq=imgaug_train_seq,
image_shape=image_shape,
num_output=num_output,
batch_size=batch_size_train)
i = 0
for x_train, y_train in my_gen_train_no_imgaug.gen():
probabilities = self.model.predict(x_train)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(train_files) / batch_size_train):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
# endregion
if compute_cf_valid:
print('calculate confusion matrix of validation dataset...')
#use the same my_gen_valid
i = 0
for x_valid, y_valid in my_gen_valid.gen():
probabilities = self.model.predict(x_valid)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(valid_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
if compute_cf_test:
print('calculate confusion matrix of test dataset...')
i = 0
for x_test, y_test in my_gen_test.gen():
probabilities = self.model.predict(x_test)
if i == 0:
probs = probabilities
else:
probs = np.vstack((probs, probabilities))
i += 1
if i == math.ceil(len(test_files) / batch_size_valid):
break
y_preds = probs.argmax(axis=-1)
y_preds = y_preds.tolist()
#endregion
my_callback = My_callback()
#endregion
#region train header layers
if epoch_traintop > 0:
op_adam_train_top = keras.optimizers.Adam(lr=1e-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
op_adam_train_top.compile(loss='mse',
optimizer=op_adam_train_top,
metrics=['mae', 'mse'])
from LIBS.CNN_Models.optimization.lookahead import Lookahead
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_traintop)
if epoch_traintop is None:
if len(df) > 10000:
epoch_traintop = 5
elif len(df) > 5000:
epoch_traintop = 8
elif len(df) > 2000:
epoch_traintop = 10
else:
epoch_traintop = 15
def scheduler_traintop(epoch):
if dict_lr_traintop is not None:
dict_lr = dict_lr_traintop
for (k, v) in dict_lr.items():
if epoch >= int(k):
lr_rate = v
print(
"epoch：%d, set learn rate: %f according to pre-defined policy."
% (epoch, lr_rate))
K.set_value(model_traintop.optimizer.lr, lr_rate)
return K.get_value(model_traintop.optimizer.lr)
change_lr_traintop = keras.callbacks.LearningRateScheduler(
scheduler_traintop)
history_top = model_traintop.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(
len(train_files) /
batch_size_train), #number of training batch
epochs=epoch_traintop,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[checkpointer_traintop, change_lr_traintop, my_callback])
#endregion
#region fine tuning all layers
if epoch_finetuning > 0:
model_finetuning = my_transfer_learning.convert_trainable_all(
model_traintop)
op_adam_finetuning = keras.optimizers.Adam(lr=1e-5,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
model_finetuning.compile(loss='mse',
optimizer=op_adam_finetuning,
metrics=['mae', 'mse'])
from LIBS.CNN_Models.optimization.lookahead import Lookahead
lookahead = Lookahead(k=5, alpha=0.5)
lookahead.inject(model_finetuning)
if epoch_finetuning is None:
if len(df) > 10000:
epoch_finetuning = 20
elif len(df) > 5000:
epoch_finetuning = 25
elif len(df) > 2000:
epoch_finetuning = 30
else:
epoch_finetuning = 40
def scheduler_finetuning(epoch):
try:
file_object = open('lr.txt')
line = file_object.readline()
file_object.close()
line = line.strip('\n') #删除换行符
lr_rate = float(line)
print("epoch：%d, set learning rate: %f by lr.txt" %
(epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
except Exception:
if dict_lr_finetuning is not None:
dict_lr = dict_lr_finetuning
for (k, v) in dict_lr.items():
if epoch >= int(k):
lr_rate = v
print(
"epoch：%d, set learn rate: %f according to pre-defined policy."
% (epoch, lr_rate))
K.set_value(model_finetuning.optimizer.lr, lr_rate)
return K.get_value(model_finetuning.optimizer.lr)
change_lr_finetuning = keras.callbacks.LearningRateScheduler(
scheduler_finetuning)
history_finetuning = model_finetuning.fit_generator(
my_gen_train.gen(),
steps_per_epoch=math.ceil(
len(train_files) /
batch_size_train), # number of training batch
epochs=epoch_finetuning,
verbose=verbose,
validation_data=my_gen_valid.gen(),
validation_steps=math.ceil(len(valid_files) / batch_size_valid),
callbacks=[
checkpointer_finetuning, change_lr_finetuning, my_callback
])
#endregion
K.clear_session() #release GPU memory