def EffNetB4(freeze=False):
model = EfficientNetB4(input_shape=(380, 380, 3),
weights='imagenet',
include_top=False,
pooling=None)
for layers in model.layers:
layers.trainable = not freeze
inputs = model.input
x = model.output
x = GlobalAveragePooling2D()(x)
out_layer = Dense(1, activation=None,
name='normal_regressor')(Dropout(0.4)(x))
model = Model(inputs, out_layer)
return model
def EfficientNet_model(input_shape=(None, None, 3),
num_classes=1,
weight='imagenet',
dropout_rate=0.5):
backbone = EfficientNetB4(weights=weight,
include_top=False,
input_shape=input_shape)
img_input = backbone.input
x = backbone.layers[342].output # consider 257, 154
x = LeakyReLU(alpha=0.1)(x)
x = Dropout(dropout_rate)(x)
x = Conv2D(960, (3, 3), activation=None, padding="same")(x)
x = LeakyReLU(alpha=0.1)(x)
x = BatchNormalization()(x)
x = Conv2D(960, (3, 3), activation=None, padding="same")(x)
x = LeakyReLU(alpha=0.1)(x)
x = BatchNormalization()(x)
x = Dropout(dropout_rate)(x)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.7)(x)
if num_classes == 1:
predictions = Dense(num_classes + 1,
activation='sigmoid',
name='predictions')(x)
else:
predictions = Dense(num_classes + 1,
activation='softmax',
name='predictions')(x)
model = Model(inputs=img_input, outputs=predictions)
if weight is not 'imagenet' and not None:
print('loading model weights from pretrained...')
model.load_weights(weight)
return model
def get_fe(fe, input_image):
if fe == 'effnetb0':
return EfficientNetB0(input_tensor=input_image, include_top=False), [
'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1'
]
elif fe == 'effnetb1':
return EfficientNetB1(input_tensor=input_image, include_top=False), [
'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1'
]
elif fe == 'effnetb2':
return EfficientNetB2(input_tensor=input_image, include_top=False), [
'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1'
]
elif fe == 'effnetb3':
return EfficientNetB3(input_tensor=input_image, include_top=False), [
'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1'
]
elif fe == 'effnetb4':
return EfficientNetB4(input_tensor=input_image, include_top=False), [
'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1'
]
elif fe == 'effnetb5':
return EfficientNetB5(input_tensor=input_image, include_top=False), [
'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1'
]
elif fe == 'd53':
return d53(input_image)
elif fe == 'mnetv2':
mnet = MobileNetV2(input_tensor=input_image, weights='imagenet')
return mnet, [
'out_relu', 'block_13_expand_relu', 'block_6_expand_relu'
]
elif fe == 'mnet':
mnet = MobileNet(input_tensor=input_image, weights='imagenet')
return mnet, ['conv_pw_13_relu', 'conv_pw_11_relu', 'conv_pw_5_relu']
elif fe == 'r50':
r50 = ResNet50(input_tensor=input_image, weights='imagenet')
return r50, ['activation_49', 'activation_40', 'activation_22']
raise ValueError('Pls put the correct fe')
def train(dataset='Fish4Knowledge',
cnn='resnet50',
batch_size=32,
epochs=50,
image_size=32,
eps=0.01):
"""
Train one checkpoint with data augmentation: random padding+cropping and horizontal flip
:param args:
:return:
"""
print(
'Dataset: %s, CNN: %s, loss: adv, epsilon: %.3f batch: %s, epochs: %s'
% (dataset, cnn, eps, batch_size, epochs))
IMAGE_SIZE = image_size
INPUT_SHAPE = (image_size, image_size, 3)
# find image folder: images are distributed in class subfolders
if dataset == 'Fish4Knowledge':
# image_path = '/home/xingjun/datasets/Fish4Knowledge/fish_image'
image_path = '/data/cephfs/punim0619/Fish4Knowledge/fish_image'
images, labels, images_val, labels_val = get_Fish4Knowledge(
image_path, train_ratio=0.8)
elif dataset == 'QUTFish':
# image_path = '/home/xingjun/datasets/QUT_fish_data'
image_path = '/data/cephfs/punim0619/QUT_fish_data'
images, labels, images_val, labels_val = get_QUTFish(image_path,
train_ratio=0.8)
elif dataset == 'WildFish':
# image_pathes = ['/home/xingjun/datasets/WildFish/WildFish_part1',
# '/home/xingjun/datasets/WildFish/WildFish_part2',
# '/home/xingjun/datasets/WildFish/WildFish_part3',
# '/home/xingjun/datasets/WildFish/WildFish_part4']
image_pathes = [
'/data/cephfs/punim0619/WildFish/WildFish_part1',
'/data/cephfs/punim0619/WildFish/WildFish_part2',
'/data/cephfs/punim0619/WildFish/WildFish_part3',
'/data/cephfs/punim0619/WildFish/WildFish_part4'
]
images, labels, images_val, labels_val = get_WildFish(image_pathes,
train_ratio=0.8)
# images, labels, images_val, labels_val = get_imagenet_googlesearch_data(image_path, num_class=NUM_CLASS)
num_classes = len(np.unique(labels))
num_images = len(images)
num_images_val = len(images_val)
print('Train: classes: %s, images: %s, val images: %s' %
(num_classes, num_images, num_images_val))
global current_index
current_index = 0
# dynamic loading a batch of data
def get_batch():
index = 1
global current_index
B = np.zeros(shape=(batch_size, IMAGE_SIZE, IMAGE_SIZE, 3))
L = np.zeros(shape=(batch_size))
while index < batch_size:
try:
img = load_img(images[current_index],
target_size=(IMAGE_SIZE, IMAGE_SIZE))
img = img_to_array(img)
img /= 255.
# if cnn == 'ResNet50': # imagenet pretrained
# mean = np.array([0.485, 0.456, 0.406])
# std = np.array([0.229, 0.224, 0.225])
# img = (img - mean)/std
## data augmentation
# random width and height shift
img = random_shift(img, 0.2, 0.2)
# random rotation
img = random_rotation(img, 10)
# random horizental flip
flip_horizontal = (np.random.random() < 0.5)
if flip_horizontal:
img = flip_axis(img, axis=1)
# # random vertical flip
# flip_vertical = (np.random.random() < 0.5)
# if flip_vertical:
# img = flip_axis(img, axis=0)
# #cutout
# eraser = get_random_eraser(v_l=0, v_h=1, pixel_level=False)
# img = eraser(img)
B[index] = img
L[index] = labels[current_index]
index = index + 1
current_index = current_index + 1
except:
traceback.print_exc()
# print("Ignore image {}".format(images[current_index]))
current_index = current_index + 1
# B = np.rollaxis(B, 3, 1)
return B, np_utils.to_categorical(L, num_classes)
global val_current_index
val_current_index = 0
# dynamic loading a batch of validation data
def get_val_batch():
index = 1
B = np.zeros(shape=(batch_size, IMAGE_SIZE, IMAGE_SIZE, 3))
L = np.zeros(shape=(batch_size))
global val_current_index
while index < batch_size:
try:
img = load_img(images_val[val_current_index],
target_size=(IMAGE_SIZE, IMAGE_SIZE))
img = img_to_array(img)
img /= 255.
# if cnn == 'ResNet50': # imagenet pretrained
# mean = np.array([0.485, 0.456, 0.406])
# std = np.array([0.229, 0.224, 0.225])
# img = (img - mean)/std
B[index] = img
L[index] = labels_val[val_current_index]
index = index + 1
val_current_index = val_current_index + 1
except:
traceback.print_exc()
# print("Ignore image {}".format(images[val_current_index]))
val_current_index = val_current_index + 1
# B = np.rollaxis(B, 3, 1)
return B, np_utils.to_categorical(L, num_classes)
# load checkpoint
if cnn == 'ResNet18':
base_model = ResNet18(input_shape=INPUT_SHAPE,
classes=num_classes,
include_top=False)
elif cnn == 'ResNet34':
base_model = ResNet34(input_shape=INPUT_SHAPE,
classes=num_classes,
include_top=False)
elif cnn == 'ResNet50':
base_model = ResNet50(include_top=False,
weights='imagenet',
input_shape=INPUT_SHAPE)
elif cnn == 'EfficientNetB1':
base_model = EfficientNetB1(input_shape=INPUT_SHAPE,
classes=num_classes,
include_top=False,
backend=keras.backend,
layers=keras.layers,
models=keras.models,
utils=keras.utils)
elif cnn == 'EfficientNetB2':
base_model = EfficientNetB2(input_shape=INPUT_SHAPE,
classes=num_classes,
include_top=False,
backend=keras.backend,
layers=keras.layers,
models=keras.models,
utils=keras.utils)
elif cnn == 'EfficientNetB3':
base_model = EfficientNetB3(input_shape=INPUT_SHAPE,
classes=num_classes,
include_top=False,
backend=keras.backend,
layers=keras.layers,
models=keras.models,
utils=keras.utils)
elif cnn == 'EfficientNetB4':
base_model = EfficientNetB4(input_shape=INPUT_SHAPE,
classes=num_classes,
include_top=False,
backend=keras.backend,
layers=keras.layers,
models=keras.models,
utils=keras.utils)
else:
warnings.warn("Error: unrecognized dataset!")
return
x = base_model.output
x = Flatten()(x)
x = Dense(num_classes, name='dense')(x)
output = Activation('softmax')(x)
model = Model(input=base_model.input, output=output, name=cnn)
# model.summary()
loss = cross_entropy
base_lr = 1e-2
sgd = SGD(lr=base_lr, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss=loss, optimizer=sgd, metrics=['accuracy'])
# AdaFGSM attack for AdvFish training
attack = AdaFGSM(model,
epsilon=float(eps),
random_start=True,
loss_func='xent',
clip_min=0.,
clip_max=1.)
# PGD attack for AdvFish training, it reduces to FGSM when set nb_iter=1
# attack = LinfPGDAttack(model,
# epsilon=float(eps),
# eps_iter=float(eps),
# nb_iter=1,
# random_start=True,
# loss_func='xent',
# clip_min=0.,
# clip_max=1.)
# always save your weights after training or during training
# create folder if not exist
if not os.path.exists('models/'):
os.makedirs('models/')
log_path = 'log/%s' % dataset
if not os.path.exists(log_path):
os.makedirs(log_path)
## loop the weight folder then load the lastest weight file continue training
model_prefix = '%s_%s_%s_%.4f_' % (dataset, cnn, 'adv', eps)
w_files = os.listdir('models/')
existing_ep = 0
# for fl in w_files:
# if model_prefix in fl:
# ep = re.search(model_prefix+"(.+?).h5", fl).group(1)
# if int(ep) > existing_ep:
# existing_ep = int(ep)
#
# if existing_ep > 0:
# weight_file = 'models/' + model_prefix + str(existing_ep) + ".h5"
# print("load previous model weights from: ", weight_file)
# model.load_weights(weight_file)
#
# log = np.load(os.path.join(log_path, 'train_log_%s_%s_%.3f.npy' % (cnn, 'adv', eps)))
#
# train_loss_log = log[0, :existing_ep+1].tolist()
# train_acc_log = log[1, :existing_ep+1].tolist()
# val_loss_log = log[2, :existing_ep+1].tolist()
# val_acc_log = log[3, :existing_ep+1].tolist()
# else:
train_loss_log = []
train_acc_log = []
val_loss_log = []
val_acc_log = []
# dynamic training
for ep in range(epochs - existing_ep):
# cosine learning rate annealing
eta_min = 1e-5
eta_max = base_lr
lr = eta_min + (eta_max -
eta_min) * (1 + math.cos(math.pi * ep / epochs)) / 2
K.set_value(model.optimizer.lr, lr)
# # step-wise learning rate annealing
# if ep in [int(epochs*0.5), int(epochs*0.75)]:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr*.1)
# print("lr decayed to {}".format(lr*.1))
current_index = 0
n_step = int(num_images / batch_size)
pbar = tqdm(range(n_step))
for stp in pbar:
b, l = get_batch()
# adversarial denoising
b_adv = attack.perturb(K.get_session(), b, l, batch_size)
train_loss, train_acc = model.train_on_batch(b_adv, l)
pbar.set_postfix(acc='%.4f' % train_acc, loss='%.4f' % train_loss)
## test acc and loss at each epoch
val_current_index = 0
y_pred = []
y_true = []
while val_current_index + batch_size < num_images_val:
b, l = get_val_batch()
pred = model.predict(b)
y_pred.extend(pred.tolist())
y_true.extend(l.tolist())
y_pred = np.clip(np.array(y_pred), 1e-7, 1.)
correct_pred = (np.argmax(y_pred, axis=1) == np.argmax(y_true, axis=1))
val_acc = np.mean(correct_pred)
val_loss = -np.sum(np.mean(y_true * np.log(y_pred),
axis=1)) / val_current_index
train_loss_log.append(train_loss)
train_acc_log.append(train_acc)
val_loss_log.append(val_loss)
val_acc_log.append(val_acc)
log = np.stack((np.array(train_loss_log), np.array(train_acc_log),
np.array(val_loss_log), np.array(val_acc_log)))
# save training log
np.save(
os.path.join(log_path,
'train_log_%s_%s_%.4f.npy' % (cnn, 'adv', eps)), log)
pbar.set_postfix(acc='%.4f' % train_acc,
loss='%.4f' % train_loss,
val_acc='%.4f' % val_acc,
val_loss='%.4f' % val_loss)
print(
"Epoch %s - loss: %.4f - acc: %.4f - val_loss: %.4f - val_acc: %.4f"
% (ep, train_loss, train_acc, val_loss, val_acc))
images, labels = shuffle(images, labels)
if ((ep + existing_ep + 1) % 5
== 0) or (ep == (epochs - existing_ep - 1)):
model_file = 'models/%s_%s_%s_%.4f_%s.h5' % (dataset, cnn, 'adv',
eps, ep + existing_ep)
model.save_weights(model_file)
def UEfficientNet(input_shape=(None, None, 3), dropout_rate=0.1):
backbone = EfficientNetB4(weights='imagenet',
include_top=False,
input_shape=input_shape)
input = backbone.input
start_neurons = 16
conv4 = backbone.layers[342].output
conv4 = LeakyReLU(alpha=0.1)(conv4)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(dropout_rate)(pool4)
# Middle
convm = Conv2D(start_neurons * 32, (3, 3), activation=None,
padding="same")(pool4)
convm = residual_block(convm, start_neurons * 32)
convm = residual_block(convm, start_neurons * 32)
convm = LeakyReLU(alpha=0.1)(convm)
deconv4 = Conv2DTranspose(start_neurons * 16, (3, 3),
strides=(2, 2),
padding="same")(convm)
uconv4 = concatenate([deconv4, conv4])
uconv4 = Dropout(dropout_rate)(uconv4)
uconv4 = Conv2D(start_neurons * 16, (3, 3),
activation=None,
padding="same")(uconv4)
uconv4 = residual_block(uconv4, start_neurons * 16)
uconv4 = residual_block(uconv4, start_neurons * 16)
uconv4 = LeakyReLU(alpha=0.1)(uconv4)
deconv3 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(uconv4)
conv3 = backbone.layers[154].output
uconv3 = concatenate([deconv3, conv3])
uconv3 = Dropout(dropout_rate)(uconv3)
uconv3 = Conv2D(start_neurons * 8, (3, 3), activation=None,
padding="same")(uconv3)
uconv3 = residual_block(uconv3, start_neurons * 8)
uconv3 = residual_block(uconv3, start_neurons * 8)
uconv3 = LeakyReLU(alpha=0.1)(uconv3)
deconv2 = Conv2DTranspose(start_neurons * 4, (3, 3),
strides=(2, 2),
padding="same")(uconv3)
conv2 = backbone.layers[92].output
uconv2 = concatenate([deconv2, conv2])
uconv2 = Dropout(0.1)(uconv2)
uconv2 = Conv2D(start_neurons * 4, (3, 3), activation=None,
padding="same")(uconv2)
uconv2 = residual_block(uconv2, start_neurons * 4)
uconv2 = residual_block(uconv2, start_neurons * 4)
uconv2 = LeakyReLU(alpha=0.1)(uconv2)
deconv1 = Conv2DTranspose(start_neurons * 2, (3, 3),
strides=(2, 2),
padding="same")(uconv2)
conv1 = backbone.layers[30].output
uconv1 = concatenate([deconv1, conv1])
uconv1 = Dropout(0.1)(uconv1)
uconv1 = Conv2D(start_neurons * 2, (3, 3), activation=None,
padding="same")(uconv1)
uconv1 = residual_block(uconv1, start_neurons * 2)
uconv1 = residual_block(uconv1, start_neurons * 2)
uconv1 = LeakyReLU(alpha=0.1)(uconv1)
uconv0 = Conv2DTranspose(start_neurons * 1, (3, 3),
strides=(2, 2),
padding="same")(uconv1)
uconv0 = Dropout(0.1)(uconv0)
uconv0 = Conv2D(start_neurons * 1, (3, 3), activation=None,
padding="same")(uconv0)
uconv0 = residual_block(uconv0, start_neurons * 1)
uconv0 = residual_block(uconv0, start_neurons * 1)
uconv0 = LeakyReLU(alpha=0.1)(uconv0)
uconv0 = Dropout(dropout_rate / 2)(uconv0)
output_layer = Conv2D(1, (1, 1), padding="same",
activation="sigmoid")(uconv0)
model = Model(input, output_layer)
model.name = 'u-xception'
return model
return x
def residual_block(blockInput, num_filters = 16):
x = LeakyReLU(alpha = 0.1)(blockInput)
x = BatchNormalization()(x)
blockInput = BatchNormalization()(blockInput)
x = convolution_block(x, num_filters, (3, 3) )
x = convolution_block(x, num_filters, (3, 3), activation = False)
x = Add()([x, blockInput])
return x
inp = Input(shape = (img_size, img_size, 3))
model_base = EfficientNetB4(
input_tensor = inp,
weights = "imagenet",
include_top = False
)
dropout_rate = 0.5
start_neurons = 8
conv4 = model_base.layers[342].output
conv4 = LeakyReLU(alpha = 0.1)(conv4)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(dropout_rate)(pool4)
convm = Conv2D(start_neurons * 32, (3, 3), activation = None, padding = "same")(pool4)
convm = residual_block(convm, start_neurons * 32)
convm = residual_block(convm, start_neurons * 32)
convm = LeakyReLU(alpha = 0.1)(convm)
