def EffNetB4(freeze=False): model = EfficientNetB4(input_shape=(380, 380, 3), weights='imagenet', include_top=False, pooling=None) for layers in model.layers: layers.trainable = not freeze inputs = model.input x = model.output x = GlobalAveragePooling2D()(x) out_layer = Dense(1, activation=None, name='normal_regressor')(Dropout(0.4)(x)) model = Model(inputs, out_layer) return model
def EfficientNet_model(input_shape=(None, None, 3), num_classes=1, weight='imagenet', dropout_rate=0.5): backbone = EfficientNetB4(weights=weight, include_top=False, input_shape=input_shape) img_input = backbone.input x = backbone.layers[342].output # consider 257, 154 x = LeakyReLU(alpha=0.1)(x) x = Dropout(dropout_rate)(x) x = Conv2D(960, (3, 3), activation=None, padding="same")(x) x = LeakyReLU(alpha=0.1)(x) x = BatchNormalization()(x) x = Conv2D(960, (3, 3), activation=None, padding="same")(x) x = LeakyReLU(alpha=0.1)(x) x = BatchNormalization()(x) x = Dropout(dropout_rate)(x) x = GlobalAveragePooling2D()(x) x = Dropout(0.7)(x) if num_classes == 1: predictions = Dense(num_classes + 1, activation='sigmoid', name='predictions')(x) else: predictions = Dense(num_classes + 1, activation='softmax', name='predictions')(x) model = Model(inputs=img_input, outputs=predictions) if weight is not 'imagenet' and not None: print('loading model weights from pretrained...') model.load_weights(weight) return model
def get_fe(fe, input_image): if fe == 'effnetb0': return EfficientNetB0(input_tensor=input_image, include_top=False), [ 'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1' ] elif fe == 'effnetb1': return EfficientNetB1(input_tensor=input_image, include_top=False), [ 'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1' ] elif fe == 'effnetb2': return EfficientNetB2(input_tensor=input_image, include_top=False), [ 'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1' ] elif fe == 'effnetb3': return EfficientNetB3(input_tensor=input_image, include_top=False), [ 'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1' ] elif fe == 'effnetb4': return EfficientNetB4(input_tensor=input_image, include_top=False), [ 'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1' ] elif fe == 'effnetb5': return EfficientNetB5(input_tensor=input_image, include_top=False), [ 'swish_last', 'block5_i_MB_swish_1', 'block3_i_MB_swish_1' ] elif fe == 'd53': return d53(input_image) elif fe == 'mnetv2': mnet = MobileNetV2(input_tensor=input_image, weights='imagenet') return mnet, [ 'out_relu', 'block_13_expand_relu', 'block_6_expand_relu' ] elif fe == 'mnet': mnet = MobileNet(input_tensor=input_image, weights='imagenet') return mnet, ['conv_pw_13_relu', 'conv_pw_11_relu', 'conv_pw_5_relu'] elif fe == 'r50': r50 = ResNet50(input_tensor=input_image, weights='imagenet') return r50, ['activation_49', 'activation_40', 'activation_22'] raise ValueError('Pls put the correct fe')
def train(dataset='Fish4Knowledge', cnn='resnet50', batch_size=32, epochs=50, image_size=32, eps=0.01): """ Train one checkpoint with data augmentation: random padding+cropping and horizontal flip :param args: :return: """ print( 'Dataset: %s, CNN: %s, loss: adv, epsilon: %.3f batch: %s, epochs: %s' % (dataset, cnn, eps, batch_size, epochs)) IMAGE_SIZE = image_size INPUT_SHAPE = (image_size, image_size, 3) # find image folder: images are distributed in class subfolders if dataset == 'Fish4Knowledge': # image_path = '/home/xingjun/datasets/Fish4Knowledge/fish_image' image_path = '/data/cephfs/punim0619/Fish4Knowledge/fish_image' images, labels, images_val, labels_val = get_Fish4Knowledge( image_path, train_ratio=0.8) elif dataset == 'QUTFish': # image_path = '/home/xingjun/datasets/QUT_fish_data' image_path = '/data/cephfs/punim0619/QUT_fish_data' images, labels, images_val, labels_val = get_QUTFish(image_path, train_ratio=0.8) elif dataset == 'WildFish': # image_pathes = ['/home/xingjun/datasets/WildFish/WildFish_part1', # '/home/xingjun/datasets/WildFish/WildFish_part2', # '/home/xingjun/datasets/WildFish/WildFish_part3', # '/home/xingjun/datasets/WildFish/WildFish_part4'] image_pathes = [ '/data/cephfs/punim0619/WildFish/WildFish_part1', '/data/cephfs/punim0619/WildFish/WildFish_part2', '/data/cephfs/punim0619/WildFish/WildFish_part3', '/data/cephfs/punim0619/WildFish/WildFish_part4' ] images, labels, images_val, labels_val = get_WildFish(image_pathes, train_ratio=0.8) # images, labels, images_val, labels_val = get_imagenet_googlesearch_data(image_path, num_class=NUM_CLASS) num_classes = len(np.unique(labels)) num_images = len(images) num_images_val = len(images_val) print('Train: classes: %s, images: %s, val images: %s' % (num_classes, num_images, num_images_val)) global current_index current_index = 0 # dynamic loading a batch of data def get_batch(): index = 1 global current_index B = np.zeros(shape=(batch_size, IMAGE_SIZE, IMAGE_SIZE, 3)) L = np.zeros(shape=(batch_size)) while index < batch_size: try: img = load_img(images[current_index], target_size=(IMAGE_SIZE, IMAGE_SIZE)) img = img_to_array(img) img /= 255. # if cnn == 'ResNet50': # imagenet pretrained # mean = np.array([0.485, 0.456, 0.406]) # std = np.array([0.229, 0.224, 0.225]) # img = (img - mean)/std ## data augmentation # random width and height shift img = random_shift(img, 0.2, 0.2) # random rotation img = random_rotation(img, 10) # random horizental flip flip_horizontal = (np.random.random() < 0.5) if flip_horizontal: img = flip_axis(img, axis=1) # # random vertical flip # flip_vertical = (np.random.random() < 0.5) # if flip_vertical: # img = flip_axis(img, axis=0) # #cutout # eraser = get_random_eraser(v_l=0, v_h=1, pixel_level=False) # img = eraser(img) B[index] = img L[index] = labels[current_index] index = index + 1 current_index = current_index + 1 except: traceback.print_exc() # print("Ignore image {}".format(images[current_index])) current_index = current_index + 1 # B = np.rollaxis(B, 3, 1) return B, np_utils.to_categorical(L, num_classes) global val_current_index val_current_index = 0 # dynamic loading a batch of validation data def get_val_batch(): index = 1 B = np.zeros(shape=(batch_size, IMAGE_SIZE, IMAGE_SIZE, 3)) L = np.zeros(shape=(batch_size)) global val_current_index while index < batch_size: try: img = load_img(images_val[val_current_index], target_size=(IMAGE_SIZE, IMAGE_SIZE)) img = img_to_array(img) img /= 255. # if cnn == 'ResNet50': # imagenet pretrained # mean = np.array([0.485, 0.456, 0.406]) # std = np.array([0.229, 0.224, 0.225]) # img = (img - mean)/std B[index] = img L[index] = labels_val[val_current_index] index = index + 1 val_current_index = val_current_index + 1 except: traceback.print_exc() # print("Ignore image {}".format(images[val_current_index])) val_current_index = val_current_index + 1 # B = np.rollaxis(B, 3, 1) return B, np_utils.to_categorical(L, num_classes) # load checkpoint if cnn == 'ResNet18': base_model = ResNet18(input_shape=INPUT_SHAPE, classes=num_classes, include_top=False) elif cnn == 'ResNet34': base_model = ResNet34(input_shape=INPUT_SHAPE, classes=num_classes, include_top=False) elif cnn == 'ResNet50': base_model = ResNet50(include_top=False, weights='imagenet', input_shape=INPUT_SHAPE) elif cnn == 'EfficientNetB1': base_model = EfficientNetB1(input_shape=INPUT_SHAPE, classes=num_classes, include_top=False, backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) elif cnn == 'EfficientNetB2': base_model = EfficientNetB2(input_shape=INPUT_SHAPE, classes=num_classes, include_top=False, backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) elif cnn == 'EfficientNetB3': base_model = EfficientNetB3(input_shape=INPUT_SHAPE, classes=num_classes, include_top=False, backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) elif cnn == 'EfficientNetB4': base_model = EfficientNetB4(input_shape=INPUT_SHAPE, classes=num_classes, include_top=False, backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) else: warnings.warn("Error: unrecognized dataset!") return x = base_model.output x = Flatten()(x) x = Dense(num_classes, name='dense')(x) output = Activation('softmax')(x) model = Model(input=base_model.input, output=output, name=cnn) # model.summary() loss = cross_entropy base_lr = 1e-2 sgd = SGD(lr=base_lr, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss=loss, optimizer=sgd, metrics=['accuracy']) # AdaFGSM attack for AdvFish training attack = AdaFGSM(model, epsilon=float(eps), random_start=True, loss_func='xent', clip_min=0., clip_max=1.) # PGD attack for AdvFish training, it reduces to FGSM when set nb_iter=1 # attack = LinfPGDAttack(model, # epsilon=float(eps), # eps_iter=float(eps), # nb_iter=1, # random_start=True, # loss_func='xent', # clip_min=0., # clip_max=1.) # always save your weights after training or during training # create folder if not exist if not os.path.exists('models/'): os.makedirs('models/') log_path = 'log/%s' % dataset if not os.path.exists(log_path): os.makedirs(log_path) ## loop the weight folder then load the lastest weight file continue training model_prefix = '%s_%s_%s_%.4f_' % (dataset, cnn, 'adv', eps) w_files = os.listdir('models/') existing_ep = 0 # for fl in w_files: # if model_prefix in fl: # ep = re.search(model_prefix+"(.+?).h5", fl).group(1) # if int(ep) > existing_ep: # existing_ep = int(ep) # # if existing_ep > 0: # weight_file = 'models/' + model_prefix + str(existing_ep) + ".h5" # print("load previous model weights from: ", weight_file) # model.load_weights(weight_file) # # log = np.load(os.path.join(log_path, 'train_log_%s_%s_%.3f.npy' % (cnn, 'adv', eps))) # # train_loss_log = log[0, :existing_ep+1].tolist() # train_acc_log = log[1, :existing_ep+1].tolist() # val_loss_log = log[2, :existing_ep+1].tolist() # val_acc_log = log[3, :existing_ep+1].tolist() # else: train_loss_log = [] train_acc_log = [] val_loss_log = [] val_acc_log = [] # dynamic training for ep in range(epochs - existing_ep): # cosine learning rate annealing eta_min = 1e-5 eta_max = base_lr lr = eta_min + (eta_max - eta_min) * (1 + math.cos(math.pi * ep / epochs)) / 2 K.set_value(model.optimizer.lr, lr) # # step-wise learning rate annealing # if ep in [int(epochs*0.5), int(epochs*0.75)]: # lr = K.get_value(model.optimizer.lr) # K.set_value(model.optimizer.lr, lr*.1) # print("lr decayed to {}".format(lr*.1)) current_index = 0 n_step = int(num_images / batch_size) pbar = tqdm(range(n_step)) for stp in pbar: b, l = get_batch() # adversarial denoising b_adv = attack.perturb(K.get_session(), b, l, batch_size) train_loss, train_acc = model.train_on_batch(b_adv, l) pbar.set_postfix(acc='%.4f' % train_acc, loss='%.4f' % train_loss) ## test acc and loss at each epoch val_current_index = 0 y_pred = [] y_true = [] while val_current_index + batch_size < num_images_val: b, l = get_val_batch() pred = model.predict(b) y_pred.extend(pred.tolist()) y_true.extend(l.tolist()) y_pred = np.clip(np.array(y_pred), 1e-7, 1.) correct_pred = (np.argmax(y_pred, axis=1) == np.argmax(y_true, axis=1)) val_acc = np.mean(correct_pred) val_loss = -np.sum(np.mean(y_true * np.log(y_pred), axis=1)) / val_current_index train_loss_log.append(train_loss) train_acc_log.append(train_acc) val_loss_log.append(val_loss) val_acc_log.append(val_acc) log = np.stack((np.array(train_loss_log), np.array(train_acc_log), np.array(val_loss_log), np.array(val_acc_log))) # save training log np.save( os.path.join(log_path, 'train_log_%s_%s_%.4f.npy' % (cnn, 'adv', eps)), log) pbar.set_postfix(acc='%.4f' % train_acc, loss='%.4f' % train_loss, val_acc='%.4f' % val_acc, val_loss='%.4f' % val_loss) print( "Epoch %s - loss: %.4f - acc: %.4f - val_loss: %.4f - val_acc: %.4f" % (ep, train_loss, train_acc, val_loss, val_acc)) images, labels = shuffle(images, labels) if ((ep + existing_ep + 1) % 5 == 0) or (ep == (epochs - existing_ep - 1)): model_file = 'models/%s_%s_%s_%.4f_%s.h5' % (dataset, cnn, 'adv', eps, ep + existing_ep) model.save_weights(model_file)
def UEfficientNet(input_shape=(None, None, 3), dropout_rate=0.1): backbone = EfficientNetB4(weights='imagenet', include_top=False, input_shape=input_shape) input = backbone.input start_neurons = 16 conv4 = backbone.layers[342].output conv4 = LeakyReLU(alpha=0.1)(conv4) pool4 = MaxPooling2D((2, 2))(conv4) pool4 = Dropout(dropout_rate)(pool4) # Middle convm = Conv2D(start_neurons * 32, (3, 3), activation=None, padding="same")(pool4) convm = residual_block(convm, start_neurons * 32) convm = residual_block(convm, start_neurons * 32) convm = LeakyReLU(alpha=0.1)(convm) deconv4 = Conv2DTranspose(start_neurons * 16, (3, 3), strides=(2, 2), padding="same")(convm) uconv4 = concatenate([deconv4, conv4]) uconv4 = Dropout(dropout_rate)(uconv4) uconv4 = Conv2D(start_neurons * 16, (3, 3), activation=None, padding="same")(uconv4) uconv4 = residual_block(uconv4, start_neurons * 16) uconv4 = residual_block(uconv4, start_neurons * 16) uconv4 = LeakyReLU(alpha=0.1)(uconv4) deconv3 = Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding="same")(uconv4) conv3 = backbone.layers[154].output uconv3 = concatenate([deconv3, conv3]) uconv3 = Dropout(dropout_rate)(uconv3) uconv3 = Conv2D(start_neurons * 8, (3, 3), activation=None, padding="same")(uconv3) uconv3 = residual_block(uconv3, start_neurons * 8) uconv3 = residual_block(uconv3, start_neurons * 8) uconv3 = LeakyReLU(alpha=0.1)(uconv3) deconv2 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv3) conv2 = backbone.layers[92].output uconv2 = concatenate([deconv2, conv2]) uconv2 = Dropout(0.1)(uconv2) uconv2 = Conv2D(start_neurons * 4, (3, 3), activation=None, padding="same")(uconv2) uconv2 = residual_block(uconv2, start_neurons * 4) uconv2 = residual_block(uconv2, start_neurons * 4) uconv2 = LeakyReLU(alpha=0.1)(uconv2) deconv1 = Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding="same")(uconv2) conv1 = backbone.layers[30].output uconv1 = concatenate([deconv1, conv1]) uconv1 = Dropout(0.1)(uconv1) uconv1 = Conv2D(start_neurons * 2, (3, 3), activation=None, padding="same")(uconv1) uconv1 = residual_block(uconv1, start_neurons * 2) uconv1 = residual_block(uconv1, start_neurons * 2) uconv1 = LeakyReLU(alpha=0.1)(uconv1) uconv0 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv1) uconv0 = Dropout(0.1)(uconv0) uconv0 = Conv2D(start_neurons * 1, (3, 3), activation=None, padding="same")(uconv0) uconv0 = residual_block(uconv0, start_neurons * 1) uconv0 = residual_block(uconv0, start_neurons * 1) uconv0 = LeakyReLU(alpha=0.1)(uconv0) uconv0 = Dropout(dropout_rate / 2)(uconv0) output_layer = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(uconv0) model = Model(input, output_layer) model.name = 'u-xception' return model
return x def residual_block(blockInput, num_filters = 16): x = LeakyReLU(alpha = 0.1)(blockInput) x = BatchNormalization()(x) blockInput = BatchNormalization()(blockInput) x = convolution_block(x, num_filters, (3, 3) ) x = convolution_block(x, num_filters, (3, 3), activation = False) x = Add()([x, blockInput]) return x inp = Input(shape = (img_size, img_size, 3)) model_base = EfficientNetB4( input_tensor = inp, weights = "imagenet", include_top = False ) dropout_rate = 0.5 start_neurons = 8 conv4 = model_base.layers[342].output conv4 = LeakyReLU(alpha = 0.1)(conv4) pool4 = MaxPooling2D((2, 2))(conv4) pool4 = Dropout(dropout_rate)(pool4) convm = Conv2D(start_neurons * 32, (3, 3), activation = None, padding = "same")(pool4) convm = residual_block(convm, start_neurons * 32) convm = residual_block(convm, start_neurons * 32) convm = LeakyReLU(alpha = 0.1)(convm)