def build_model(columns, rows, depth, output_units, kernel_size): # Based on: https://keras.io/examples/vision/3D_image_classification/ pool_size = 2 # Handle small dimensions if (columns <= 2 or rows <= 2): pool_size = 1 kernel_size = 1 # Handle odd dimension lengths if ((columns % 2 != 0) or (rows % 2 != 0)): kernel_size = 1 inputs = keras.Input((depth, columns, rows, 1)) x = layers.Conv3D(filters=64, kernel_size=kernel_size, activation="relu")(inputs) x = layers.MaxPool3D(pool_size=pool_size)(x) x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512, activation="relu")(x) x = layers.Dropout(0.3)(x) outputs = layers.Dense(units=output_units, activation="sigmoid")(x) model = keras.Model(inputs, outputs, name="3DCNN") return model
def build_network(input_shapes, output_size, training, name = 'TreatmentRecommder'): ''' build the network for covid-19 prediction of how long a patient can be cured ''' dtype = tf.float32 #treatment information treatment_info = KL.Input(shape = input_shapes[0], dtype = dtype, name='treatment_info') #imaing information: CNN features from CT images image_info = KL.Input(shape = input_shapes[1]+[1], dtype = dtype, name='image_info') base_filters = 16 x11 = KL.Conv3D(base_filters, (3, 3, 3), activation='relu', padding='same', name = 'x11')(image_info) x12 = KL.Conv3D(base_filters, (3, 3, 3), activation='relu', padding='same', name = 'x12')(x11) x13 = KL.Conv3D(base_filters, (3, 3, 3), activation='relu', padding='same', name = 'x13')(x12) d1 = KL.MaxPool3D()(x13) x21 = KL.Conv3D(base_filters*2, (3, 3, 3), activation='relu', padding='same', name = 'x21')(d1) x22 = KL.Conv3D(base_filters*2, (3, 3, 3), activation='relu', padding='same', name = 'x22')(x21) d2 = KL.MaxPool3D()(x22) x31 = KL.Conv3D(base_filters*4, (3, 3, 3), activation='relu', padding='same', name = 'x31')(d2) x32 = KL.Conv3D(base_filters*4, (3, 3, 3), activation='relu', padding='same', name = 'x32')(x31) d3 = KL.MaxPool3D()(x32) x41 = KL.Conv3D(base_filters*8, (3, 3, 3), activation='relu', padding='same', name = 'x41')(d3) x42 = KL.Conv3D(base_filters*8, (3, 3, 3), activation='relu', padding='same', name = 'x42')(x41) d4 = KL.MaxPool3D()(x42) x51 = KL.Conv3D(base_filters*16, (3, 3, 3), activation='relu', padding='same', name = 'x51')(d4) x52 = KL.Conv3D(base_filters*16, (3, 3, 3), activation='relu', padding='same', name = 'x52')(x51) d5 = KL.MaxPool3D()(x52) cnn_GAP = KL.GlobalAveragePooling3D(name='CNN_GAP')(d5) cnn_cof = KL.Dense(1, activation='relu', name='cnn_cof')(cnn_GAP) #patient information patient_info = KL.Input(shape = input_shapes[2], dtype = dtype, name='patient_info') pcnn_info = KL.Concatenate()([patient_info, cnn_cof]) #cured probability distruibution subnetwork w_pcnn_info = SA_Module(pcnn_info, training) fc1 = KL.Dense(256, activation='relu', name='fc1')(KL.Concatenate()([w_pcnn_info, cnn_GAP, treatment_info])) fc2 = KL.Dense(512, activation='relu', name='fc2')(fc1) fc3 = KL.Dense(512, activation='relu', name='fc3')(fc2) fc_cls = KL.Dense(256, activation='relu', name='fc_cls')(fc3) fc_cls = KL.Dropout(0.4)(fc_cls, training = training) severity_cls_preds = KL.Dense(output_size[0],activation='softmax', name='severity_cls_preds')(fc_cls) fc_reg = KL.Dense(256, activation='relu', name='fc_reg')(fc3) fc_reg = KL.Dropout(0.4)(fc_reg, training = training) risk_reg_preds = KL.Dense(output_size[1],activation='softmax', name='risk_reg_preds')(fc_reg) model = KM.Model([treatment_info,image_info,patient_info], [severity_cls_preds, risk_reg_preds], name=name) return model
def create_model(height=240, width=320): # shape of input: 1 block has 10 frames x height x width x 3 channels (RGB) input = tf.keras.Input((10, height, width, 3)) # 1st Conv3D block includes Conv3D with 8 filters, MaxPool3D and BatchNormalization x = layers.Conv3D(filters=8, kernel_size=(3, 3, 3), activation='relu')(input) x = layers.MaxPool3D(pool_size=(2, 2, 2))(x) x = layers.BatchNormalization()(x) # 2nd Conv3D block includes Conv3D with 16 filters, MaxPool3D and BatchNormalization x = layers.Conv3D(filters=16, kernel_size=(3, 3, 3), activation='relu')(x) x = layers.MaxPool3D(pool_size=(2, 2, 2))(x) x = layers.BatchNormalization()(x) # 3rd Conv3D block includes Conv3D with 32 filters, MaxPool3D and BatchNormalization x = layers.Conv3D(filters=32, kernel_size=(3, 3, 3), activation='relu')(input) x = layers.MaxPool3D(pool_size=(1, 2, 2))(x) x = layers.BatchNormalization()(x) # Fully-connected block includes GlobalAveragePooling3D, Fully-Connected layer with 512 units and DropOut for Regularization x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512, activation='relu')(x) x = layers.DropOut(0.7)(x) # output shape (1,) produces value between [0, 1] output = layers.Dense(units=1, activation='sigmoid')(x) model = tf.keras.Model(input, output, name='3DCNN') return model
def encoder(): resnet = nobrainer.models.highresnet( n_classes=n_classes, input_shape=input_shape ) input = tf.keras.layers.Input(shape=input_shape) resnet_out = resnet(input) x = layers.GlobalAveragePooling3D(name="backbone_pool")(resnet_out) x = layers.Dense( projection_dim, use_bias=False, kernel_regularizer=regularizers.l2(weight_decay), )(x) x = layers.BatchNormalization()(x) x = layers.LeakyReLU()(x) x = layers.Dense( projection_dim, use_bias=False, kernel_regularizer=regularizers.l2(weight_decay), )(x) output = layers.BatchNormalization()(x) encoder_model = tf.keras.Model(input, output, name="encoder") return encoder_model
def resNet(sideLength): '''resNet 3D implementation''' input = keras.Input(shape=(sideLength, sideLength, sideLength, 1)) x = layers.Conv3D(filters=64, kernel_size=7, strides=2, padding='same')(input) x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) x = layers.MaxPooling3D(2, strides=1)(x) x = first_block(x, 64, 256) for i in range(2): x = residual_block(x, 64, 256) x = first_block(x, 128, 512) for i in range(3): x = residual_block(x, 128, 512) x = first_block(x, 256, 1024) for i in range(5): x = residual_block(x, 256, 1024) #at the end x = layers.GlobalAveragePooling3D()(x) x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) x = layers.Flatten()(x) x = layers.Dense(512, activation='relu')(x) x = layers.Dropout(0.2)(x) output = layers.Dense(1, activation='sigmoid')(x) model = keras.Model(input, output, name='resnet') return model
def get_model(width=128, height=128, depth=64): """build a 3D convolutional neural network model""" inputs = keras.Input((width, height, depth, 1)) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu")(inputs) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.Conv3D(filters=128, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.Conv3D(filters=256, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512, activation="relu")(x) x = layers.Dropout(0.3)(x) outputs = layers.Dense(units=1, activation="sigmoid")(x) # Define the model. model = keras.Model(inputs, outputs, name="3dcnn") return model
def __init__(self, dim): super(_AP, self).__init__() if dim == 1: self.pool = layers.GlobalAveragePooling1D() elif dim == 2: self.pool = layers.GlobalAveragePooling2D() elif dim == 3: self.pool = layers.GlobalAveragePooling3D()
def build_model(columns, rows, depth, output_units): # Based on: https://keras.io/examples/vision/3D_image_classification/ inputs = keras.Input((depth, columns, rows, 1)) x = layers.Conv3D(filters=64, kernel_size=settings.KERNEL_SIZE, activation="relu")(inputs) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512, activation="relu")(x) x = layers.Dropout(0.3)(x) outputs = layers.Dense(units=output_units, activation="sigmoid")(x) model = keras.Model(inputs, outputs, name="3DCNN") return model
def get_model(input_shape=(None, None, None, 1)): inputs = layers.Input(shape=input_shape) x = layers.Conv3D(filters=96, kernel_size=1, activation="relu")(inputs) x = layers.MaxPool3D(strides=2)(x) x = layers.BatchNormalization()(x) x = fire_block(x, 128) x = fire_block(x, 128) x = fire_block(x, 256) x = layers.MaxPool3D(strides=2)(x) x = fire_block(x, 256) x = fire_block(x, 384) x = fire_block(x, 384) x = fire_block(x, 512) x = layers.MaxPool3D(strides=2)(x) x = fire_block(x, 512) x = layers.Conv3D(filters=1, kernel_size=1, activation="relu")(x) x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Softmax()(x) return tf.keras.Model(inputs=inputs, outputs=x, name="SqueezeNet")
def get_model_8L_(width=128, height=128, depth=64): """Build a 3D convolutional neural network model.""" inputs = keras.Input((width, height, depth, 1)) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu")(inputs) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization(center=True, scale=True)(x) x = layers.Dropout(0.2)(x) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization(center=True, scale=True)(x) x = layers.Dropout(0.5)(x) ### add 64N layer ################################################## x = layers.Conv3D(filters=64, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization(center=True, scale=True)(x) x = layers.Dropout(0.3)(x) x = layers.Conv3D(filters=128, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=1)(x) x = layers.BatchNormalization(center=True, scale=True)(x) x = layers.Dropout(0.3)(x) x = layers.Conv3D(filters=256, kernel_size=3, activation="relu")(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512, activation="relu")(x) x = layers.Dropout(0.3)(x) outputs = layers.Dense(units=1, activation="sigmoid")(x) # outputs = layers.Dense(units=1, activation="tanh", )(x) # outputs = layers.Dense(units=1, activation="relu", )(x) model = keras.Model(inputs, outputs, name="3dcnn") return model
def get_model(width=224, height=224, depth=32): """Build a 3D convolutional neural network model.""" inputs = keras.Input((width, height, depth, 1)) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu", padding='same')(inputs) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu", padding='same')(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.Conv3D(filters=128, kernel_size=3, activation="relu", padding='same')(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.Conv3D(filters=256, kernel_size=3, activation="relu", padding='same')(x) x = layers.MaxPool3D(pool_size=2)(x) x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512, activation="relu")(x) x = layers.Dropout(0.3)(x) outputs = layers.Dense(units=51, activation="softmax")(x) model = keras.Model(inputs, outputs, name="3dcnn") return model
def RFAB(x, filters, kernel_size, r): """Residual Feature attention Block""" x_res = x # Attention x = conv3d_weightnorm(filters, kernel_size)(x) x = layers.ReLU()(x) x = conv3d_weightnorm(filters, kernel_size)(x) x_to_scale = x x = layers.GlobalAveragePooling3D()(x) # expand to 1x1x1xc for i in range(3): x = layers.Lambda(lambda x: tf.expand_dims(x, axis=(-2)))(x) x = conv3d_weightnorm(int(filters/r), 1)(x) x = layers.ReLU()(x) x = conv3d_weightnorm(filters, 1, activation='sigmoid')(x) x_scaled = x_to_scale * x return x_scaled + x_res
def get_model(width=128, height=128, depth=8): """Build a 3D convolutional neural network model.""" inputs = keras.Input((width, height, depth, 1)) x = layers.Conv3D(filters=32, kernel_size=3, activation="relu", padding='same')(inputs) x = layers.Conv3D(filters=32, kernel_size=3, activation="relu", padding='same')(x) x = layers.Conv3D(filters=32, kernel_size=3, activation="relu", padding='same')(x) x = layers.MaxPool3D(pool_size=(2, 2, 2), padding="same")(x) # x = layers.Dropout(0.25)(x) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu", padding='same')(x) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu", padding='same')(x) x = layers.Conv3D(filters=64, kernel_size=3, activation="relu", padding='same')(x) x = layers.MaxPool3D(pool_size=(2, 2, 2), padding="same")(x) # x = layers.Dropout(0.25)(x) x = layers.Conv3D(filters=128, kernel_size=3, activation="relu", padding='same')(x) x = layers.Conv3D(filters=128, kernel_size=3, activation="relu", padding='same')(x) # x = layers.MaxPool3D(pool_size=(3,3,3), padding="same")(x) # x = layers.Conv3D(filters=128, kernel_size=3, activation="relu",padding='same')(x) # x = layers.Conv3D(filters=128, kernel_size=3, activation="relu",padding='same')(x) # x = layers.MaxPool3D(pool_size=(3,3,3), padding="same")(x) # x = layers.BatchNormalization()(x) # x = layers.Conv3D(filters=64, kernel_size=3, activation="relu",padding='same')(x) # x = layers.MaxPool3D(pool_size=(2,2,2))(x) # x = layers.BatchNormalization()(x) # x = layers.Conv3D(filters=128, kernel_size=3, activation="relu",padding='same')(x) # x = layers.MaxPool3D(pool_size=(2,2,2))(x) # x = layers.BatchNormalization()(x) # x = layers.Conv3D(filters=256, kernel_size=3, activation="relu",padding='same')(x) # # x = layers.MaxPool3D(pool_size=2)(x) # x = layers.BatchNormalization()(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(units=512)(x) x = layers.BatchNormalization()(x) x = layers.ReLU()(x) # x = layers.Dropout(0.3)(x) x = layers.Dense(units=256)(x) x = layers.BatchNormalization()(x) x = layers.ReLU()(x) # x = layers.Dropout(0.3)(x) # x = layerss.Dropout(0.3)(x) outputs = layers.Dense(units=8, activation="softmax")(x) # Define the model. model = keras.Model(inputs, outputs, name="3dcnn") return model
def get_pyramidnet(dataset, depth, alpha, num_classes, bottleneck=False): if dataset.startswith('cifar'): inplanes = 16 if bottleneck == True: n = int((depth - 2) / 9) block = bottleneck else: n = int((depth - 2) / 6) block = basicblock addrate = alpha / (3 * n * 1.0) inp = layers.Input(shape=(32, 32, 3)) x = inp x = layers.Lambda( lambda v: tf.tile(v[..., None, :], [1, 1, 1, num_rot, 1]))(x) x = conv3x3(x, inplanes, stride=1) x = batchnorm(x) featuremap_dim = inplanes x, featuremap_dim = pyramidal_make_layer(x, block, n, 1, featuremap_dim, addrate) x, featuremap_dim = pyramidal_make_layer(x, block, n, 2, featuremap_dim, addrate) x, featuremap_dim = pyramidal_make_layer(x, block, n, 2, featuremap_dim, addrate) x = batchnorm(x) x = activation(x) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(num_classes)(x) outp = x elif dataset == 'imagenet': blocks = { 18: basicblock, 34: basicblock, 50: bottleneck, 101: bottleneck, 152: bottleneck, 200: bottleneck } layers_ = { 18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3], 200: [3, 24, 36, 3] } if layers_.get(depth) is None: if bottleneck == True: blocks[depth] = bottleneck temp_cfg = int((depth - 2) / 12) else: blocks[depth] = basicblock temp_cfg = int((depth - 2) / 8) layers_[depth] = [temp_cfg, temp_cfg, temp_cfg, temp_cfg] print('=> the layer configuration for each stage is set to', layers_[depth]) inplanes = 64 addrate = alpha / (sum(layers_[depth]) * 1.0) inp = layers.Input(shape=(32, 32, 3)) x = inp x = conv3x3(x, inplanes, stride=2, kernel_size=7) # x = layers.Conv2D(inplanes, kernel_size=7, # stride=2, padding='same', use_bias=False)(x) x = batchnorm(x) x = activation(x) x = layers.MaxPool3D(pool_size=(3, 3, 1), strides=(2, 2, 1), padding='same')(x) featuremap_dim = inplanes x, featuremap_dim = pyramidal_make_layer(x, blocks[depth], layers_[depth][0], 1, featuremap_dim, addrate) x, featuremap_dim = pyramidal_make_layer(x, blocks[depth], layers_[depth][1], 2, featuremap_dim, addrate) x, featuremap_dim = pyramidal_make_layer(x, blocks[depth], layers_[depth][2], 2, featuremap_dim, addrate) x, featuremap_dim = pyramidal_make_layer(x, blocks[depth], layers_[depth][3], 2, featuremap_dim, addrate) x = batchnorm(x) x = activation(x) x = layers.GlobalAvergePooling3D()(x) x = layers.Dense(num_classes)(x) outp = x return models.Model(inp, outp)
def __init__(self, ch= 32): super(discriminator, self).__init__() self.ch = ch # self.inputBN = layers.BatchNormalization() self.xD0 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter1_layer_1')) # if self.hparams[HP_BN_UNITS] : x = layers.BatchNormalization()(x) # self.xD0BN = layers.BatchNormalization() self.xR0 = layers.LeakyReLU() self.xD1 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter1_layer_2')) # if self.hparams[HP_BN_UNITS] : x = layers.BatchNormalization()(x) # self.xD1BN = layers.BatchNormalization() self.xR1 = layers.LeakyReLU() self.xD2 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter1_layer_3')) # if self.hparams[HP_BN_UNITS] : x = layers.BatchNormalization()(x) # self.xD2BN = layers.BatchNormalization() self.xR2 = layers.LeakyReLU() self.yD0 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter2_layer_1')) # if self.hparams[HP_BN_UNITS] : y = layers.BatchNormalization()(y) # self.yD0BN = layers.BatchNormalization() self.yR0 = layers.LeakyReLU() self.yD1 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter2_layer_2')) # if self.hparams[HP_BN_UNITS] : y = layers.BatchNormalization()(y) # self.yD1BN = layers.BatchNormalization() self.yR1 = layers.LeakyReLU() self.yD2 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter2_layer_3')) # if self.hparams[HP_BN_UNITS] : y = layers.BatchNormalization()(y) # self.yD2BN = layers.BatchNormalization() self.yR2 = layers.LeakyReLU() self.zD0 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter3_layer_1')) # if self.hparams[HP_BN_UNITS] : z = layers.BatchNormalization()(z) # self.zD0BN = layers.BatchNormalization() self.zR0 = layers.LeakyReLU() self.zD1 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter3_layer_2')) # if self.hparams[HP_BN_UNITS] : z = layers.BatchNormalization()(z) # self.zD1BN = layers.BatchNormalization() self.zR1 = layers.LeakyReLU() self.zD2 = tfa.layers.SpectralNormalization(layers.Dense(512, name = 'parameter3_layer_3')) # if self.hparams[HP_BN_UNITS] : z = layers.BatchNormalization()(z) # self.zD2BN = layers.BatchNormalization() self.zR2 = layers.LeakyReLU() # self.concateD = tfa.layers.SpectralNormalization(layers.Dense(ch*32)) # self.concateR = layers.LeakyReLU() # self.conv0 = tfa.layers.SpectralNormalization(layers.Conv3D(ch, kernel_size=3, strides=2, padding='same', use_bias=False)) #self.resR0 = layers.LeakyReLU() # self.conv1 = tfa.layers.SpectralNormalization(layers.Conv3D(2*ch, kernel_size=3, strides=2, padding='same', use_bias=False)) # self.resR1 = layers.LeakyReLU() # self.conv2 = tfa.layers.SpectralNormalization(layers.Conv3D(4*ch, kernel_size=3, strides=2, padding='same', use_bias=False)) # self.resR2 = layers.LeakyReLU() # self.conv3 = tfa.layers.SpectralNormalization(layers.Conv3D(8*ch, kernel_size=3, strides=2, padding='same', use_bias=False)) # self.resR3 = layers.LeakyReLU() self.res0 = ResBlock_discriminator(ch*2, ksize=4, shortcut=True) self.res1 = ResBlock_discriminator(ch*4, ksize=4 , shortcut=True) self.res2 = ResBlock_discriminator(ch*8, ksize=4, shortcut=True) self.res3 = ResBlock_discriminator(ch*16, ksize=4, shortcut=True) # self.res4 = ResBlock_discriminator(ch*8, ksize=3) # self.res5 = ResBlock_discriminator(ch*16, ksize=3) # self.res6 = ResBlock_discriminator(ch*16, ksize=3) # self.conv4 = tfa.layers.SpectralNormalization(layers.Conv3D(1, kernel_size=4, strides=1, padding='valid', use_bias=False)) self.GAV3D = layers.GlobalAveragePooling3D() #self.out = tfa.layers.SpectralNormalization(layers.Dense(1, kernel_initializer=initializers.he_normal())) # self.conv = tfa.layers.SpectralNormalization(layers.Dense(ch*32)) self.multiple = layers.Multiply() self.sum = tf.math.reduce_sum self.out = layers.add self.outputD = tfa.layers.SpectralNormalization(layers.Dense(1)) self.outputActivation = layers.Activation('linear')
def to_real_keras_layer(stub_layer): if isinstance(stub_layer, StubConv1d): return layers.Conv1D(stub_layer.filters, stub_layer.kernel_size, strides=stub_layer.stride, input_shape=stub_layer.input.shape, padding='same') # padding elif isinstance(stub_layer, StubConv2d): return layers.Conv2D(stub_layer.filters, stub_layer.kernel_size, strides=stub_layer.stride, input_shape=stub_layer.input.shape, padding='same') # padding elif isinstance(stub_layer, StubConv3d): return layers.Conv3D(stub_layer.filters, stub_layer.kernel_size, strides=stub_layer.stride, input_shape=stub_layer.input.shape, padding='same') # padding # TODO: Spatial Dropout elif isinstance(stub_layer, (StubDropout1d, StubDropout2d, StubDropout3d)): return layers.Dropout(stub_layer.rate) # elif isinstance(stub_layer, StubDropout2d): # return layers.SpatialDropout2D(stub_layer.rate) # elif isinstance(stub_layer, StubDropout3d): # return layers.SpatialDropout3D(stub_layer.rate) elif isinstance(stub_layer, StubAvgPooling1d): return layers.AveragePooling1D(stub_layer.kernel_size, strides=stub_layer.stride) elif isinstance(stub_layer, StubAvgPooling2d): return layers.AveragePooling2D(stub_layer.kernel_size, strides=stub_layer.stride) elif isinstance(stub_layer, StubAvgPooling3d): return layers.AveragePooling3D(stub_layer.kernel_size, strides=stub_layer.stride) elif isinstance(stub_layer, StubGlobalPooling1d): return layers.GlobalAveragePooling1D() elif isinstance(stub_layer, StubGlobalPooling2d): return layers.GlobalAveragePooling2D() elif isinstance(stub_layer, StubGlobalPooling3d): return layers.GlobalAveragePooling3D() elif isinstance(stub_layer, StubPooling1d): return layers.MaxPooling1D(stub_layer.kernel_size, strides=stub_layer.stride) elif isinstance(stub_layer, StubPooling2d): return layers.MaxPooling2D(stub_layer.kernel_size, strides=stub_layer.stride) elif isinstance(stub_layer, StubPooling3d): return layers.MaxPooling3D(stub_layer.kernel_size, strides=stub_layer.stride) elif isinstance(stub_layer, (StubBatchNormalization1d, StubBatchNormalization2d, StubBatchNormalization3d)): return layers.BatchNormalization(input_shape=stub_layer.input.shape) elif isinstance(stub_layer, StubSoftmax): return layers.Activation('softmax') elif isinstance(stub_layer, StubReLU): return layers.Activation('relu') elif isinstance(stub_layer, StubFlatten): return layers.Flatten() elif isinstance(stub_layer, StubAdd): return layers.Add() elif isinstance(stub_layer, StubConcatenate): return layers.Concatenate() elif isinstance(stub_layer, StubDense): return layers.Dense(stub_layer.units, input_shape=(stub_layer.input_units, ))
def resnet50_3d(inputs, filter_ratio=1, n=2, include_fc_layer=False, logits=True, kernal1=(1, 1, 1), kernal3=(1, 3, 3), kernal7=(1, 7, 7), num_layers=None): """ :param inputs: Keras Input object with desire shape :type inputs: :param filter_ratio: :type filter_ratio: :param n: # of categories :type n: integer :param include_fc_layer: :type include_fc_layer: :return: :rtype: """ # --- Define kwargs dictionary kwargs1 = { 'kernel_size': kernal1, 'padding': 'valid', } kwargs3 = { 'kernel_size': kernal3, 'padding': 'same', } kwargs7 = { 'kernel_size': kernal7, 'padding': 'valid', } # --- Define block components conv1 = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs1)(x) conv3 = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs3)(x) relu = lambda x: layers.LeakyReLU()(x) conv7 = lambda x, filters, strides: layers.Conv3D( filters=filters, strides=strides, **kwargs7)(x) max_pool = lambda x, pool_size, strides: layers.MaxPooling3D( pool_size=pool_size, strides=strides, padding='valid')(x) norm = lambda x: layers.BatchNormalization()(x) add = lambda x, y: layers.Add()([x, y]) zeropad = lambda x, padding: layers.ZeroPadding3D(padding=padding)(x) # --- Residual blocks # conv blocks conv_1 = lambda filters, x, strides: relu( norm(conv1(x, filters, strides=strides))) conv_2 = lambda filters, x: relu(norm(conv3(x, filters, strides=1))) conv_3 = lambda filters, x: norm(conv1(x, filters, strides=1)) conv_sc = lambda filters, x, strides: norm( conv1(x, filters, strides=strides)) conv_block = lambda filters1, filters2, x, strides: relu( add(conv_sc(filters2, x, strides), conv_3(filters2, conv_2(filters1, conv_1(filters1, x, strides))))) # identity blocks identity_1 = lambda filters, x: relu(norm(conv1(x, filters, strides=1))) identity_2 = lambda filters, x: relu(norm(conv3(x, filters, strides=1))) identity_3 = lambda filters, x: norm(conv1(x, filters, strides=1)) identity_block = lambda filters1, filters2, x: relu( add( identity_3(filters2, identity_2(filters1, identity_1(filters1, x)) ), x)) # --- ResNet-50 backbone # stage 1 c2 1/4 res1 = max_pool(zeropad( relu( norm( conv7(zeropad(inputs, (0, 3, 3)), int(64 * filter_ratio), strides=(1, 2, 2)))), (0, 1, 1)), (1, 3, 3), strides=(1, 2, 2)) # stage 2 c2 1/4 res2 = layers.Lambda(lambda x: x, name='c2-output')(identity_block( int(64 * filter_ratio), int(256 * filter_ratio), identity_block( int(64 * filter_ratio), int(256 * filter_ratio), conv_block(int(64 * filter_ratio), int(256 * filter_ratio), res1, strides=1)))) # stage 3 c3 1/8 res3 = layers.Lambda(lambda x: x, name='c3-output')(identity_block( int(128 * filter_ratio), int(512 * filter_ratio), identity_block( int(128 * filter_ratio), int(512 * filter_ratio), identity_block( int(128 * filter_ratio), int(512 * filter_ratio), conv_block(int(128 * filter_ratio), int(512 * filter_ratio), res2, strides=(1, 2, 2)))))) # stage 4 c4 1/16 res4 = layers.Lambda(lambda x: x, name='c4-output')(identity_block( int(256 * filter_ratio), int(1024 * filter_ratio), identity_block( int(256 * filter_ratio), int(1024 * filter_ratio), identity_block( int(256 * filter_ratio), int(1024 * filter_ratio), identity_block( int(256 * filter_ratio), int(1024 * filter_ratio), identity_block( int(256 * filter_ratio), int(1024 * filter_ratio), conv_block(int(256 * filter_ratio), int(1024 * filter_ratio), res3, strides=(1, 2, 2)))))))) # stage 5 c5 1/32 res5 = layers.Lambda(lambda x: x, name='c5-output')(identity_block( int(512 * filter_ratio), int(2048 * filter_ratio), identity_block( int(512 * filter_ratio), int(2048 * filter_ratio), conv_block(int(512 * filter_ratio), int(2048 * filter_ratio), res4, strides=(1, 2, 2))))) if num_layers: avg_pool = layers.GlobalAveragePooling3D()( [res1, res2, res3, res4, res5][num_layers - 1]) else: avg_pool = layers.GlobalAveragePooling3D()(res5) flatten = layers.Flatten()(avg_pool) if logits: logits = layers.Dense(n)(flatten) else: logits = layers.Dense(n, activation='softmax') if include_fc_layer: model = Model(inputs=inputs, outputs=logits) else: model = Model(inputs=inputs, outputs=res5) return model
def ResNet(stack_fn, preact, use_bias, model_name='resnet', include_top=True, input_tensor=None, input_shape=None, pooling=None, classes=1000, **kwargs): """Instantiates the ResNet, ResNetV2, and ResNeXt architecture. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments stack_fn: a function that returns output tensor for the stacked residual blocks. preact: whether to use pre-activation or not (True for ResNetV2, False for ResNet and ResNeXt). use_bias: whether to use biases for convolutional layers or not (True for ResNet and ResNetV2, False for ResNeXt). model_name: string, model name. include_top: whether to include the fully-connected layer at the top of the network. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 224)` (with `channels_first` data format). It should have exactly 3 inputs channels. pooling: optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 x = layers.ZeroPadding3D(padding=3, name='conv1_pad')(img_input) x = layers.Conv3D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x) if preact is False: x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='conv1_bn')(x) x = layers.Activation('relu', name='conv1_relu')(x) x = layers.ZeroPadding3D(padding=1, name='pool1_pad')(x) x = layers.MaxPooling3D(3, strides=2, name='pool1_pool')(x) x = stack_fn(x) if preact is True: x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='post_bn')(x) x = layers.Activation('relu', name='post_relu')(x) if include_top: x = layers.GlobalAveragePooling3D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='probs')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling3D(name='avg_pool')(x) elif pooling == 'max': x = layers.GlobalMaxPooling3D(name='max_pool')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = keras_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name=model_name) return model
def main(): arg_list = None args = parseArgs(arg_list) # grab training data filepath = 'data/train_sample_videos' datapath = os.path.join(filepath, 'metadata.json') data = pd.read_json(datapath).T if args.sample: files = [os.path.join(filepath, f) for f in data.index][:20] labels = data.label.values[:20] else: files = [os.path.join(filepath, f) for f in data.index] labels = data.label.values x_train, x_test, y_train, y_test = train_test_split( files, labels, test_size=float(args.test_split)) class_weights = compute_class_weight( 'balanced', np.unique(y_train), y_train) for k, v in zip(np.unique(y_train), class_weights): print(k, v) y_train = list(map(lambda x: 0 if x == 'REAL' else 1, y_train)) y_test = list(map(lambda x: 0 if x == 'REAL' else 1, y_test)) y_train = to_categorical(y_train, num_classes=2) y_test = to_categorical(y_test, num_classes=2) print(len(x_train), len(y_train), len(x_test), len(y_test)) # validation data val_path = 'data/test_videos' if args.sample: val_files = [os.path.join(val_path, f) for f in os.listdir(val_path)][:8] else: val_files = [os.path.join(val_path, f) for f in os.listdir(val_path)] print('number of validation files', len(val_files)) # generate datasets batch_size = args.batch_size segment_size = args.segment_size rsz = (128, 128) train_data = input_fn( x_train, y_train, segment_size=segment_size, batch_size=batch_size, rsz=rsz) test_data = input_fn( x_test, y_test, segment_size=segment_size, batch_size=batch_size, rsz=rsz) val_data = input_fn( files=val_files, segment_size=segment_size, batch_size=batch_size, rsz=rsz) rgb_input = tf.keras.Input( shape=(segment_size, rsz[0], rsz[1], 3), name='rgb_input') flow_input = tf.keras.Input( shape=(segment_size - 1, rsz[0], rsz[1], 2), name='flow_input') # TODO: make OO # RGB MODEL # block 1 x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(rgb_input) x = layers.Conv3D( filters=8, kernel_size=4, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(x) block1_output = layers.MaxPool3D( pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same' )(x) # block 2 x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(block1_output) x = layers.Conv3D( filters=8, kernel_size=4, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(x) block2_output = layers.add([x, block1_output]) # block 3 x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(block2_output) x = layers.Conv3D( filters=8, kernel_size=4, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(x) block3_output = layers.add([x, block2_output]) x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(block3_output) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(64, activation='relu')(x) x = layers.Dropout(0.5)(x) rgb_outputs = layers.Dense(2, activation='softmax')(x) rgb_model = Model(inputs=rgb_input, outputs=rgb_outputs) rgb_model.summary() # FLOW MODEL x = layers.ConvLSTM2D( filters=8, kernel_size=3, strides=1, padding='same', data_format='channels_last', return_sequences=True, dropout=0.5 )(flow_input) x = layers.BatchNormalization()(x) x = layers.ConvLSTM2D( filters=8, kernel_size=3, strides=1, padding='same', data_format='channels_last', return_sequences=True, dropout=0.5 )(x) x = layers.BatchNormalization()(x) x = layers.ConvLSTM2D( filters=8, kernel_size=3, strides=1, padding='same', data_format='channels_last', return_sequences=False, dropout=0.5 )(x) x = layers.BatchNormalization()(x) x = layers.Flatten()(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dropout(0.5)(x) flow_output = layers.Dense(2)(x) flow_model = Model(inputs=flow_input, outputs=flow_output) flow_model.summary() # FINAL MODEL final_average = layers.average([rgb_outputs, flow_output]) x = layers.Flatten()(final_average) final_output = layers.Dense( 2, activation='softmax', name='final_output')(x) model = Model( inputs={"rgb_input": rgb_input, "flow_input": flow_input}, outputs=final_output, name='my_model' ) model.summary() # tf.keras.utils.plot_model( # model, # to_file='model.png', # show_shapes=True, # show_layer_names=True # ) # TRAIN dt = datetime.now().strftime('%Y%m%d_%H%M%S') opt = tf.keras.optimizers.Adam() if args.save_checkpoints: save_path = f'data/model_checkpoints/{dt}/ckpt' ckpt = tf.keras.callbacks.ModelCheckpoint( filepath=save_path, save_best_only=False, save_weights_only=True ) ckpt = [ckpt] else: ckpt = [] model.compile( optimizer=opt, loss='categorical_crossentropy', metrics=['acc']) model.fit( x=train_data.repeat(), validation_data=test_data.repeat(), epochs=args.epochs, verbose=1, class_weight=class_weights, steps_per_epoch=len(x_train) // batch_size, validation_steps=len(x_test) // batch_size, callbacks=ckpt ) # EVAL print('\n\n---------------------------------------------------------') print('predicting on validation data') start = time.time() preds = model.predict( val_data, verbose=1, steps=len(val_files) // batch_size ) print('prediction time: ', time.time() - start) preds = np.argmax(preds, axis=1) df = pd.DataFrame(columns=['filename', 'label']) df.filename = [v.split('/')[-1] for v in val_files] df.label = preds df.to_csv(f'data/submission_{dt}.csv', index=False)