def AttentionResNet18(shape=(224, 224, 3), n_channels=64, n_classes=5, dropout=0, regularization=0.01): """ Attention ResNet-18 with an attention-layer, variable imput size """ regularizer = l2(regularization) input_ = Input(shape=shape) x = Conv2D(n_channels, (7, 7), strides=(2, 2), padding='same')(input_) # 112x112 x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(3, 3), strides=( 2, 2), padding='same')(x) # 56x56 x = residual_block(x, output_channels=n_channels * 16, stride=4) # 14x14 # ATTENTION LAYER x = attention_block(x, encoder_depth=1) # bottleneck 7x7 x = residual_block(x, output_channels=n_channels * 32, stride=2) # 7x7 pool_size = (x.get_shape()[1], x.get_shape()[2]) x = AveragePooling2D(pool_size=pool_size, strides=(1, 1))(x) x = Flatten()(x) if dropout: x = Dropout(dropout)(x) output = Dense(n_classes, kernel_regularizer=regularizer, activation='softmax')(x) model = Model(input_, output) return model
def __init__(self, signal, params): self.input = signal self.params = params model = blocks.residual_block(signal, bn=True) for i in range(7): model = blocks.residual_block(model) self.logits = tf.keras.layers.Dense(5)(model)
def __init__(self, signal, params): self.input = signal self.params = params model = blocks.residual_block(signal, bn=True) for i in range(4): model = blocks.residual_block(model) max_dilation = 64 i = 1 while i <= max_dilation: model = blocks.tcn_block(model, i) i = i * 2 self.logits = tf.keras.layers.Dense(5)(model)
def __init__(self, signal, params): self.input = signal self.params = params model = blocks.residual_block(signal, bn=True) for i in range(4): model = blocks.residual_block(model) skip_connections = [] max_dilation = 64 i = 1 while i <= max_dilation: jump, model = blocks.tcn_block_both_directions(model, i) skip_connections.append(jump) i = i * 2 model = tf.keras.layers.Add()(skip_connections) self.logits = tf.keras.layers.Dense(5)(model)
def AttentionResNetCifar10(shape=(32, 32, 3), n_channels=32, n_classes=10): """ Attention-56 ResNet for Cifar10 Dataset https://arxiv.org/abs/1704.06904 """ input_ = Input(shape=shape) x = Conv2D(n_channels, (5, 5), padding='same')(input_) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 16x16 x = residual_block(x, input_channels=32, output_channels=128) x = attention_block(x, encoder_depth=2) x = residual_block(x, input_channels=128, output_channels=256, stride=2) # 8x8 x = attention_block(x, encoder_depth=1) x = residual_block(x, input_channels=256, output_channels=512, stride=2) # 4x4 x = attention_block(x, encoder_depth=1) x = residual_block(x, input_channels=512, output_channels=1024) x = residual_block(x, input_channels=1024, output_channels=1024) x = residual_block(x, input_channels=1024, output_channels=1024) x = AveragePooling2D(pool_size=(4, 4), strides=(1, 1))(x) # 1x1 x = Flatten()(x) output = Dense(n_classes, activation='softmax')(x) model = Model(input_, output) return model
def AttentionResNetCifar10(shape=(32, 32, 3), n_channels=32, n_classes=10): input_ = Input(shape=shape) x = Conv2D(n_channels, (5, 5), padding='same')(input_) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPool2D(pool_size=(2, 2))(x) # 16x16 x = residual_block(x, input_channels=32, output_channels=128) x = attention_block(x, encoder_depth=2) x = Conv2D(x.get_shape()[-1].value, (1, 1),name='residual_attention_stage1')(x) x = residual_block(x, input_channels=128, output_channels=256, stride=2) # 8x8 x = attention_block(x, encoder_depth=1) x = Conv2D(x.get_shape()[-1].value, (1, 1),name='residual_attention_stage2')(x) x = residual_block(x, input_channels=256, output_channels=512, stride=2) # 4x4 x = attention_block(x, encoder_depth=1) #第三个Attention Module在论文中是不需要要skip_connections的,所以encoder_depth在此处应该为0,不过代码为了方便还是设为1 x = Conv2D(x.get_shape()[-1].value, (1, 1),name='residual_attention_stage3')(x) x = residual_block(x, input_channels=512, output_channels=1024) x = residual_block(x, input_channels=1024, output_channels=1024) x = residual_block(x, input_channels=1024, output_channels=1024) x = AveragePooling2D(pool_size=(4, 4), strides=(1, 1))(x) # 1x1 x = Flatten()(x) output = Dense(n_classes, activation='softmax')(x) model = Model(input_, output) return model
def __init__(self, signal, params): self.input = signal self.params = params max_dilation = 128 model = self.input for i in range(3): model = blocks.residual_block(model, i == 0) i = 1 skip_connections = [] while i <= max_dilation: model, skip = blocks.wavenet_bidirectional_block(model, i) skip_connections.append(skip) i = i * 2 skip_sum = tf.keras.layers.Add()(skip_connections) self.logits = tf.keras.layers.Dense(5)(skip_sum)
def AttentionResNet92(shape=(224, 224, 3), n_channels=64, n_classes=100, dropout=0, regularization=0.01): """ Attention-92 ResNet https://arxiv.org/abs/1704.06904 """ regularizer = l2(regularization) input_ = Input(shape=shape) x = Conv2D(n_channels, (7, 7), strides=(2, 2), padding='same')(input_) # 112x112 x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) # 56x56 x = residual_block(x, output_channels=n_channels * 4) # 56x56 x = attention_block(x, encoder_depth=3) # bottleneck 7x7 x = residual_block(x, output_channels=n_channels * 8, stride=2) # 28x28 x = attention_block(x, encoder_depth=2) # bottleneck 7x7 x = attention_block(x, encoder_depth=2) # bottleneck 7x7 x = residual_block(x, output_channels=n_channels * 16, stride=2) # 14x14 x = attention_block(x, encoder_depth=1) # bottleneck 7x7 x = attention_block(x, encoder_depth=1) # bottleneck 7x7 x = attention_block(x, encoder_depth=1) # bottleneck 7x7 x = residual_block(x, output_channels=n_channels * 32, stride=2) # 7x7 x = residual_block(x, output_channels=n_channels * 32) x = residual_block(x, output_channels=n_channels * 32) pool_size = (x.get_shape()[1].value, x.get_shape()[2].value) x = AveragePooling2D(pool_size=pool_size, strides=(1, 1))(x) x = Flatten()(x) if dropout: x = Dropout(dropout)(x) output = Dense(n_classes, kernel_regularizer=regularizer, activation='softmax')(x) model = Model(input_, output) return model
def __call__(self, x, reuse=False, is_training=True): nb_upsampling = int(np.log2(self.target_size[0] // 16)) with tf.variable_scope(self.name) as vs: if reuse: vs.reuse_variables() _x = first_block(x, (16, 16), self.noise_dim, 'deconv', self.normalization, is_training) residual_inputs = conv_block(_x, is_training=is_training, filters=64, activation_='relu', sampling='same', normalization=self.normalization, dropout_rate=0., mode='conv_first') with tf.variable_scope('residual_blocks'): for i in range(16): _x = residual_block(_x, is_training=is_training, filters=64, activation_='relu', sampling='same', normalization=self.normalization, dropout_rate=0., mode='conv_first') _x = conv_block(_x, is_training=is_training, filters=64, activation_='relu', sampling='same', normalization=self.normalization, dropout_rate=0., mode='conv_first') _x += residual_inputs with tf.variable_scope('upsampling_blocks'): for i in range(nb_upsampling): _x = conv_block(_x, is_training=is_training, filters=64, activation_='relu', sampling=self.upsampling, normalization=self.normalization, dropout_rate=0., mode='conv_first') __x = _x for _ in range(3): __x = conv_block(__x, is_training=is_training, filters=64, activation_='relu', sampling='same', normalization=self.normalization, dropout_rate=0., mode='conv_first') _x += __x _x = conv_block(_x, is_training=is_training, kernel_size=(9, 9), filters=self.channel, activation_=self.last_activation, sampling='same', normalization=None, dropout_rate=0., mode='conv_first') return _x