def build_single_unet_per_channel(input_shape, last_activation, n_depth=2, n_filter_base=16, kernel_size=(3, 3, 3), n_conv_per_depth=2, activation="relu", batch_norm=False, dropout=0.0, pool_size=(2, 2, 2), residual=False, prob_out=False, eps_scale=1e-3): """ TODO """ if last_activation is None: raise ValueError( "last activation has to be given (e.g. 'sigmoid', 'relu')!") all((s % 2 == 1 for s in kernel_size)) or _raise( ValueError('kernel size should be odd in all dimensions.')) channel_axis = -1 if backend_channels_last() else 1 n_dim = len(kernel_size) conv = Conv2D if n_dim == 2 else Conv3D num_channels = input_shape[channel_axis] input = Input(input_shape, name="input") out_channels = [] num_channel_out = 1 for i in range(num_channels): c = Lambda(lambda x: x[:, ..., i:i + 1])(input) unet = unet_block(n_depth, n_filter_base, kernel_size, activation=activation, dropout=dropout, batch_norm=batch_norm, n_conv_per_depth=n_conv_per_depth, pool=pool_size, prefix='channel_{}'.format(i))(c) final = conv(num_channel_out, (1, ) * n_dim, activation='linear')(unet) if residual: if not (num_channel_out == 1 if backend_channels_last() else num_channel_out == 1): raise ValueError( "number of input and output channels must be the same for a residual net." ) final = Add()([final, input]) final = Activation(activation=last_activation)(final) if prob_out: scale = conv(num_channel_out, (1, ) * n_dim, activation='softplus')(unet) scale = Lambda(lambda x: x + np.float32(eps_scale))(scale) final = Concatenate(axis=channel_axis)([final, scale]) out_channels.append(final) if len(out_channels) > 1: output = Concatenate(axis=channel_axis)(out_channels) return Model(inputs=input, outputs=output) else: return Model(inputs=input, outputs=out_channels[0])
def Attention_Wavenet(n_steps, n_features, activation): # convolutional operation parameters n_filters = 32 filter_width = 2 dilation_rates = [2**i for i in range(9)] * 3 # define an input history series and pass it through a stack of dilated causal convolution blocks. history_seq = Input(shape=[n_steps, n_features]) x = history_seq skips = [] for dilation_rate in dilation_rates: # preprocessing - equivalent to time-distributed dense x = Conv1D(16, 1, padding='same', activation=activation)(x) # filter convolution x_f = Conv1D(filters=n_filters, kernel_size=filter_width, padding='causal', dilation_rate=dilation_rate)(x) x_f = Dropout(0.1)(x_f) # gating convolution x_g = Conv1D(filters=n_filters, kernel_size=filter_width, padding='causal', dilation_rate=dilation_rate)(x) x_g = Dropout(0.1)(x_g) # multiply filter and gating branches z = Multiply()([Activation('tanh')(x_f), Activation('sigmoid')(x_g)]) # postprocessing - equivalent to time-distributed dense z = Conv1D(16, 1, padding='same', activation=activation)(z) # residual connection x = Add()([x, z]) # collect skip connections skips.append(z) # add all skip connection outputs out = Activation(activation)(Add()(skips)) out = Attention(use_scale=True, causal=True)([out, out]) # final time-distributed dense layers out = Conv1D(16, 1, activation=activation, padding='same')(out) out = MaxPool1D(pool_size=2)(out) out = keras.layers.Flatten()(out) out = Dense(get_output_dim(n_steps * n_features), kernel_initializer='lecun_normal', activation='selu')(out) encoder = Model(inputs=[history_seq], outputs=[out]) decoder = keras.models.Sequential([ keras.layers.Reshape( [get_output_dim(n_steps * n_features), 1, 1], input_shape=[get_output_dim(n_steps * n_features)]), Conv2DTranspose(filters=32, kernel_size=3, activation=activation), Conv2DTranspose(filters=16, kernel_size=3, activation=activation), keras.layers.Flatten(), Dense(n_steps * n_features), keras.layers.Reshape([n_steps, n_features]) ]) return encoder, decoder
def create_uNet(input_shape, nclasses, filters=[30, 45, 60], lambda_regularization=None, activation='elu'): if lambda_regularization is not None: lambda_regularization = keras.regularizers.l2(lambda_regularization) tensor_list = [] input_tensor = Input(shape=input_shape, name="input") # you could make a loop # to append and pop # 256x256 tensor = Convolution2D(filters[0], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(input_tensor) tensor = Convolution2D(filters[1], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) ############################# tensor_list.append(tensor) tensor = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(tensor) # 128x128 tensor = Convolution2D(filters[2], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) tensor = Convolution2D(filters[1], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) tensor_list.append(tensor) tensor = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(tensor) # 64x64 tensor = Convolution2D(filters[2], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) tensor = Convolution2D(filters[1], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) # upsample # 128x128 tensor = UpSampling2D(size=2)( tensor) # take 1 pixel and expand it out to 2 x 2 tensor = Convolution2D(filters[1], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) tensor = Add()([tensor, tensor_list.pop()]) # upsample tensor = UpSampling2D(size=2)( tensor) # take 1 pixel and expand it out to 2 x 2 # 256 x 256 tensor = Convolution2D(filters[1], kernel_size=(3, 3), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation=activation)(tensor) tensor = Add()([tensor, tensor_list.pop()]) ############################# output_tensor = Convolution2D(nclasses, kernel_size=(1, 1), padding='same', use_bias=True, kernel_initializer='random_uniform', bias_initializer='zeros', kernel_regularizer=lambda_regularization, activation='softmax', name='output')(tensor) model = Model(inputs=input_tensor, outputs=output_tensor) opt = keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(), optimizer=opt, metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]) return model
# The discriminator_model is more complex. It takes both real image samples and random # noise seeds as input. The noise seed is run through the generator model to get # generated images. Both real and generated images are then run through the # discriminator. Although we could concatenate the real and generated images into a # single tensor, we don't (see model compilation for why). real_samples = Input(shape=X_train.shape[1:]) generator_input_for_discriminator = Input(shape=(100, )) generated_samples_for_discriminator = generator( generator_input_for_discriminator) discriminator_output_from_generator = discriminator( generated_samples_for_discriminator) discriminator_output_from_real_samples = discriminator(real_samples) # We also need to generate weighted-averages of real and generated samples, # to use for the gradient norm penalty. averaged_samples = Add()([real_samples, generated_samples_for_discriminator]) # We then run these samples through the discriminator as well. Note that we never # really use the discriminator output for these samples - we're only running them to # get the gradient norm for the gradient penalty loss. averaged_samples_out = discriminator(averaged_samples) # The gradient penalty loss function requires the input averaged samples to get # gradients. However, Keras loss functions can only have two arguments, y_true and # y_pred. We get around this by making a partial() of the function with the averaged # samples here. partial_gp_loss = partial(gradient_penalty_loss, averaged_samples=averaged_samples, gradient_penalty_weight=GRADIENT_PENALTY_WEIGHT) # Functions need names or Keras will throw an error partial_gp_loss.__name__ = 'gradient_penalty'
# convolution을 연결할 때 (some possible problems: gradient vanishing) # input(784) -> Dense(32)- relu--> * Dense(32) -relu- -> Dense(32) -relu- -> Dense(10) -softmax- # 필터를 통과할 수록 이미지의 크기가 작아진다 # ResidualNet은 * 와 같은 지점에서 건너 띄어서 바로 출력층으로 가는 것? # input(784) -> Dense(32)- relu - + add -> Dense(10) -softmax- # 지름길로 가버리는 것 # + add 가 일어난다 # + 와 concatenate을 구별해야한다 # + : element, element끼리 더하기 -> 32가 입력으로 들어온다 (더하기의 개념) -> 32개와 32개를 더해줌 i.e. ((1,2),(3,4)) + ((5,6),(7,8)) = ((6, 8), (10, 12)) # concatenate: 더해주는 것 -> 64개가 입력으로 들어오는 것 i.e. ((1,2),(3,4)) + ((5,6),(7,8)) = ((1,2),(3,4),(5,6),(7,8)) input_tensor = Input(shape=(784, )) sc = Dense(32, activation='relu')(input_tensor) # 첫번째 Dense를 지나서 나온 출력값을 sc라고 부름 x = Dense(32, activation='relu')(sc) x = Dense(32, activation='relu')(x) x = Add()([x, sc]) # concatenate은 함수라서 () 에 값을 줬지만, Add는 클래스이름이라 먼저 호출 하고 뒤에 붙여줌 output_tensor = Dense(10, 'softmax')(x) # 개념적으로 구현해 보았다 # Model 생성 model = Model(input_tensor, output_tensor) model.summary() # n_input * n_output + n_b # 784 * 32 + 32 = 25120 # 32 * 32 + 32 = 1056 # 신경망이 깊어지면 깊어질 수록 기울기가 소실된다는 문제가 있었다 # 지름길을 통과하는 네트워크를 하나를 만들어서 기울기 손실이 없는 네트워크 한개를 끝까지 보내면 -> 마지막 단계에서 소실되어도 모든 망을 통과한 값 + 소실되지 않은 값을 더해줘서 계산해주는
def getResidualBlock(I, filter_size, featmaps, stage, block, shortcut, convArgs, bnArgs, d): """Get residual block.""" activation = d.act drop_prob = d.dropout nb_fmaps1, nb_fmaps2 = featmaps conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' #if K.image_data_format() == 'channels_first' and K.ndim(I) != 3: # channel_axis = 1 #else: channel_axis = -1 if d.model == "real": O = BatchNormalization(name=bn_name_base + '_2a', **bnArgs)(I) elif d.model == "complex": O = ComplexBN(name=bn_name_base + '_2a', **bnArgs)(I) O = Spline()(O) #Activation(activation)(O) if shortcut == 'regular' or d.spectral_pool_scheme == "nodownsample": if d.model == "real": O = Conv2D(nb_fmaps1, filter_size, name=conv_name_base + '2a', **convArgs)(O) elif d.model == "complex": O = ComplexConv2D(nb_fmaps1, filter_size, name=conv_name_base + '2a', **convArgs)(O) elif shortcut == 'projection': if d.spectral_pool_scheme == "proj": O = applySpectralPooling(O, d) if d.model == "real": O = Conv2D(nb_fmaps1, filter_size, name=conv_name_base + '2a', strides=(2, 2), **convArgs)(O) elif d.model == "complex": O = ComplexConv2D(nb_fmaps1, filter_size, name=conv_name_base + '2a', strides=(2, 2), **convArgs)(O) if d.model == "real": O = BatchNormalization(name=bn_name_base + '_2b', **bnArgs)(O) O = Spline()(O) #Activation(activation)(O) O = Conv2D(nb_fmaps2, filter_size, name=conv_name_base + '2b', **convArgs)(O) elif d.model == "complex": O = ComplexBN(name=bn_name_base + '_2b', **bnArgs)(O) O = Spline()(O) #Activation(activation)(O) O = ComplexConv2D(nb_fmaps2, filter_size, name=conv_name_base + '2b', **convArgs)(O) if shortcut == 'regular': O = Add()([O, I]) elif shortcut == 'projection': if d.spectral_pool_scheme == "proj": I = applySpectralPooling(I, d) if d.model == "real": X = Conv2D( nb_fmaps2, (1, 1), name=conv_name_base + '1', strides=(2, 2) if d.spectral_pool_scheme != "nodownsample" else (1, 1), **convArgs)(I) O = Concatenate(channel_axis)([X, O]) elif d.model == "complex": X = ComplexConv2D( nb_fmaps2, (1, 1), name=conv_name_base + '1', strides=(2, 2) if d.spectral_pool_scheme != "nodownsample" else (1, 1), **convArgs)(I) O_real = Concatenate(channel_axis)( [X[..., :X.shape[-1] // 2], O[..., :O.shape[-1] // 2]]) O_imag = Concatenate(channel_axis)( [X[..., X.shape[-1] // 2:], O[..., O.shape[-1] // 2:]]) O = Concatenate(channel_axis)([O_real, O_imag]) return O
def _make_block_basic( self, input_tensor, first_block=True, filters=64, stride=2, radix=1, avd=False, avd_first=False, is_first=False ): """Conv2d_BN_Relu->Bn_Relu_Conv2d """ x = input_tensor x = BatchNormalization(axis=self.channel_axis, epsilon=1.001e-5)(x) x = Activation(self.active)(x) short_cut = x inplanes = input_tensor.shape[-1] if stride != 1 or inplanes != filters * self.block_expansion: if self.avg_down: if self.dilation == 1: short_cut = AveragePooling2D(pool_size=stride, strides=stride, padding="same", data_format="channels_last")( short_cut ) else: short_cut = AveragePooling2D(pool_size=1, strides=1, padding="same", data_format="channels_last")(short_cut) short_cut = Conv2D( filters, kernel_size=1, strides=1, padding="same", kernel_initializer="he_normal", use_bias=False, data_format="channels_last", )(short_cut) else: short_cut = Conv2D( filters, kernel_size=1, strides=stride, padding="same", kernel_initializer="he_normal", use_bias=False, data_format="channels_last", )(short_cut) group_width = int(filters * (self.bottleneck_width / 64.0)) * self.cardinality avd = avd and (stride > 1 or is_first) avd_first = avd_first if avd: avd_layer = AveragePooling2D(pool_size=3, strides=stride, padding="same", data_format="channels_last") stride = 1 if avd and avd_first: x = avd_layer(x) if radix >= 1: x = self._SplAtConv2d( x, filters=group_width, kernel_size=3, stride=stride, dilation=self.dilation, groups=self.cardinality, radix=radix, ) else: x = Conv2D( filters, kernel_size=3, strides=stride, padding="same", kernel_initializer="he_normal", dilation_rate=self.dilation, use_bias=False, data_format="channels_last", )(x) if avd and not avd_first: x = avd_layer(x) # print('can') x = BatchNormalization(axis=self.channel_axis, epsilon=1.001e-5)(x) x = Activation(self.active)(x) x = Conv2D( filters, kernel_size=3, strides=1, padding="same", kernel_initializer="he_normal", dilation_rate=self.dilation, use_bias=False, data_format="channels_last", )(x) m2 = Add()([x, short_cut]) return m2
filters=12, kernel_size=(1, 1), padding='valid', data_format='channels_last', use_bias=True)(B2) B3 = LeakyReLU()(B3) B4 = LocallyConnected2D(name="layerB4", filters=15, kernel_size=(2, 3), strides=(2, 2), padding='valid', data_format='channels_last', activation=None, use_bias=True)(B3) C = Add()([B4, in1merge]) C = LeakyReLU()(C) #D = LocallyConnected2D( name="layerD", filters=10, kernel_size=(1,1), strides=(1,1), padding='valid', data_format='channels_last', use_bias=True )( C ) #D = LeakyReLU()( D ) E = LocallyConnected2D(name="layerE", filters=15, kernel_size=(2, 1), strides=(2, 1), padding='valid', data_format='channels_last', use_bias=True)(C) E = LeakyReLU()(E) #Epool = MaxPooling2D(pool_size=(2, 1), strides=(2,1), padding='valid', data_format='channels_last')( E ) #print( Epool.shape ) F = LocallyConnected2D(name="layerF",
def convolutional_block(X, f, filters, stage, block, s=2): """ Implementation of the convolutional block as defined in Figure 4 Arguments: X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev) f -- integer, specifying the shape of the middle CONV's window for the main path filters -- python list of integers, defining the number of filters in the CONV layers of the main path stage -- integer, used to name the layers, depending on their position in the network block -- string/character, used to name the layers, depending on their position in the network s -- Integer, specifying the stride to be used Returns: X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C) """ # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value X_shortcut = X ##### MAIN PATH ##### # First component of main path X = Conv2D(F1, (1, 1), strides=(s, s), name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X) X = Activation('relu')(X) # Second component of main path (≈3 lines) X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path (≈2 lines) X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X) ##### SHORTCUT PATH #### (≈2 lines) X_shortcut = Conv2D(filters=F3, kernel_size=(1, 1), strides=(s, s), padding='valid', name=conv_name_base + '1', kernel_initializer=glorot_uniform(seed=0))(X_shortcut) X_shortcut = BatchNormalization(axis=3, name=bn_name_base + '1')(X_shortcut) # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines) X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X
from tensorflow.keras.models import Model from tensorflow.keras.layers import Input, Activation, Concatenate, Add, Multiply from unit_test.helper import tf_random_seed model_list = [] for activation_item in ['relu', 'sigmoid', 'softmax']: for merge_item in [Concatenate(axis=-1), Add(), Multiply()]: tf_random_seed() placeholder = Input((32, 32, 3), name='data') x_0 = Activation(activation_item)(placeholder) x_1 = Activation(activation_item)(placeholder) x = merge_item([x_0, x_1]) model = Model( placeholder, x, name= f'model_single_layer_{activation_item}_{merge_item.__class__.__name__.lower()}' ) model_list.append(model)
def fireModule(x, squeeze=16, expand=64, kernels=((1, 1), (3, 3)), bypass=False, complexBypass=False): if (len(kernels) != 2): raise Exception( 'The length of kernels must equal 2\nGiven kernel: %s' % str(kernels)) lowKernel = kernels[0] highKernel = kernels[1] y = Conv2D(squeeze, lowKernel, padding='same', name='%d_%dConv%dSqueeze' % (lowKernel[0], lowKernel[1], callCounter.counter))(x) y = Activation('relu', name='Relu1%d' % callCounter.counter)(y) left = Conv2D(expand, highKernel, padding='same', name='%d_%dConv%dLeft' % (highKernel[0], highKernel[1], callCounter.counter))(y) left = Activation('relu', name='Relu2%d' % callCounter.counter)(left) right = Conv2D(expand, lowKernel, padding='same', name='%d_%dConv%dRight' % (lowKernel[0], lowKernel[1], callCounter.counter))(y) right = Activation('relu', name='Relu3%d' % callCounter.counter)(right) concat = Concatenate(axis=-1)([left, right]) if (complexBypass): additionalConv = Conv2D(expand * 2, (1, 1), name='1_1AdditionalConv%d' % callCounter.counter)(x) concat = Add()([concat, additionalConv]) elif (bypass): if (str(x.shape) != str(concat.shape)): raise Exception( 'The shape of input and output must be same.\nInput shape: %s\nOutput shape: %s' % (str(x.shape), str(concat.shape))) concat = Add(name='Add%d' % callCounter.counter)([concat, x]) callCounter.counter += 1 return concat
def Segnet(height=576,width=576,channel=3,n_labels=2): input_shape = (height, width, channel) kernel = 3 args = {"ksize": (1,2,2,1), "strides":(1,2,2,1)} pool_size = (1,2,2,1) inputs = Input(input_shape) x = Conv2D(64, (kernel, kernel), padding="same",name="conv_1_1")(inputs) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(64, (kernel, kernel), padding="same",name="conv_1_2")(x) x = BatchNormalization()(x) x = Activation("relu")(x) pool_1, mask_1 = Lambda(MaxPool2DWithArgmax, arguments=args)(x) x = Conv2D(128, (kernel, kernel), padding="same",name="conv_2_1")(pool_1) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(128, (kernel, kernel), padding="same",name="conv_2_2")(x) x = BatchNormalization()(x) x = Activation("relu")(x) model_A = Model(inputs=inputs,outputs=x) model_B = input_other() # combined = Concatenate()([model_A.output,model_B.output]) combined = Add()([model_A.output,model_B.output]) x = combined pool_2, mask_2 = Lambda(MaxPool2DWithArgmax, arguments=args)(x) x = Conv2D(256, (kernel, kernel), padding="same")(pool_2) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(256, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(256, (1, 1), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) pool_3, mask_3 = Lambda(MaxPool2DWithArgmax, arguments=args)(x) x = Conv2D(512, (kernel, kernel), padding="same")(pool_3) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (1, 1), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) pool_4, mask_4 = Lambda(MaxPool2DWithArgmax, arguments=args)(x) x = Conv2D(512, (kernel, kernel), padding="same")(pool_4) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (1, 1), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) pool_5, mask_5 = Lambda(MaxPool2DWithArgmax, arguments=args)(x) # ====================================================================== x = Lambda(Unpool2D, arguments={"factor": pool_size})([pool_5, mask_5]) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_4]) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(512, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(256, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_3]) x = Conv2D(256, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(256, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(128, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_2]) x = Conv2D(128, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(64, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Lambda(Unpool2D, arguments={"factor": pool_size})([x, mask_1]) x = Conv2D(64, (kernel, kernel), padding="same")(x) x = BatchNormalization()(x) x = Activation("relu")(x) x = Conv2D(n_labels, (1, 1), padding="valid")(x) x = BatchNormalization()(x) x = Activation("relu")(x) # x = Reshape( # (input_shape[0] * input_shape[1], n_labels), # input_shape=(input_shape[0], input_shape[1], n_labels), # )(x) outputs = Activation("softmax")(x) return Model(inputs=[model_A.input,model_B.input], outputs=outputs, name="MySegNet_4")
def _build_model(self): input_layer = self._input() d0 = input_layer d0 = self._conv_k3_activation_possible_batchnorm(64)(d0) d0 = self._conv_k3_activation_possible_batchnorm(64)(d0) d1 = self._max_pooling()(d0) d1 = self._conv_k3_activation_possible_batchnorm(128)(d1) d1 = self._conv_k3_activation_possible_batchnorm(128)(d1) d2 = self._max_pooling()(d1) d2 = self._conv_k3_activation_possible_batchnorm(256)(d2) d2 = self._conv_k3_activation_possible_batchnorm(256)(d2) d3 = self._max_pooling()(d2) d3 = self._conv_k3_activation_possible_batchnorm(512)(d3) d3 = self._conv_k3_activation_possible_batchnorm(512)(d3) d3 = self._possible_dropout()(d3) d4 = self._max_pooling()(d3) d4 = self._conv_k3_activation_possible_batchnorm(1024)(d4) d4 = self._conv_k3_activation_possible_batchnorm(1024)(d4) d4 = self._possible_dropout()(d4) u3 = self._trans_conv(512)(d4) u3 = Concatenate()([d3, u3]) u3 = self._conv_k3_activation_possible_batchnorm(512)(u3) u3 = self._conv_k3_activation_possible_batchnorm(512)(u3) u2 = self._trans_conv(256)(u3) u2 = Concatenate()([d2, u2]) u2 = self._conv_k3_activation_possible_batchnorm(256)(u2) u2 = self._conv_k3_activation_possible_batchnorm(256)(u2) u1 = self._trans_conv(128)(u2) u1 = Concatenate()([d1, u1]) u1 = self._conv_k3_activation_possible_batchnorm(128)(u1) u1 = self._conv_k3_activation_possible_batchnorm(128)(u1) u0 = self._trans_conv(64)(u1) u0 = Concatenate()([d0, u0]) u0 = self._conv_k3_activation_possible_batchnorm(64)(u0) u0 = self._conv_k3_activation_possible_batchnorm(64)(u0) difference_layer = self.get_difference_layer()(u0) output_layer = Add(name=self._output_name)( [input_layer, difference_layer]) if self._output_difference_layer: model = Model(inputs=input_layer, outputs=[output_layer, difference_layer], name=self.name) else: model = Model(inputs=input_layer, outputs=output_layer, name=self.name) self._model = model self._input_layer = input_layer self._output_layer = output_layer self._difference_layer = difference_layer
def residual_block(X, filters): shortcut = X X = conv_block(X, filters, 5, 1) X = Add()([shortcut, X]) X = common_layers(X) return X
def get_train_model(config): h, w = config.IMAGE_SHAPE[:2] if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6): raise Exception("Image size must be dividable by 2 at least 6 times " "to avoid fractions when downscaling and upscaling." "For example, use 256, 320, 384, 448, 512, ... etc. ") # 输入进来的图片必须是2的6次方以上的倍数 input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]], name="input_image") # meta包含了一些必要信息 input_image_meta = Input(shape=[config.IMAGE_META_SIZE], name="input_image_meta") # RPN建议框网络的真实框信息 input_rpn_match = Input(shape=[None, 1], name="input_rpn_match", dtype=tf.int32) input_rpn_bbox = Input(shape=[None, 4], name="input_rpn_bbox", dtype=tf.float32) # 种类信息 input_gt_class_ids = Input(shape=[None], name="input_gt_class_ids", dtype=tf.int32) # 框的位置信息 input_gt_boxes = Input(shape=[None, 4], name="input_gt_boxes", dtype=tf.float32) # 标准化到0-1之间 gt_boxes = Lambda(lambda x: norm_boxes_graph(x, K.shape(input_image)[1:3]))( input_gt_boxes) # mask语义分析信息 # [batch, height, width, MAX_GT_INSTANCES] if config.USE_MINI_MASK: input_gt_masks = Input( shape=[config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None], name="input_gt_masks", dtype=bool) else: input_gt_masks = Input( shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None], name="input_gt_masks", dtype=bool) # 获得Resnet里的压缩程度不同的一些层 _, C2, C3, C4, C5 = get_resnet(input_image, stage5=True, train_bn=config.TRAIN_BN) # 组合成特征金字塔的结构 # P5长宽共压缩了5次 # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5) # P4长宽共压缩了4次 # Height/16,Width/16,256 P4 = Add(name="fpn_p4add")([ UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4) ]) # P4长宽共压缩了3次 # Height/8,Width/8,256 P3 = Add(name="fpn_p3add")([ UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3) ]) # P4长宽共压缩了2次 # Height/4,Width/4,256 P2 = Add(name="fpn_p2add")([ UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2) ]) # 各自进行一次256通道的卷积,此时P2、P3、P4、P5通道数相同 # Height/4,Width/4,256 P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p2")(P2) # Height/8,Width/8,256 P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p3")(P3) # Height/16,Width/16,256 P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p4")(P4) # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p5")(P5) # 在建议框网络里面还有一个P6用于获取建议框 # Height/64,Width/64,256 P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5) # P2, P3, P4, P5, P6可以用于获取建议框 rpn_feature_maps = [P2, P3, P4, P5, P6] # P2, P3, P4, P5用于获取mask信息 mrcnn_feature_maps = [P2, P3, P4, P5] anchors = get_anchors(config, config.IMAGE_SHAPE) # 拓展anchors的shape,第一个维度拓展为batch_size anchors = np.broadcast_to(anchors, (config.BATCH_SIZE, ) + anchors.shape) # 将anchors转化成tensor的形式 anchors = Lambda(lambda x: tf.Variable(anchors), name="anchors")(input_image) # 建立RPN模型 rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) rpn_class_logits, rpn_class, rpn_bbox = [], [], [] # 获得RPN网络的预测结果,进行格式调整,把五个特征层的结果进行堆叠 for p in rpn_feature_maps: logits, classes, bbox = rpn([p]) rpn_class_logits.append(logits) rpn_class.append(classes) rpn_bbox.append(bbox) rpn_class_logits = Concatenate(axis=1, name="rpn_class_logits")(rpn_class_logits) rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class) rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox) # 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是 # rpn_class_logits : Batch_size, num_anchors, 2 # rpn_class : Batch_size, num_anchors, 2 # rpn_bbox : Batch_size, num_anchors, 4 proposal_count = config.POST_NMS_ROIS_TRAINING # Batch_size, proposal_count, 4 rpn_rois = ProposalLayer(proposal_count=proposal_count, nms_threshold=config.RPN_NMS_THRESHOLD, name="ROI", config=config)([rpn_class, rpn_bbox, anchors]) active_class_ids = Lambda(lambda x: parse_image_meta_graph(x)[ "active_class_ids"])(input_image_meta) if not config.USE_RPN_ROIS: # 使用外部输入的建议框 input_rois = Input(shape=[config.POST_NMS_ROIS_TRAINING, 4], name="input_roi", dtype=np.int32) # Normalize coordinates target_rois = Lambda( lambda x: norm_boxes_graph(x, K.shape(input_image)[1:3]))(input_rois) else: # 利用预测到的建议框进行下一步的操作 target_rois = rpn_rois """找到建议框的ground_truth Inputs: proposals: [batch, N, (y1, x1, y2, x2)]建议框 gt_class_ids: [batch, MAX_GT_INSTANCES]每个真实框对应的类 gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]真实框的位置 gt_masks: [batch, height, width, MAX_GT_INSTANCES]真实框的语义分割情况 Returns: rois: [batch, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)]内部真实存在目标的建议框 target_class_ids: [batch, TRAIN_ROIS_PER_IMAGE]每个建议框对应的类 target_deltas: [batch, TRAIN_ROIS_PER_IMAGE, (dy, dx, log(dh), log(dw)]每个建议框应该有的调整参数 target_mask: [batch, TRAIN_ROIS_PER_IMAGE, height, width]每个建议框语义分割情况 """ rois, target_class_ids, target_bbox, target_mask =\ DetectionTargetLayer(config, name="proposal_targets")([ target_rois, input_gt_class_ids, gt_boxes, input_gt_masks]) # 找到合适的建议框的classifier预测结果 mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\ fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta, config.POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN, fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) # 找到合适的建议框的mask预测结果 mrcnn_mask = build_fpn_mask_graph(rois, mrcnn_feature_maps, input_image_meta, config.MASK_POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN) output_rois = Lambda(lambda x: x * 1, name="output_rois")(rois) # Losses rpn_class_loss = Lambda(lambda x: rpn_class_loss_graph(*x), name="rpn_class_loss")( [input_rpn_match, rpn_class_logits]) rpn_bbox_loss = Lambda(lambda x: rpn_bbox_loss_graph(config, *x), name="rpn_bbox_loss")( [input_rpn_bbox, input_rpn_match, rpn_bbox]) class_loss = Lambda(lambda x: mrcnn_class_loss_graph(*x), name="mrcnn_class_loss")([ target_class_ids, mrcnn_class_logits, active_class_ids ]) bbox_loss = Lambda(lambda x: mrcnn_bbox_loss_graph(*x), name="mrcnn_bbox_loss")( [target_bbox, target_class_ids, mrcnn_bbox]) mask_loss = Lambda(lambda x: mrcnn_mask_loss_graph(*x), name="mrcnn_mask_loss")( [target_mask, target_class_ids, mrcnn_mask]) # Model inputs = [ input_image, input_image_meta, input_rpn_match, input_rpn_bbox, input_gt_class_ids, input_gt_boxes, input_gt_masks ] if not config.USE_RPN_ROIS: inputs.append(input_rois) outputs = [ rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois, rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss ] model = Model(inputs, outputs, name='mask_rcnn') return model
def create_darknet53(IMG_SIZE, num_categories=4): inputs = Input(shape=(IMG_SIZE, IMG_SIZE, 3)) x = Block1(inputs, [3, 3], [1, 2], [32, 64]) x1 = Block1(x, [1, 3], [1, 1], [32, 64]) x = Add()([x, x1]) x = Block2(x, 3, 2, 128) x1 = Block1(x, [1, 3], [1, 1], [64, 128]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [64, 128]) x = Add()([x, x1]) x = Block2(x, 3, 2, 256) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [128, 256]) x = Add()([x, x1]) x = Block2(x, 3, 2, 512) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [256, 512]) x = Add()([x, x1]) x = Block2(x, 3, 2, 1024) x1 = Block1(x, [1, 3], [1, 1], [512, 1024]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [512, 1024]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [512, 1024]) x = Add()([x, x1]) x1 = Block1(x, [1, 3], [1, 1], [512, 1024]) x = Add()([x, x1]) x = AveragePooling2D()(x) x = Flatten()(x) outputs = Dense(num_categories, activation='softmax')(x) model = Model(inputs, outputs) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model
def Deep_Hedging_Model(N = None, d = None, m = None, \ risk_free = None, dt = None, initial_wealth = 0.0, epsilon = 0.0, \ final_period_cost = False, strategy_type = None, use_batch_norm = None, \ kernel_initializer = "he_uniform", \ activation_dense = "relu", activation_output = "linear", delta_constraint = None, share_stretegy_across_time = False, cost_structure = "proportional"): # State variables. prc = Input(shape=(1, ), name="prc_0") information_set = Input(shape=(1, ), name="information_set_0") inputs = [prc, information_set] for j in range(N + 1): if j < N: # Define the inputs for the strategy layers here. if strategy_type == "simple": helper1 = information_set elif strategy_type == "recurrent": if j == 0: # Tensorflow hack to deal with the dimension problem. # Strategy at t = -1 should be 0. # There is probably a better way but this works. # Constant tensor doesn't work. strategy = Lambda(lambda x: x * 0.0)(prc) helper1 = Concatenate()([information_set, strategy]) # Determine if the strategy function depends on time t or not. if not share_stretegy_across_time: strategy_layer = Strategy_Layer(d = d, m = m, use_batch_norm = use_batch_norm, \ kernel_initializer = kernel_initializer, \ activation_dense = activation_dense, \ activation_output = activation_output, delta_constraint = delta_constraint, \ day = j) else: if j == 0: # Strategy does not depend on t so there's only a single # layer at t = 0 strategy_layer = Strategy_Layer(d = d, m = m, use_batch_norm = use_batch_norm, \ kernel_initializer = kernel_initializer, \ activation_dense = activation_dense, \ activation_output = activation_output, delta_constraint = delta_constraint, \ day = j) strategyhelper = strategy_layer(helper1) # strategy_-1 is set to 0 # delta_strategy = strategy_{t+1} - strategy_t if j == 0: delta_strategy = strategyhelper else: delta_strategy = Subtract(name="diff_strategy_" + str(j))([strategyhelper, strategy]) if cost_structure == "proportional": # Proportional transaction cost absolutechanges = Lambda(lambda x: K.abs(x), name="absolutechanges_" + str(j))(delta_strategy) costs = Dot(axes=1)([absolutechanges, prc]) costs = Lambda(lambda x: epsilon * x, name="cost_" + str(j))(costs) elif cost_structure == "constant": # Tensorflow hack.. costs = Lambda(lambda x: epsilon + x * 0.0)(prc) if j == 0: wealth = Lambda(lambda x: initial_wealth - x, name="costDot_" + str(j))(costs) else: wealth = Subtract(name="costDot_" + str(j))([wealth, costs]) # Wealth for the next period # w_{t+1} = w_t + (strategy_t-strategy_{t+1})*prc_t # = w_t - delta_strategy*prc_t mult = Dot(axes=1)([delta_strategy, prc]) wealth = Subtract(name="wealth_" + str(j))([wealth, mult]) # Accumulate interest rate for next period. FV_factor = np.exp(risk_free * dt) wealth = Lambda(lambda x: x * FV_factor)(wealth) prc = Input(shape=(1, ), name="prc_" + str(j + 1)) information_set = Input(shape=(1, ), name="information_set_" + str(j + 1)) strategy = strategyhelper if j != N - 1: inputs += [prc, information_set] else: inputs += [prc] else: # The paper assumes no transaction costs for the final period # when the position is liquidated. if final_period_cost: if cost_structure == "proportional": # Proportional transaction cost absolutechanges = Lambda(lambda x: K.abs(x), name="absolutechanges_" + str(j))(strategy) costs = Dot(axes=1)([absolutechanges, prc]) costs = Lambda(lambda x: epsilon * x, name="cost_" + str(j))(costs) elif cost_structure == "constant": # Tensorflow hack.. costs = Lambda(lambda x: epsilon + x * 0.0)(prc) wealth = Subtract(name="costDot_" + str(j))([wealth, costs]) # Wealth for the final period # -delta_strategy = strategy_t mult = Dot(axes=1)([strategy, prc]) wealth = Add()([wealth, mult]) # Add the terminal payoff of any derivatives. payoff = Input(shape=(1, ), name="payoff") inputs += [payoff] wealth = Add(name="wealth_" + str(j))([wealth, payoff]) return Model(inputs=inputs, outputs=wealth)
def create_model(opt, metrics, loss, trainable_pretrained=True, input_shape=(224, 224, 3)): old_model = MobileNet(input_shape=input_shape, weights='imagenet', include_top=False) old_model.trainable = trainable_pretrained original_image = Lambda( lambda x: x, name='original_image', # trainable=True )(old_model.input) x = old_model.output y_names = [ "conv_pw_11_relu", "conv_pw_5_relu", "conv_pw_3_relu", "conv_pw_1_relu" ] f_nums = [1024, 64, 64, 64] ys = [ Conv2D(f_num, kernel_size=1, name=f'skip_hair_conv_{i}')( old_model.get_layer(name=name).output) for i, (name, f_num) in enumerate(zip(y_names, f_nums)) ] + [None] for i in range(5): y = ys[i] x = UpSampling2D(name=f'upsampling_hair_{i}')(x) if y is not None: x = Add(name=f'skip_hair_add_{i}')([x, y]) x = DepthwiseConv2D( kernel_size=3, padding='same', name=f'depth_conv2d_hair_{i}', kernel_initializer=GlorotNormal(seed=(i + 1)), )(x) x = Conv2D( 64, kernel_size=1, padding='same', name=f'conv2d_hair_{i}', kernel_regularizer=L2(2e-5), kernel_initializer=GlorotNormal(seed=11 * (i + 1)), )(x) x = ReLU(name=f'relu_hair_{i}')(x) x = Conv2D( # 1, 2, kernel_size=1, padding='same', name='conv2d_hair_final', kernel_regularizer=L2(2e-5), kernel_initializer=GlorotNormal(seed=0))(x) x = Softmax(name='sigmoid_hair_final')(x) x = Concatenate()([x, original_image]) # x = Activation('sigmoid', name='sigmoid_hair_final')(x) model = Model(old_model.input, x) if opt: model.compile( optimizer=opt, loss=loss, metrics=metrics, ) return model
def _shuffle_unit(inputs, in_channels, out_channels, groups, bottleneck_ratio, strides=2, stage=1, block=1): """ creates a shuffleunit Parameters ---------- inputs: Input tensor of with `channels_last` data format in_channels: number of input channels out_channels: number of output channels strides: An integer or tuple/list of 2 integers, specifying the strides of the convolution along the width and height. groups: int(1) number of groups per channel bottleneck_ratio: float bottleneck ratio implies the ratio of bottleneck channels to output channels. For example, bottleneck ratio = 1 : 4 means the output feature map is 4 times the width of the bottleneck feature map. stage: int(1) stage number block: int(1) block number Returns ------- """ if K.image_data_format() == 'channels_last': bn_axis = -1 else: bn_axis = 1 prefix = 'stage%d/block%d' % (stage, block) #if strides >= 2: #out_channels -= in_channels # default: 1/4 of the output channel of a ShuffleNet Unit bottleneck_channels = int(out_channels * bottleneck_ratio) groups = (1 if stage == 2 and block == 1 else groups) x = _group_conv(inputs, in_channels, out_channels=bottleneck_channels, groups=(1 if stage == 2 and block == 1 else groups), name='%s/1x1_gconv_1' % prefix) x = CustomBatchNormalization(axis=bn_axis, name='%s/bn_gconv_1' % prefix)(x) x = Activation('relu', name='%s/relu_gconv_1' % prefix)(x) x = Lambda(channel_shuffle, arguments={'groups': groups}, name='%s/channel_shuffle' % prefix)(x) x = DepthwiseConv2D(kernel_size=(3, 3), padding="same", use_bias=False, strides=strides, name='%s/1x1_dwconv_1' % prefix)(x) x = CustomBatchNormalization(axis=bn_axis, name='%s/bn_dwconv_1' % prefix)(x) x = _group_conv( x, bottleneck_channels, out_channels=out_channels if strides == 1 else out_channels - in_channels, groups=groups, name='%s/1x1_gconv_2' % prefix) x = CustomBatchNormalization(axis=bn_axis, name='%s/bn_gconv_2' % prefix)(x) if strides < 2: ret = Add(name='%s/add' % prefix)([x, inputs]) else: avg = AveragePooling2D(pool_size=3, strides=2, padding='same', name='%s/avg_pool' % prefix)(inputs) ret = Concatenate(bn_axis, name='%s/concat' % prefix)([x, avg]) ret = Activation('relu', name='%s/relu_out' % prefix)(ret) return ret
def __init__(self, input_dim=1, output_dim=1, exo_dim=0, backcast_length=10, forecast_length=1, stack_types=(TREND_BLOCK, SEASONALITY_BLOCK), nb_blocks_per_stack=3, thetas_dim=(4, 8), share_weights_in_stack=False, hidden_layer_units=256, nb_harmonics=None): self.stack_types = stack_types self.nb_blocks_per_stack = nb_blocks_per_stack self.thetas_dim = thetas_dim self.units = hidden_layer_units self.share_weights_in_stack = share_weights_in_stack self.backcast_length = backcast_length self.forecast_length = forecast_length self.input_dim = input_dim self.output_dim = output_dim self.exo_dim = exo_dim self.input_shape = (self.backcast_length, self.input_dim) self.exo_shape = (self.backcast_length, self.exo_dim) self.output_shape = (self.forecast_length, self.output_dim) self.weights = {} self.nb_harmonics = nb_harmonics assert len(self.stack_types) == len(self.thetas_dim) x = Input(shape=self.input_shape, name='input_variable') x_ = {} for k in range(self.input_dim): x_[k] = Lambda(lambda z: z[..., k])(x) e_ = {} if self.has_exog(): e = Input(shape=self.exo_shape, name='exos_variables') for k in range(self.exo_dim): e_[k] = Lambda(lambda z: z[..., k])(e) else: e = None y_ = {} for stack_id in range(len(self.stack_types)): stack_type = self.stack_types[stack_id] nb_poly = self.thetas_dim[stack_id] for block_id in range(self.nb_blocks_per_stack): backcast, forecast = self.create_block(x_, e_, stack_id, block_id, stack_type, nb_poly) for k in range(self.input_dim): x_[k] = Subtract()([x_[k], backcast[k]]) if stack_id == 0 and block_id == 0: y_[k] = forecast[k] else: y_[k] = Add()([y_[k], forecast[k]]) for k in range(self.input_dim): y_[k] = Reshape(target_shape=(self.forecast_length, 1))(y_[k]) x_[k] = Reshape(target_shape=(self.backcast_length, 1))(x_[k]) if self.input_dim > 1: y_ = Concatenate()([y_[ll] for ll in range(self.input_dim)]) x_ = Concatenate()([x_[ll] for ll in range(self.input_dim)]) else: y_ = y_[0] x_ = x_[0] if self.input_dim != self.output_dim: y_ = Dense(self.output_dim, activation='linear', name='reg_y')(y_) x_ = Dense(self.output_dim, activation='linear', name='reg_x')(x_) inputs_x = [x, e] if self.has_exog() else x n_beats_forecast = Model(inputs_x, y_, name=self._FORECAST) n_beats_backcast = Model(inputs_x, x_, name=self._BACKCAST) self.models = { model.name: model for model in [n_beats_backcast, n_beats_forecast] } self.cast_type = self._FORECAST
def _main(args): config_path = os.path.expanduser(args.config_path) weights_path = os.path.expanduser(args.weights_path) assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format( config_path) assert weights_path.endswith( '.weights'), '{} is not a .weights file'.format(weights_path) output_path = os.path.expanduser(args.output_path) assert output_path.endswith( '.h5'), 'output path {} is not a .h5 file'.format(output_path) output_root = os.path.splitext(output_path)[0] # Load weights and config. print('Loading weights.') weights_file = open(weights_path, 'rb') major, minor, revision = np.ndarray(shape=(3, ), dtype='int32', buffer=weights_file.read(12)) if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000: seen = np.ndarray(shape=(1, ), dtype='int64', buffer=weights_file.read(8)) else: seen = np.ndarray(shape=(1, ), dtype='int32', buffer=weights_file.read(4)) print('Weights Header: ', major, minor, revision, seen) print('Parsing Darknet config.') unique_config_file = unique_config_sections(config_path) cfg_parser = configparser.ConfigParser() cfg_parser.read_file(unique_config_file) print('Creating Keras model.') input_layer = Input(shape=(None, None, 3)) input_layer = Input(shape=(416, 416, 3)) prev_layer = input_layer all_layers = [] weight_decay = float(cfg_parser['net_0']['decay'] ) if 'net_0' in cfg_parser.sections() else 5e-4 count = 0 out_index = [] for section in cfg_parser.sections(): print('Parsing section {}'.format(section)) if section.startswith('convolutional'): filters = int(cfg_parser[section]['filters']) size = int(cfg_parser[section]['size']) stride = int(cfg_parser[section]['stride']) pad = int(cfg_parser[section]['pad']) activation = cfg_parser[section]['activation'] batch_normalize = 'batch_normalize' in cfg_parser[section] padding = 'same' if pad == 1 and stride == 1 else 'valid' # Setting weights. # Darknet serializes convolutional weights as: # [bias/beta, [gamma, mean, variance], conv_weights] prev_layer_shape = K.int_shape(prev_layer) weights_shape = (size, size, prev_layer_shape[-1], filters) darknet_w_shape = (filters, weights_shape[2], size, size) weights_size = np.product(weights_shape) print('conv2d', 'bn' if batch_normalize else ' ', activation, weights_shape) conv_bias = np.ndarray(shape=(filters, ), dtype='float32', buffer=weights_file.read(filters * 4)) count += filters if batch_normalize: bn_weights = np.ndarray(shape=(3, filters), dtype='float32', buffer=weights_file.read(filters * 12)) count += 3 * filters bn_weight_list = [ bn_weights[0], # scale gamma conv_bias, # shift beta bn_weights[1], # running mean bn_weights[2] # running var ] conv_weights = np.ndarray(shape=darknet_w_shape, dtype='float32', buffer=weights_file.read(weights_size * 4)) count += weights_size # DarkNet conv_weights are serialized Caffe-style: # (out_dim, in_dim, height, width) # We would like to set these to Tensorflow order: # (height, width, in_dim, out_dim) conv_weights = np.transpose(conv_weights, [2, 3, 1, 0]) conv_weights = [conv_weights] if batch_normalize else [ conv_weights, conv_bias ] # Handle activation. act_fn = None if activation == 'leaky': pass # Add advanced activation later. elif activation != 'linear': raise ValueError( 'Unknown activation function `{}` in section {}'.format( activation, section)) # Create Conv2D layer if stride > 1: # Darknet uses left and top padding instead of 'same' mode prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer) conv_layer = (Conv2D(filters, (size, size), strides=(stride, stride), kernel_regularizer=l2(weight_decay), use_bias=not batch_normalize, weights=conv_weights, activation=act_fn, padding=padding))(prev_layer) if batch_normalize: conv_layer = (BatchNormalization( weights=bn_weight_list))(conv_layer) prev_layer = conv_layer if activation == 'linear': all_layers.append(prev_layer) elif activation == 'leaky': act_layer = LeakyReLU(alpha=0.1)(prev_layer) prev_layer = act_layer all_layers.append(act_layer) elif section.startswith('route'): ids = [int(i) for i in cfg_parser[section]['layers'].split(',')] layers = [all_layers[i] for i in ids] if len(layers) > 1: print('Concatenating route layers:', layers) concatenate_layer = Concatenate()(layers) all_layers.append(concatenate_layer) prev_layer = concatenate_layer else: skip_layer = layers[0] # only one layer to route all_layers.append(skip_layer) prev_layer = skip_layer elif section.startswith('maxpool'): size = int(cfg_parser[section]['size']) stride = int(cfg_parser[section]['stride']) all_layers.append( MaxPooling2D(pool_size=(size, size), strides=(stride, stride), padding='same')(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('shortcut'): index = int(cfg_parser[section]['from']) activation = cfg_parser[section]['activation'] assert activation == 'linear', 'Only linear activation supported.' all_layers.append(Add()([all_layers[index], prev_layer])) prev_layer = all_layers[-1] elif section.startswith('upsample'): stride = int(cfg_parser[section]['stride']) assert stride == 2, 'Only stride=2 supported.' all_layers.append(UpSampling2D(stride)(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('yolo'): out_index.append(len(all_layers) - 1) all_layers.append(None) prev_layer = all_layers[-1] elif section.startswith('net'): pass else: raise ValueError( 'Unsupported section header type: {}'.format(section)) # Create and save model. if len(out_index) == 0: out_index.append(len(all_layers) - 1) model = Model(inputs=input_layer, outputs=[all_layers[i] for i in out_index]) print(model.summary()) if args.weights_only: model.save_weights('{}'.format(output_path)) print('Saved Keras weights to {}'.format(output_path)) else: model.save('{}'.format(output_path)) print('Saved Keras model to {}'.format(output_path)) # Check to see if all weights have been read. remaining_weights = len(weights_file.read()) / 4 weights_file.close() print('Read {} of {} from Darknet weights.'.format( count, count + remaining_weights)) if remaining_weights > 0: print('Warning: {} unused weights'.format(remaining_weights))
def f(net): # format of conv_params: # [ [nb_col="kernel width", nb_row="kernel height", # subsample="(stride_vertical,stride_horizontal)", # border_mode="same" or "valid"] ] # B(3,3): orignal <<basic>> block if direction == 'up': conv_params = [[3, 3, (1, 1), "same"], [3, 3, (1, 1), "same"]] else: conv_params = [[3, 3, stride, "same"], [3, 3, (1, 1), "same"]] n_bottleneck_plane = n_output_plane # Residual block for i, v in enumerate(conv_params): if i == 0: if n_input_plane != n_output_plane: net = BatchNormalization(axis=CHANNEL_AXIS)(net) net = Activation("relu")(net) convs = net else: convs = BatchNormalization(axis=CHANNEL_AXIS)(net) convs = Activation("relu")(convs) convs = Conv2D(n_bottleneck_plane, (v[0], v[1]), strides=v[2], padding=v[3], kernel_initializer=WEIGHT_INIT, kernel_regularizer=l2(WEIGHT_DECAY), use_bias=USE_BIAS)(convs) if direction == 'up': convs = UpSampling2D(stride)(convs) else: convs = BatchNormalization(axis=CHANNEL_AXIS)(convs) convs = Activation("relu")(convs) if dropout_probability > 0: convs = Dropout(dropout_probability)(convs) convs = Conv2D(n_bottleneck_plane, (v[0], v[1]), strides=v[2], padding=v[3], kernel_initializer=WEIGHT_INIT, kernel_regularizer=l2(WEIGHT_DECAY), use_bias=USE_BIAS)(convs) # Shortcut Conntection: identity function or 1x1 convolutional # (depends on difference between input & output shape - this # corresponds to whether we are using the first block in each # group; see _layer() ). if n_input_plane != n_output_plane: shortcut_stride = 1 if direction == 'up' else stride shortcut = Conv2D(n_output_plane, (1, 1), strides=shortcut_stride, padding="same", kernel_initializer=WEIGHT_INIT, kernel_regularizer=l2(WEIGHT_DECAY), use_bias=USE_BIAS)(net) if direction == 'up': shortcut = UpSampling2D(stride)(shortcut) else: if stride == 1: shortcut = net elif direction == 'up': shortcut = UpSampling2D(stride)(net) else: shortcut = AveragePooling2D(stride)(net) return Add()([convs, shortcut])
def darknet_residual(x, filters): prev = x x = darknet_conv(x, filters // 2, 1) x = darknet_conv(x, filters, 3) x = Add()([prev, x]) return x
def CNNResBlockModel(config): def regularization(lamda): if config.regularization_method == 'L2': return keras.regularizers.l2(lamda) elif config.regularization_method == 'L1': return keras.regularizers.l1(lamda) else: raise Exception('Use Only L2 / L1 regularization') def activation(activation_name, x): if activation_name == 'leaky_relu': return LeakyReLU(alpha=config.alpha)(x) else: return Activation(activation_name)(x) def highway_layer(value, gate_bias=-3): # https://towardsdatascience.com/review-highway-networks-gating-function-to-highway-image-classification-5a33833797b5 nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel() dim = K.int_shape(value)[-1] # gate_bias_initializer = tensorflow.keras.initializers.Constant(gate_bias) # gate = Dense(units=dim, bias_initializer=gate_bias_initializer)(value) # gate = Activation("sigmoid")(gate) # TODO (just for yellow color...) NOTE: to keep dimensions matched, convolution gate instead of regular sigmoid # gate (T in paper) gate = Conv2D(size_list[i_hidden + config.CNN_ResBlock_conv_per_block - 1], kernel_size=filt_list[-1], padding='same', activation='sigmoid', bias_initializer=tensorflow.keras.initializers.Constant( gate_bias))(value) # negated (C in paper) negated_gate = Lambda(lambda x: 1.0 - x, output_shape=(size_list[-1], ))(gate) # use ResBlock as the Transformation transformed = ResBlock(x=value) transformed_gated = Multiply()([gate, transformed]) # UpSample value if needed if value.shape.as_list()[-1] != negated_gate.shape.as_list()[-1]: r = negated_gate.shape.as_list()[-1] / value.shape.as_list()[-1] assert not (bool(r % 1)) value = tf.keras.layers.UpSampling3D(size=(1, 1, int(r)))(value) identity_gated = Multiply()([negated_gate, value]) value = Add()([transformed_gated, identity_gated]) return value def skip_connection_layer(value): nonlocal i_hidden # use ResBlock as the Transformation transformed = ResBlock(x=value) if value.shape.as_list()[-1] != transformed.shape.as_list()[-1]: r = transformed.shape.as_list()[-1] / value.shape.as_list()[-1] assert not (bool(r % 1)) # apply convolution as transformation value = Conv2D(size_list[i_hidden - 1], kernel_size=filt_list[i_hidden - 1], padding='same')(value) value = Add()([value, transformed]) return value def ResBlock(x): for i in range(config.CNN_ResBlock_conv_per_block): nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel() lamda_cnn = 0.0 if config.use_l2_in_cnn is False else lamda x = Conv2D(size_list[i_hidden], kernel_size=filt_list[i_hidden], padding='same', bias_regularizer=regularization(lamda_cnn), kernel_regularizer=regularization(lamda_cnn), kernel_initializer=kernel_initalizer)(x) x = activation(activation_name, x) if config.use_batch_norm is True: x = BatchNormalization()(x) i_hidden = i_hidden + 1 return x def ResBlockLane(x): nonlocal i_hidden # ResBlocks for i in range(len(config.CNN_ResBlock_highway)): if config.CNN_ResBlock_highway[i] == "Highway": x = highway_layer(value=x) elif config.CNN_ResBlock_highway[i] == "Skip": x = skip_connection_layer(value=x) elif config.CNN_ResBlock_highway[i] == "None": x = ResBlock(x=x) else: raise Exception('only Highway/Skip/None is allowed !') # MaxPool and Dropout if config.CNN_ResBlock_dropout[i] != 0: x = Dropout(rate=config.CNN_ResBlock_dropout[i])(x) x = MaxPooling2D(pool_size=pool_list[i])(x) return x global background_implicit_inference # parameters kernel_initalizer = config.kernel_initalizer # default is 'glorot_uniform' lamda = config.Regularization_term p_dropout = config.dropout activation_name = config.activation filt_dim2_list = config.Filter_shape_dim1 if config.Filter_shape_symmetric else config.Filter_shape_dim2 filt_list = [(x, y) for x, y in zip(config.Filter_shape_dim1, filt_dim2_list)] pool_list = [ (x, y) for x, y in zip(config.Pool_shape_dim1, config.Pool_shape_dim2) ] size_list = config.hidden_size dense_list = config.Dense_size input_shape = config.model_input_dim p_dropout_conv1d = config.CNN_ResBlock_dropout_conv1d p_dropout_after_all_conv2d = config.dropout_after_all_conv2d p_dropout_dense = config.Dense_dropout i_hidden = 0 # Input Layer input_layer = Input(shape=input_shape) assert len(size_list) == len(filt_list) assert len(pool_list) == len(config.CNN_ResBlock_highway) == len( config.CNN_ResBlock_dropout) assert config.CNN_ResBlock_conv_per_block * len( config.CNN_ResBlock_highway) == len(size_list) assert len(config.Conv1D_size) == len(config.Conv1D_kernel) if config.ResBlockDouble: x1 = input_layer x1 = ResBlockLane(x1) i_hidden = 0 # zero the hidden sizes counter x2 = input_layer x2 = ResBlockLane(x2) x = Add()([x1, x2]) else: x = input_layer # ResBlocks x = ResBlockLane(x) # Flatten x = Flatten()(x) if p_dropout_after_all_conv2d != 0: x = Dropout(rate=p_dropout_after_all_conv2d)(x) # Conv1D if len(config.Conv1D_size) != 0: x = tf.expand_dims(x, axis=-1) for i in range(len(config.Conv1D_size)): x = Conv1D(filters=config.Conv1D_size[i], kernel_size=config.Conv1D_kernel[i], kernel_initializer=kernel_initalizer)(x) x = activation(activation_name, x) if config.use_batch_norm is True: x = BatchNormalization()(x) if p_dropout_conv1d[i] != 0.0: x = Dropout(rate=p_dropout_conv1d[1])(x) # post-Conv1D if len(config.Conv1D_size) != 0: x = MaxPooling1D(pool_size=config.Conv1D_pool)(x) # x = BatchNormalization()(x) x = Flatten()(x) # Dense for i in range(len(dense_list)): x = Dense(dense_list[i], kernel_regularizer=regularization(lamda), kernel_initializer=kernel_initalizer)(x) x = activation(activation_name, x) if config.use_batch_norm is True: x = BatchNormalization()(x) if p_dropout_dense[i] != 0.0: x = Dropout(rate=p_dropout_dense[i])(x) # x = Dropout(rate=p_dropout)(x) # x = BatchNormalization()(x) if config.learn_background: x = Dense(3, activation='softmax')(x) else: x = Dense(1, activation='sigmoid')(x) output_layer = x model = Model(input_layer, output_layer) if config.learn_background: if config.background_implicit_inference: background_implicit_inference = True model = BlockBackgroundModel(input_layer, output_layer) # else: # model = Model(input_layer, output_layer) # model.summary() return model
def _pep_block(inputs, proj_filters, filters, stride, expansion, block_id): #in_channels = backend.int_shape(inputs)[-1] in_channels = inputs.shape.as_list()[-1] pointwise_conv_filters = int(filters) x = inputs prefix = 'pep_block_{}_'.format(block_id) # Pre-project x = Conv2D(proj_filters, kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'preproject')(x) x = CustomBatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'preproject_BN')(x) x = ReLU(6., name=prefix + 'preproject_relu')(x) # Expand #x = Conv2D(int(expansion * in_channels), kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'expand')(x) x = Conv2D(int(expansion * proj_filters), kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'expand')(x) x = CustomBatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'expand_BN')(x) x = ReLU(6., name=prefix + 'expand_relu')(x) # Depthwise if stride == 2: x = ZeroPadding2D(padding=correct_pad(K, x, 3), name=prefix + 'pad')(x) x = DepthwiseConv2D(kernel_size=3, strides=stride, activation=None, use_bias=False, padding='same' if stride == 1 else 'valid', name=prefix + 'depthwise')(x) x = CustomBatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x) x = ReLU(6., name=prefix + 'depthwise_relu')(x) # Project x = Conv2D(pointwise_conv_filters, kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'project')(x) x = CustomBatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'project_BN')(x) if in_channels == pointwise_conv_filters and stride == 1: return Add(name=prefix + 'add')([inputs, x]) return x
def create_new_det(input_shape=(800, 700, 35), kernel_regularizer=None, downsample_factor=4, reg_channels=8): ''' This architecture key differences: # Params: 3,969,031 ''' K.clear_session() KERNEL_REG = kernel_regularizer KERNEL_SIZE = { "Layer1": 7, "Block1": 3, "Block2": 3, "Block3": 3, "Block4": 3, "Header": 3, "Out": 1 } PADDING = 'same' FILTERS = 256 CLASS_CHANNELS = 1 OUT_KERNEL_SIZE = 1 REGRESS_CHANNELS = reg_channels BiFPN_filters = 128 inp = Input(shape=input_shape) x = inp with tf.name_scope("Backbone"): with tf.name_scope("Block1"): x = Conv2D(filters=FILTERS // 8, kernel_size=KERNEL_SIZE['Layer1'], padding=PADDING, kernel_regularizer=KERNEL_REG, use_bias=False)(x) x = BatchNormalization()(x) x = ReLU()(x) x = conv_block(x, filters=FILTERS // 8, kernel_size=KERNEL_SIZE['Block1'], num_layers=1) block1_out = x # Downsize twice block1_out = AveragePooling2D()(block1_out) block1_out = create_sep_conv_block(block1_out, FILTERS // 8, KERNEL_SIZE['Block1'], num_layers=1) block1_out = AveragePooling2D()(block1_out) block1_out = create_sep_conv_block(block1_out, FILTERS // 8, KERNEL_SIZE['Block1'], num_layers=1) # change number of channels from X -> BiFPN channels block1_out = Conv2D(filters=BiFPN_filters, kernel_size=1, padding=PADDING, use_bias=False)(block1_out) block1_out = BatchNormalization()(block1_out) block1_out = ReLU()(block1_out) with tf.name_scope("Block2"): #--------Downsampling--------# x = MaxPooling2D()(x) #--------Downsampling--------# x = Conv2D(filters=FILTERS // 4, kernel_size=KERNEL_SIZE['Block2'], padding=PADDING, use_bias=False)(x) x = BatchNormalization()(x) x = ReLU()(x) x = conv_block(x, filters=FILTERS // 4, kernel_size=KERNEL_SIZE['Block2'], num_layers=1) block2_out = x # Downsize once block2_out = AveragePooling2D()(block2_out) block2_out = create_sep_conv_block(block2_out, FILTERS // 4, KERNEL_SIZE['Block1'], num_layers=1) # change number of channels from X -> BiFPN channels block2_out = Conv2D(filters=BiFPN_filters, kernel_size=1, padding=PADDING, use_bias=False)(block2_out) block2_out = BatchNormalization()(block2_out) block2_out = ReLU()(block2_out) with tf.name_scope("Block3"): #--------Downsampling--------# x = MaxPooling2D()(x) #--------Downsampling--------# x = Conv2D(filters=FILTERS // 2, kernel_size=KERNEL_SIZE['Block3'], padding=PADDING, use_bias=False)(x) x = BatchNormalization()(x) x = ReLU()(x) x = conv_block(x, filters=FILTERS // 2, kernel_size=KERNEL_SIZE['Block3'], num_layers=2) block3_out = x # change number of channels from X -> BiFPN channels block3_out = Conv2D(filters=BiFPN_filters, kernel_size=1, padding=PADDING, use_bias=False)(block3_out) block3_out = BatchNormalization()(block3_out) block3_out = ReLU()(block3_out) with tf.name_scope("Block4"): #--------Downsampling--------# x = MaxPooling2D()(x) #--------Downsampling--------# x = Conv2D(filters=FILTERS // 1, kernel_size=KERNEL_SIZE['Block4'], padding=PADDING, use_bias=False)(x) x = BatchNormalization()(x) x = ReLU()(x) x = conv_block(x, filters=FILTERS // 1, kernel_size=KERNEL_SIZE['Block4'], num_layers=5) block4_out = x # upsmaple once block4_out = tf.image.resize(block4_out, size=get_new_shape(block4_out, 2), method=RESIZE_METHOD) block4_out = SeparableConv2D(filters=FILTERS // 1, kernel_size=KERNEL_SIZE['Block4'], padding=PADDING)(block4_out) # change number of channels from X -> BiFPN channels block4_out = Conv2D(filters=BiFPN_filters, kernel_size=1, padding=PADDING, use_bias=False)(block4_out) block4_out = BatchNormalization()(block4_out) block4_out = ReLU()(block4_out) out1, out2, out3, out4 = build_BiFPN_v2(block1_out, block2_out, block3_out, block4_out, filters=BiFPN_filters) out5, out6, out7, out8 = build_BiFPN_v2(block1_out, block2_out, block3_out, block4_out, filters=BiFPN_filters) with tf.name_scope("FinalOutput"): out1 = Add()([out1, out2, out3, out4]) out1 = create_conv_block(out1, filters=BiFPN_filters, kernel_size=3, num_layers=1) out2 = Add()([out5, out6, out7, out8]) out2 = create_conv_block(out2, filters=BiFPN_filters, kernel_size=3, num_layers=1) concat = Add()([out1, out2]) concat = create_conv_block(concat, filters=BiFPN_filters, kernel_size=3, num_layers=1) branch_1 = create_conv_block(concat, filters=BiFPN_filters, kernel_size=3, num_layers=2) branch_2 = create_conv_block(concat, filters=BiFPN_filters, kernel_size=3, num_layers=2) obj = Conv2D(CLASS_CHANNELS, kernel_size=OUT_KERNEL_SIZE, padding=PADDING, activation='sigmoid', name='objectness_map')(branch_1) geo = Conv2D(REGRESS_CHANNELS, kernel_size=OUT_KERNEL_SIZE, padding=PADDING, name='geometric_map')(branch_2) out = Concatenate(axis=-1, name='output_map')([obj, geo]) model = Model(inp, out) return model
def DarknetResidual(x, filters): prev = x x = DarknetConv(x, filters // 2, 1) x = DarknetConv(x, filters, 3) x = Add()([prev, x]) return x
def get_predict_model(config): h, w = config.IMAGE_SHAPE[:2] if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6): raise Exception("Image size must be dividable by 2 at least 6 times " "to avoid fractions when downscaling and upscaling." "For example, use 256, 320, 384, 448, 512, ... etc. ") # 输入进来的图片必须是2的6次方以上的倍数 input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]], name="input_image") # meta包含了一些必要信息 input_image_meta = Input(shape=[config.IMAGE_META_SIZE], name="input_image_meta") # 输入进来的先验框 input_anchors = Input(shape=[None, 4], name="input_anchors") # 获得Resnet里的压缩程度不同的一些层 _, C2, C3, C4, C5 = get_resnet(input_image, stage5=True, train_bn=config.TRAIN_BN) # 组合成特征金字塔的结构 # P5长宽共压缩了5次 # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5) # P4长宽共压缩了4次 # Height/16,Width/16,256 P4 = Add(name="fpn_p4add")([ UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4) ]) # P4长宽共压缩了3次 # Height/8,Width/8,256 P3 = Add(name="fpn_p3add")([ UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3) ]) # P4长宽共压缩了2次 # Height/4,Width/4,256 P2 = Add(name="fpn_p2add")([ UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2) ]) # 各自进行一次256通道的卷积,此时P2、P3、P4、P5通道数相同 # Height/4,Width/4,256 P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p2")(P2) # Height/8,Width/8,256 P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p3")(P3) # Height/16,Width/16,256 P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p4")(P4) # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p5")(P5) # 在建议框网络里面还有一个P6用于获取建议框 # Height/64,Width/64,256 P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5) # P2, P3, P4, P5, P6可以用于获取建议框 rpn_feature_maps = [P2, P3, P4, P5, P6] # P2, P3, P4, P5用于获取mask信息 mrcnn_feature_maps = [P2, P3, P4, P5] anchors = input_anchors # 建立RPN模型 rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) rpn_class_logits, rpn_class, rpn_bbox = [], [], [] # 获得RPN网络的预测结果,进行格式调整,把五个特征层的结果进行堆叠 for p in rpn_feature_maps: logits, classes, bbox = rpn([p]) rpn_class_logits.append(logits) rpn_class.append(classes) rpn_bbox.append(bbox) rpn_class_logits = Concatenate(axis=1, name="rpn_class_logits")(rpn_class_logits) rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class) rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox) # 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是 # rpn_class_logits : Batch_size, num_anchors, 2 # rpn_class : Batch_size, num_anchors, 2 # rpn_bbox : Batch_size, num_anchors, 4 proposal_count = config.POST_NMS_ROIS_INFERENCE # Batch_size, proposal_count, 4 # 对先验框进行解码 rpn_rois = ProposalLayer(proposal_count=proposal_count, nms_threshold=config.RPN_NMS_THRESHOLD, name="ROI", config=config)([rpn_class, rpn_bbox, anchors]) # 获得classifier的结果 mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\ fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta, config.POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN, fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) detections = DetectionLayer(config, name="mrcnn_detection")( [rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta]) detection_boxes = Lambda(lambda x: x[..., :4])(detections) # 获得mask的结果 mrcnn_mask = build_fpn_mask_graph(detection_boxes, mrcnn_feature_maps, input_image_meta, config.MASK_POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN) # 作为输出 model = Model([input_image, input_image_meta, input_anchors], [ detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class, rpn_bbox ], name='mask_rcnn') return model
def _convert(self, arc_seq, verbose=True): out_filters, pool_layers = self.get_out_filters(self.model_space) inp = get_layer(x=None, state=self.inputs) # this is assuming all choices have the same out_filters stem_conv = Operation('conv1d', kernel_size=8, filters=out_filters[0], activation="linear") x = self.res_layer(stem_conv, self.wsf, inp, name="stem_conv", add_conv1_under_pool=self.add_conv1_under_pool) start_idx = 0 layers = [] for layer_id in range(len(self.model_space)): if verbose: print("start_idx=%i, layer id=%i, out_filters=%i x %i" % (start_idx, layer_id, out_filters[layer_id], self.wsf)) count = arc_seq[start_idx] this_layer = self.model_space[layer_id][count] if verbose: print(this_layer) if layer_id == 0: x = self.res_layer( this_layer, self.wsf, x, name="L%i" % layer_id, add_conv1_under_pool=self.add_conv1_under_pool) else: x = self.res_layer( this_layer, self.wsf, layers[-1], name="L%i" % layer_id, add_conv1_under_pool=self.add_conv1_under_pool) if layer_id > 0: skip = arc_seq[start_idx + 1:start_idx + layer_id + 1] skip_layers = [ layers[i] for i in range(len(layers)) if skip[i] == 1 ] if verbose: print("skip=%s" % skip) if len(skip_layers): skip_layers.append(x) x = Add(name="L%i_resAdd" % layer_id)(skip_layers) x = BatchNormalization(name="L%i_resBn" % layer_id)(x) if self.dropout_rate != 0: x = Dropout(self.dropout_rate, name="L%i_dropout" % layer_id)(x) layers.append(x) if layer_id in pool_layers: pooled_layers = [] for i, layer in enumerate(layers): pooled_layers.append( self.factorized_reduction_layer( layer, out_filters[layer_id + 1] * self.wsf, name="pool_at_%i_from_%i" % (layer_id, i))) if verbose: print("pooled@%i, %s" % (layer_id, pooled_layers)) layers = pooled_layers start_idx += 1 + layer_id if verbose: print('-' * 80) # fully-connected layer if self.flatten_mode == 'GAP': x = GlobalAveragePooling1D()(x) elif self.flatten_mode == 'Flatten': x = Flatten()(x) else: raise Exception("Unknown flatten mode: %s" % self.flatten_mode) if self.dropout_rate != 0: x = Dropout(self.dropout_rate)(x) x = Dense(units=self.fc_units, activation="relu")(x) out = get_layer(x=x, state=self.outputs) model = Model(inputs=inp, outputs=out) return model
def inverted_block(self, x, strides=(1, 1), **metaparameters): """ Construct an Inverted Residual Block x : input to the block strides : strides n_filters : number of filters alpha : width multiplier expansion : multiplier for expanding number of filters """ n_filters = metaparameters['n_filters'] alpha = metaparameters['alpha'] if 'alpha' in metaparameters: alpha = metaparameters['alpha'] else: alpha = self.alpha if 'expansion' in metaparameters: expansion = metaparameters['expansion'] else: expansion = self.expansion del metaparameters['n_filters'] # Remember input shortcut = x # Apply the width filter to the number of feature maps for the pointwise convolution filters = int(n_filters * alpha) n_channels = int(x.shape[3]) # Dimensionality Expansion (non-first block) if expansion > 1: # 1x1 linear convolution x = self.Conv2D(x, expansion * n_channels, (1, 1), padding='same', use_bias=False, **metaparameters) x = self.BatchNormalization(x) x = self.ReLU(x) # Strided convolution to match number of filters if strides == (2, 2): x = ZeroPadding2D(padding=((0, 1), (0, 1)))(x) padding = 'valid' else: padding = 'same' # Depthwise Convolution x = self.DepthwiseConv2D(x, (3, 3), strides, padding=padding, use_bias=False, **metaparameters) x = self.BatchNormalization(x) x = self.ReLU(x) # Linear Pointwise Convolution x = self.Conv2D(x, filters, (1, 1), strides=(1, 1), padding='same', use_bias=False, **metaparameters) x = self.BatchNormalization(x) # Number of input filters matches the number of output filters if n_channels == filters and strides == (1, 1): x = Add()([shortcut, x]) return x