def add_resnet_unit(path, **params): block_input = path # add Conv2D path = L.Convolution2D( filters=params["filters_per_layer"], kernel_size=params["filter_width"], activation='linear', padding='same', kernel_regularizer=R.l2(.0001), bias_regularizer=R.l2(.0001))(path) print path path = L.BatchNormalization( beta_regularizer=R.l2(.0001), gamma_regularizer=R.l2(.0001))(path) print path path = L.Activation('relu')(path) print path path = L.Convolution2D( filters=params["filters_per_layer"], kernel_size=params["filter_width"], activation='linear', padding='same', kernel_regularizer=R.l2(.0001), bias_regularizer=R.l2(.0001))(path) print path path = L.BatchNormalization( beta_regularizer=R.l2(.0001), gamma_regularizer=R.l2(.0001))(path) print path path = L.Add()([block_input, path]) print path path = L.Activation('relu')(path) print path return path
def build_model(self): l2_regularization_kernel = 1e-5 # Input Layer input = layers.Input(shape=(self.state_size,), name='input_states') # Hidden Layers model = layers.Dense(units=300, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(input) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) model = layers.Dense(units=400, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) model = layers.Dense(units=200, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) # Our output layer - a fully connected layer output = layers.Dense(units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(l2_regularization_kernel), kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), name='output_actions')(model) # Keras model self.model = models.Model(inputs=input, outputs=output) # Define loss and optimizer action_gradients = layers.Input(shape=(self.action_size,)) loss = K.mean(-action_gradients * output) optimizer = optimizers.Adam(lr=1e-4) update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function(inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=update_operation)
def build_model(self): l2_kernel_regularization = 1e-5 # Define input layers input_states = layers.Input(shape=(self.state_size, ), name='input_states') input_actions = layers.Input(shape=(self.action_size, ), name='input_actions') # Hidden layers for states model_states = layers.Dense( units=32, kernel_regularizer=regularizers.l2(l2_kernel_regularization))( input_states) model_states = layers.BatchNormalization()(model_states) model_states = layers.LeakyReLU(1e-2)(model_states) model_states = layers.Dense( units=64, kernel_regularizer=regularizers.l2(l2_kernel_regularization))( model_states) model_states = layers.BatchNormalization()(model_states) model_states = layers.LeakyReLU(1e-2)(model_states) # Hidden layers for actions model_actions = layers.Dense( units=64, kernel_regularizer=regularizers.l2(l2_kernel_regularization))( input_actions) model_actions = layers.BatchNormalization()(model_actions) model_actions = layers.LeakyReLU(1e-2)(model_actions) # Both models merge here model = layers.add([model_states, model_actions]) # Fully connected and batch normalization model = layers.Dense(units=32, kernel_regularizer=regularizers.l2( l2_kernel_regularization))(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) # Q values / output layer Q_values = layers.Dense( units=1, activation=None, kernel_regularizer=regularizers.l2(l2_kernel_regularization), kernel_initializer=initializers.RandomUniform(minval=-5e-3, maxval=5e-3), name='output_Q_values')(model) # Keras wrap the model self.model = models.Model(inputs=[input_states, input_actions], outputs=Q_values) optimizer = optimizers.Adam(lr=1e-2) self.model.compile(optimizer=optimizer, loss='mse') action_gradients = K.gradients(Q_values, input_actions) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
def double_conv_layer(x, filter_size, size, dropout, batch_norm=False): ''' construction of a double convolutional layer using SAME padding RELU nonlinear activation function :param x: input :param filter_size: size of convolutional filter :param size: number of filters :param dropout: FLAG & RATE of dropout. if < 0 dropout cancelled, if > 0 set as the rate :param batch_norm: flag of if batch_norm used, if True batch normalization :return: output of a double convolutional layer ''' axis = 3 conv = layers.Conv2D(size, (filter_size, filter_size), padding='same')(x) if batch_norm is True: conv = layers.BatchNormalization(axis=axis)(conv) conv = layers.Activation('relu')(conv) conv = layers.Conv2D(size, (filter_size, filter_size), padding='same')(conv) if batch_norm is True: conv = layers.BatchNormalization(axis=axis)(conv) conv = layers.Activation('relu')(conv) if dropout > 0: conv = layers.Dropout(dropout)(conv) shortcut = layers.Conv2D(size, kernel_size=(1, 1), padding='same')(x) if batch_norm is True: shortcut = layers.BatchNormalization(axis=axis)(shortcut) res_path = layers.add([shortcut, conv]) return res_path
def conv_block(input_tensor, num_filters): encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(input_tensor) encoder = layers.BatchNormalization()(encoder) encoder = layers.Activation('relu')(encoder) encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(encoder) encoder = layers.BatchNormalization()(encoder) encoder = layers.Activation('relu')(encoder) return encoder
def build_model(self): """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size, ), name='states') '''# Add hidden layers net = layers.Dense(units=32, activation='relu')(states) net = layers.Dense(units=64, activation='relu')(net) net = layers.Dense(units=32, activation='relu')(net) # Try different layer sizes, activations, add batch normalization, regularizers, etc. # Add final output layer with sigmoid activation raw_actions = layers.Dense(units=self.action_size, activation='sigmoid', name='raw_actions')(net) ''' ################################### # Add hidden layers net = layers.Dense(units=400, kernel_regularizer=regularizers.l2(1e-6))(states) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) net = layers.Dense(units=300, kernel_regularizer=regularizers.l2(1e-6))(net) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) # Add final output layer with sigmoid activation raw_actions = layers.Dense( units=self.action_size, activation='sigmoid', name='raw_actions', kernel_initializer=initializers.RandomUniform(minval=-0.003, maxval=0.003))(net) ####################################### # Scale [0, 1] output for each action dimension to proper range actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, name='actions')(raw_actions) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) # Define loss function using action value (Q value) gradients action_gradients = layers.Input(shape=(self.action_size, )) loss = K.mean(-action_gradients * actions) # Incorporate any additional losses here (e.g. from regularizers) # Define optimizer and training function optimizer = optimizers.Adam(lr=1e-6) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=updates_op)
def decoder_block(input_tensor, concat_tensor, num_filters): decoder = layers.Conv2DTranspose(num_filters, (2, 2), strides=(2, 2), padding='same')(input_tensor) decoder = layers.concatenate([concat_tensor, decoder], axis=-1) decoder = layers.BatchNormalization()(decoder) decoder = layers.Activation('relu')(decoder) decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder) decoder = layers.BatchNormalization()(decoder) decoder = layers.Activation('relu')(decoder) decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder) decoder = layers.BatchNormalization()(decoder) decoder = layers.Activation('relu')(decoder) return decoder
def residual_block(y, nb_channels_in, nb_channels_out, _strides=(1, 1), _project_shortcut=False): """ Our network consists of a stack of residual blocks. These blocks have the same topology, and are subject to two simple rules: - If producing spatial maps of the same size, the blocks share the same hyper-parameters (width and filter sizes). - Each time the spatial map is down-sampled by a factor of 2, the width of the blocks is multiplied by a factor of 2. """ shortcut = y # we modify the residual building block as a bottleneck design to make the network more economical y = layers.Conv2D(nb_channels_in, kernel_size=(1, 1), strides=(1, 1), padding='same')(y) y = add_common_layers(y) # ResNeXt (identical to ResNet when `cardinality` == 1) y = grouped_convolution(y, nb_channels_in, _strides=_strides) y = add_common_layers(y) y = layers.Conv2D(nb_channels_out, kernel_size=(1, 1), strides=(1, 1), padding='same')(y) # batch normalization is employed after aggregating the transformations and before adding to the shortcut y = layers.BatchNormalization()(y) # identity shortcuts used directly when the input and output are of the same dimensions if _project_shortcut or _strides != (1, 1): # when the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions) # when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2 shortcut = layers.Conv2D(nb_channels_out, kernel_size=(1, 1), strides=_strides, padding='same')(shortcut) shortcut = layers.BatchNormalization()(shortcut) y = layers.add([shortcut, y]) # relu is performed right after each batch normalization, # expect for the output of the block where relu is performed after the adding to the shortcut y = layers.LeakyReLU()(y) return y
def create_model(self, img_shape, num_class): self.handle_dim_ordering() K.set_learning_phase(True) #model = models.Sequential() inputs = layers.Input(shape = img_shape) x = self.conv_bn_relu(inputs, (3, 3),(1, 1), 32, 'conv0_1') net = self.conv_bn_relu(x, (3, 3), (1, 1), 64, 'conv0_2') bn1 = layers.BatchNormalization(momentum=0.99, name='conv0_3_bn')(self.conv( net, (3, 3), (1, 1), 32, 'conv0_3')) act1 = layers.Activation('relu')(bn1) bn2, act2 = self.down_block(act1, 32, 'down1') bn3, act3 = self.down_block(act2, 32, 'down2') bn4, act4 = self.down_block(act3, 32, 'down3') bn5, act5 = self.down_block(act4, 32, 'down4') bn6, act6 = self.down_block(act5, 32, 'down5') bn7, act7 = self.down_block(act6, 32, 'down6') temp = self.up_block(act7, bn6, 32, 'up6') temp = self.up_block(temp, bn5, 32, 'up5') temp = self.up_block(temp, bn4, 32, 'up4') temp = self.up_block(temp, bn3, 32, 'up3') temp = self.up_block(temp, bn2, 32, 'up2') temp = self.up_block(temp, bn1, 32, 'up1') output = self.conv(temp, (1, 1), (1, 1), num_class, 'output') model = models.Model(outputs=output, inputs=inputs) print(model.summary()) return model
def attention_block(x, gating, inter_shape): shape_x = K.int_shape(x) shape_g = K.int_shape(gating) theta_x = layers.Conv2D(inter_shape, (2, 2), strides=(2, 2), padding='same')(x) # 16 shape_theta_x = K.int_shape(theta_x) phi_g = layers.Conv2D(inter_shape, (1, 1), padding='same')(gating) upsample_g = layers.Conv2DTranspose( inter_shape, (3, 3), strides=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]), padding='same')(phi_g) # 16 concat_xg = layers.add([upsample_g, theta_x]) act_xg = layers.Activation('relu')(concat_xg) psi = layers.Conv2D(1, (1, 1), padding='same')(act_xg) sigmoid_xg = layers.Activation('sigmoid')(psi) shape_sigmoid = K.int_shape(sigmoid_xg) upsample_psi = layers.UpSampling2D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2]))( sigmoid_xg) # 32 upsample_psi = expend_as(upsample_psi, shape_x[3]) y = layers.multiply([upsample_psi, x]) result = layers.Conv2D(shape_x[3], (1, 1), padding='same')(y) result_bn = layers.BatchNormalization()(result) return result_bn
def get_network(name, input_dim, output_dim, hidden_dim, num_layers, PKLparams=None, batchnorm=False, is_shooter=False, row_sparse=False, add_pklayers=False, filt_size=None): """Return a NN with the specified parameters and a list of PKBias layers. """ with tf.variable_scope(name): M = models.Sequential(name=name) PKbias_layers = [] M.add(layers.InputLayer(input_shape=(None, input_dim), name="Input")) if batchnorm: M.add(layers.BatchNormalization(name="BatchNorm")) if filt_size is not None: M.add( layers.ZeroPadding1D(padding=(filt_size - 1, 0), name="ZeroPadding")) M.add( layers.Conv1D(filters=hidden_dim, kernel_size=filt_size, padding="valid", activation=tf.nn.relu, name="Conv1D")) for i in range(num_layers): with tf.variable_scope("PK_Bias"): if is_shooter and add_pklayers: if row_sparse: PK_bias = PKRowBiasLayer(M, PKLparams, name="PKRowBias_%s" % (i + 1)) else: PK_bias = PKBiasLayer(M, PKLparams, name="PKBias_%s" % (i + 1)) PKbias_layers.append(PK_bias) M.add(PK_bias) if i == num_layers - 1: M.add( layers.Dense(output_dim, activation="linear", name="Dense_%s" % (i + 1))) else: M.add( layers.Dense(hidden_dim, activation="relu", name="Dense_%s" % (i + 1))) return M, PKbias_layers
def conv2_bn(x, filts, k=3, stride=1, rate=1, name=None, pad='same'): x = l.Conv2D(filts, (k, k), strides=(stride, stride), dilation_rate=rate, padding=pad, name=name)(x) x = l.BatchNormalization(name=name + '_bn')(x) x = l.Activation('relu', name=name + '_relu')(x) return x
def res_block(inputs, size): kernel_l2_reg = 1e-3 net = layers.Dense(size, activation=None, kernel_regularizer=regularizers.l2(kernel_l2_reg), kernel_initializer=initializers.RandomUniform( minval=-5e-3, maxval=5e-3))(inputs) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) net = layers.Dense(size, activation=None, kernel_regularizer=regularizers.l2(kernel_l2_reg), kernel_initializer=initializers.RandomUniform( minval=-5e-3, maxval=5e-3))(net) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) net = layers.add([inputs, net]) return net
def build_model(self): states = layers.Input(shape=(self.state_size,), name='inputStates') # Hidden Layers model = layers.Dense(units=128, activation='linear')(states) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) model = layers.Dense(units=256, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) model = layers.Dense(units=512, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) model = layers.Dense(units=128, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) output = layers.Dense( units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(0.01), name='outputActions')(model) #Keras self.model = models.Model(inputs=states, outputs=output) #Definint Optimizer actionGradients = layers.Input(shape=(self.action_size,)) loss = K.mean(-actionGradients * output) optimizer = optimizers.Adam() update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, actionGradients, K.learning_phase()], outputs=[], updates=update_operation)
def residual_layer(input_tensor, nb_in_filters=64, nb_bottleneck_filters=16, filter_sz=3, stage=0, reg=0.0): bn_name = 'bn' + str(stage) conv_name = 'conv' + str(stage) relu_name = 'relu' + str(stage) merge_name = 'add' + str(stage) # batchnorm-relu-conv, from nb_in_filters to nb_bottleneck_filters via 1x1 conv if stage>1: # first activation is just after conv1 x = layers.BatchNormalization(axis=-1, name=bn_name+'a')(input_tensor) x = layers.Activation('relu', name=relu_name+'a')(x) else: x = input_tensor x = layers.Conv2D(nb_bottleneck_filters, (1, 1), kernel_initializer='glorot_normal', kernel_regularizer=regularizers.l2(reg), use_bias=False, name=conv_name+'a')(x) # batchnorm-relu-conv, from nb_bottleneck_filters to nb_bottleneck_filters via FxF conv x = layers.BatchNormalization(axis=-1, name=bn_name+'b')(x) x = layers.Activation('relu', name=relu_name+'b')(x) x = layers.Conv2D(nb_bottleneck_filters, (filter_sz, filter_sz), padding='same', kernel_initializer='glorot_normal', kernel_regularizer=regularizers.l2(reg), use_bias = False, name=conv_name+'b')(x) # batchnorm-relu-conv, from nb_in_filters to nb_bottleneck_filters via 1x1 conv x = layers.BatchNormalization(axis=-1, name=bn_name+'c')(x) x = layers.Activation('relu', name=relu_name+'c')(x) x = layers.Conv2D(nb_in_filters, (1, 1), kernel_initializer='glorot_normal', kernel_regularizer=regularizers.l2(reg), name=conv_name+'c')(x) # merge x = layers.add([x, input_tensor], name=merge_name) return x
def build_model(self): #Define input layers inputStates = layers.Input(shape=(self.state_size, ), name='inputStates') inputActions = layers.Input(shape=(self.action_size, ), name='inputActions') # Hidden layers for states modelS = layers.Dense(units=128, activation='linear')(inputStates) modelS = layers.BatchNormalization()(modelS) modelS = layers.LeakyReLU(0.01)(modelS) modelS = layers.Dropout(0.3)(modelS) modelS = layers.Dense(units=256, activation='linear')(modelS) modelS = layers.BatchNormalization()(modelS) modelS = layers.LeakyReLU(0.01)(modelS) modelS = layers.Dropout(0.3)(modelS) modelA = layers.Dense(units=256, activation='linear')(inputActions) modelA = layers.LeakyReLU(0.01)(modelA) modelA = layers.BatchNormalization()(modelA) modelA = layers.Dropout(0.5)(modelA) #Merging the models model = layers.add([modelS, modelA]) model = layers.Dense(units=256, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) #Q Layer Qvalues = layers.Dense(units=1, activation=None, name='outputQvalues')(model) #Keras model self.model = models.Model(inputs=[inputStates, inputActions], outputs=Qvalues) optimizer = optimizers.Adam() self.model.compile(optimizer=optimizer, loss='mse') actionGradients = K.gradients(Qvalues, inputActions) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=actionGradients)
def SE_block(x, out_dim, ratio, name, batch_norm=False): """ self attention squeeze-excitation block, attention mechanism on channel dimension :param x: input feature map :return: attention weighted on channel dimension feature map """ # Squeeze: global average pooling x_s = layers.GlobalAveragePooling2D(data_format=None)(x) # Excitation: bottom-up top-down FCs if batch_norm: x_s = layers.BatchNormalization()(x_s) x_e = layers.Dense(units=out_dim // ratio)(x_s) x_e = layers.Activation('relu')(x_e) if batch_norm: x_e = layers.BatchNormalization()(x_e) x_e = layers.Dense(units=out_dim)(x_e) x_e = layers.Activation('sigmoid')(x_e) x_e = layers.Reshape((1, 1, out_dim), name=name + 'channel_weight')(x_e) result = layers.multiply([x, x_e]) return result
def depth2_bn(x, filts, stride=1, multiplier=1, k=3, name=None, pad='same'): x = dw.DepthWiseConv2D(filts, (k, k), strides=(stride, stride), depth_multiplier=multiplier, padding=pad, use_bias=False, name=name + '_dw')(x) x = l.BatchNormalization(name=name + '_bn')(x) x = l.Activation('relu', name=name + '_relu')(x) return x
def conv2d_bn(x, filters, num_row, num_col, padding='same', strides=(1, 1)): if K.image_data_format() == 'channels_first': bn_axis = 1 else: bn_axis = 3 x = layers.Conv2D(filters, (num_row, num_col), strides=strides, padding=padding)(x) # use_bias=False, x = layers.BatchNormalization(axis=bn_axis)(x) # scale=False, x = layers.Activation('relu')(x) return x
def gating_signal(input, out_size, batch_norm=False): """ resize the down layer feature map into the same dimension as the up layer feature map using 1x1 conv :param input: down-dim feature map :param out_size:output channel number :return: the gating feature map with the same dimension of the up layer feature map """ x = layers.Conv2D(out_size, (1, 1), padding='same')(input) if batch_norm: x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) return x
def up_block(self, act, bn, f, name): x = layers.UpSampling2D( size=(2,2), name='upsample_{}'.format(name))(act) temp = layers.concatenate([bn, x], axis=1) temp = self.conv_bn_relu(temp, (3, 3), (1, 1), 2*f, 'layer2_{}'.format(name)) temp = layers.BatchNormalization(self.conv(temp, (3, 3), (1, 1), f, 'layer3_{}'.format( name)), momentum=0.99, name='layer3_bn_{}'.format(name)) #bn = layers.add([bn,x]) bn = self.shortcut(x, bn) act = layers.Activation('relu')(bn) return act
def add_resnet_unit(path, **params): """Add a resnet unit to path Returns new path """ block_input = path # add Conv2D path = L.Convolution2D( filters=params["filters_per_layer"], kernel_size=params["filter_width"], activation='linear', padding='same', kernel_regularizer=R.l2(.0001), bias_regularizer=R.l2(.0001))(path) # add BatchNorm path = L.BatchNormalization( beta_regularizer=R.l2(.0001), gamma_regularizer=R.l2(.0001))(path) # add ReLU path = L.Activation('relu')(path) # add Conv2D path = L.Convolution2D( filters=params["filters_per_layer"], kernel_size=params["filter_width"], activation='linear', padding='same', kernel_regularizer=R.l2(.0001), bias_regularizer=R.l2(.0001))(path) # add BatchNorm path = L.BatchNormalization( beta_regularizer=R.l2(.0001), gamma_regularizer=R.l2(.0001))(path) # Merge 'input layer' with the path path = L.Add()([block_input, path]) # add ReLU path = L.Activation('relu')(path) return path
def down_block(self, data, f, name): x = layers.MaxPooling2D(pool_size=(2,2), strides=(2,2))(data) # temp = conv_bn_relu(data, (3, 3), (2, 2), (1, 1), # f, 'layer1_{}'.format(name)) temp = self.conv_bn_relu(x, (3, 3), (1, 1), 2*f, 'layer2_{}'.format(name)) bn = layers.BatchNormalization(momentum=0.99, name='layer3_bn_{}'.format(name))(self.conv(temp, (3, 3), (1, 1), f, 'layer3_{}'.format( name))) #bn = layers.add([bn,x]) bn = self.shortcut(x,bn) act = layers.Activation('relu')(bn) return bn, act
def attention_block(x, gating, inter_shape, name): """ self gated attention, attention mechanism on spatial dimension :param x: input feature map :param gating: gate signal, feature map from the lower layer :param inter_shape: intermedium channle numer :param name: name of attention layer, for output :return: attention weighted on spatial dimension feature map """ shape_x = K.int_shape(x) shape_g = K.int_shape(gating) theta_x = layers.Conv2D(inter_shape, (2, 2), strides=(2, 2), padding='same')(x) # 16 shape_theta_x = K.int_shape(theta_x) phi_g = layers.Conv2D(inter_shape, (1, 1), padding='same')(gating) upsample_g = layers.Conv2DTranspose( inter_shape, (3, 3), strides=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]), padding='same')(phi_g) # 16 # upsample_g = layers.UpSampling2D(size=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]), # data_format="channels_last")(phi_g) concat_xg = layers.add([upsample_g, theta_x]) act_xg = layers.Activation('relu')(concat_xg) psi = layers.Conv2D(1, (1, 1), padding='same')(act_xg) sigmoid_xg = layers.Activation('sigmoid')(psi) shape_sigmoid = K.int_shape(sigmoid_xg) upsample_psi = layers.UpSampling2D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2]), name=name + '_weight')(sigmoid_xg) # 32 upsample_psi = expend_as(upsample_psi, shape_x[3]) y = layers.multiply([upsample_psi, x]) result = layers.Conv2D(shape_x[3], (1, 1), padding='same')(y) result_bn = layers.BatchNormalization()(result) return result_bn
def build_model(self): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" # Define input layers states = layers.Input(shape=(self.state_size, ), name='states') actions = layers.Input(shape=(self.action_size, ), name='actions') # Add hidden layer(s) for state pathway net_states = layers.Dense( units=32, activation='relu', use_bias=False, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(states) net_states = layers.BatchNormalization()(net_states) net_states = layers.LeakyReLU(1e-2)(net_states) net_states = layers.Dense( units=64, activation='relu', use_bias=False, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(net_states) net_states = layers.BatchNormalization()(net_states) net_states = layers.LeakyReLU(1e-2)(net_states) net_states = layers.Dense( units=128, activation='relu', use_bias=False, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(net_states) net_states = layers.BatchNormalization()(net_states) net_states = layers.LeakyReLU(1e-2)(net_states) # Add hidden layer(s) for action pathway net_actions = layers.Dense( units=32, activation='relu', use_bias=False, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(actions) net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.LeakyReLU(1e-2)(net_actions) net_actions = layers.Dense( units=64, activation='relu', use_bias=False, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(net_actions) net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.LeakyReLU(1e-2)(net_actions) net_actions = layers.Dense( units=128, activation='relu', use_bias=False, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(net_actions) net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.LeakyReLU(1e-2)(net_actions) # Try different layer sizes, activations, add batch normalization, regularizers, etc. # Combine state and action pathways net = layers.Add()([net_states, net_actions]) net = layers.Activation('relu')(net) # Add more layers to the combined network if needed # Add final output layer to prduce action values (Q values) Q_values = layers.Dense(units=1, name='q_values')(net) # Create Keras model self.model = models.Model(inputs=[states, actions], outputs=Q_values) # Define optimizer and compile model for training with built-in loss function optimizer = optimizers.Adam() self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used by actor model) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
'/home/manoolia/code/python/kaggle/titanic-challange/input/train.csv', '/home/manoolia/code/python/kaggle/titanic-challange/input/test.csv' ) data.load_data() model = models.Sequential() model.add(layers.Dense( units=240, activation=activations.sigmoid, input_shape=[5,] )) model.add(layers.Dropout(rate=0.2)) model.add(layers.BatchNormalization()) model.add(layers.Dense( units=160, activation=activations.relu, )) model.add(layers.Dropout(rate=0.2)) model.add(layers.BatchNormalization()) model.add(layers.Dense( units=80, activation=activations.sigmoid, )) model.add(layers.Dropout(rate=0.2)) model.add(layers.BatchNormalization()) model.add(layers.Dense(
def Attention_ResUNet_PA(dropout_rate=0.0, batch_norm=True): ''' Rsidual UNet construction, with attention gate convolution: 3*3 SAME padding pooling: 2*2 VALID padding upsampling: 3*3 VALID padding final convolution: 1*1 :param dropout_rate: FLAG & RATE of dropout. if < 0 dropout cancelled, if > 0 set as the rate :param batch_norm: flag of if batch_norm used, if True batch normalization :return: model ''' # input data # dimension of the image depth inputs = layers.Input((INPUT_SIZE, INPUT_SIZE, INPUT_CHANNEL), dtype=tf.float32) axis = 3 # Downsampling layers # DownRes 1, double residual convolution + pooling conv_128 = double_conv_layer(inputs, FILTER_SIZE, FILTER_NUM, dropout_rate, batch_norm) pool_64 = layers.MaxPooling2D(pool_size=(2, 2))(conv_128) # DownRes 2 conv_64 = double_conv_layer(pool_64, FILTER_SIZE, 2 * FILTER_NUM, dropout_rate, batch_norm) pool_32 = layers.MaxPooling2D(pool_size=(2, 2))(conv_64) # DownRes 3 conv_32 = double_conv_layer(pool_32, FILTER_SIZE, 4 * FILTER_NUM, dropout_rate, batch_norm) pool_16 = layers.MaxPooling2D(pool_size=(2, 2))(conv_32) # DownRes 4 conv_16 = double_conv_layer(pool_16, FILTER_SIZE, 8 * FILTER_NUM, dropout_rate, batch_norm) pool_8 = layers.MaxPooling2D(pool_size=(2, 2))(conv_16) # DownRes 5, convolution only conv_8 = double_conv_layer(pool_8, FILTER_SIZE, 16 * FILTER_NUM, dropout_rate, batch_norm) # Upsampling layers # UpRes 6, attention gated concatenation + upsampling + double residual convolution # channel attention block se_conv_16 = SE_block(conv_16, out_dim=8 * FILTER_NUM, ratio=SE_RATIO, name='att_16') # spatial attention block gating_16 = gating_signal(conv_8, 8 * FILTER_NUM, batch_norm) att_16 = attention_block(se_conv_16, gating_16, 8 * FILTER_NUM, name='att_16') # attention re-weight & concatenate up_16 = layers.UpSampling2D(size=(UP_SAMP_SIZE, UP_SAMP_SIZE), data_format="channels_last")(conv_8) up_16 = layers.concatenate([up_16, att_16], axis=axis) up_conv_16 = double_conv_layer(up_16, FILTER_SIZE, 8 * FILTER_NUM, dropout_rate, batch_norm) # UpRes 7 # channel attention block se_conv_32 = SE_block(conv_32, out_dim=4 * FILTER_NUM, ratio=SE_RATIO, name='att_32') # spatial attention block gating_32 = gating_signal(up_conv_16, 4 * FILTER_NUM, batch_norm) att_32 = attention_block(se_conv_32, gating_32, 4 * FILTER_NUM, name='att_32') # attention re-weight & concatenate up_32 = layers.UpSampling2D(size=(UP_SAMP_SIZE, UP_SAMP_SIZE), data_format="channels_last")(up_conv_16) up_32 = layers.concatenate([up_32, att_32], axis=axis) up_conv_32 = double_conv_layer(up_32, FILTER_SIZE, 4 * FILTER_NUM, dropout_rate, batch_norm) # UpRes 8 # channel attention block se_conv_64 = SE_block(conv_64, out_dim=2 * FILTER_NUM, ratio=SE_RATIO, name='att_64') # spatial attention block gating_64 = gating_signal(up_conv_32, 2 * FILTER_NUM, batch_norm) att_64 = attention_block(se_conv_64, gating_64, 2 * FILTER_NUM, name='att_64') # attention re-weight & concatenate up_64 = layers.UpSampling2D(size=(UP_SAMP_SIZE, UP_SAMP_SIZE), data_format="channels_last")(up_conv_32) up_64 = layers.concatenate([up_64, att_64], axis=axis) up_conv_64 = double_conv_layer(up_64, FILTER_SIZE, 2 * FILTER_NUM, dropout_rate, batch_norm) # UpRes 9 # channel attention block se_conv_128 = SE_block(conv_128, out_dim=FILTER_NUM, ratio=SE_RATIO, name='att_128') # spatial attention block gating_128 = gating_signal(up_conv_64, FILTER_NUM, batch_norm) # attention re-weight & concatenate att_128 = attention_block(se_conv_128, gating_128, FILTER_NUM, name='att_128') up_128 = layers.UpSampling2D(size=(UP_SAMP_SIZE, UP_SAMP_SIZE), data_format="channels_last")(up_conv_64) up_128 = layers.concatenate([up_128, att_128], axis=axis) up_conv_128 = double_conv_layer(up_128, FILTER_SIZE, FILTER_NUM, dropout_rate, batch_norm) # 1*1 convolutional layers # valid padding # batch normalization # sigmoid nonlinear activation conv_final = layers.Conv2D(OUTPUT_MASK_CHANNEL, kernel_size=(1, 1))(up_conv_128) conv_final = layers.BatchNormalization(axis=axis)(conv_final) conv_final = layers.Activation('relu')(conv_final) # Model integration model = models.Model(inputs, conv_final, name="AttentionSEResUNet") return model
def optimizedNework(self): self.model.add( layers.Conv2D(filters=16, kernel_size=(7, 7), padding='same', name='image_array', input_shape=(SIZE_FACE, SIZE_FACE, 1))) self.model.add(layers.BatchNormalization()) self.model.add( layers.Convolution2D(filters=16, kernel_size=(7, 7), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add(layers.Activation('relu')) self.model.add( layers.AveragePooling2D(pool_size=(2, 2), padding='same')) self.model.add(layers.Dropout(.5)) self.model.add( layers.Conv2D(filters=32, kernel_size=(5, 5), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add( layers.Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add(layers.Activation('relu')) self.model.add( layers.AveragePooling2D(pool_size=(2, 2), padding='same')) self.model.add(layers.Dropout(.5)) self.model.add( layers.Conv2D(filters=64, kernel_size=(3, 3), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add( layers.Conv2D(filters=64, kernel_size=(3, 3), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add(layers.Activation('relu')) self.model.add( layers.AveragePooling2D(pool_size=(2, 2), padding='same')) self.model.add(layers.Dropout(.5)) self.model.add( layers.Conv2D(filters=128, kernel_size=(3, 3), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add( layers.Convolution2D(filters=128, kernel_size=(3, 3), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add(layers.Activation('relu')) self.model.add( layers.AveragePooling2D(pool_size=(2, 2), padding='same')) self.model.add(layers.Dropout(.5)) self.model.add( layers.Conv2D(filters=256, kernel_size=(3, 3), padding='same')) self.model.add(layers.BatchNormalization()) self.model.add( layers.Conv2D(filters=3, kernel_size=(3, 3), padding='same')) self.model.add(layers.GlobalAveragePooling2D()) self.model.add(layers.Activation('softmax', name='predictions'))
def add_common_layers(y): y = layers.BatchNormalization()(y) y = layers.LeakyReLU()(y) return y
def bn_relu(self, data, name): return layers.Activation('relu')(layers.BatchNormalization(momentum=0.99, name='bn_{}'.format(name))(data))