def __init__(self, hidden_size=1024, intermediate_hidden_size=4096, dropout=0.0, init_stddev=0.02, shared_intermediate=False, shared_output=False, **kwargs): super(FeedForward, self).__init__(**kwargs) self.hidden_size = hidden_size self.intermediate_hidden_size = intermediate_hidden_size self.dropout = dropout self.init_stddev = init_stddev self.shared_intermediate = shared_intermediate self.intermediate_dense = self.get_layer( shared_intermediate, 'ff_intermediate_dense', layers.Dense, intermediate_hidden_size, kernel_initializer=initializers.TruncatedNormal( stddev=init_stddev), activation=gelu, ) self.output_dense = self.get_layer( shared_output, 'ff_output_dense', layers.Dense, hidden_size, kernel_initializer=initializers.TruncatedNormal( stddev=init_stddev), )
def __init__(self, filters, kernel_size, name, use_bn=True, use_res=True, padding='SAME', use_bias=True): super().__init__(name=name) self.use_bn = use_bn self.use_res = use_res stddev = np.sqrt(2 / (kernel_size**2 * filters)) self.conv1 = layers.Conv3D( filters, kernel_size, padding=padding, use_bias=use_bias, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='conv1') self.conv2 = layers.Conv3D( filters, kernel_size, padding=padding, use_bias=use_bias, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='conv2') if use_bn: self.bn1 = layers.BatchNormalization(momentum=0.99, name='bn1') self.bn2 = layers.BatchNormalization(momentum=0.99, name='bn2') if use_res: self.res = _Residual('res')
def __init__(self, filters, kernel_size, pool_size, name, concat_or_add='concat', use_bn=True, use_res=True, padding='SAME', use_bias=True): super().__init__(name=name) self.use_bn = use_bn self.use_res = use_res self.concat_or_add = concat_or_add stddev = np.sqrt(2 / (kernel_size ** 2 * filters)) self.deconv = layers.Conv2DTranspose(filters // 2, kernel_size, strides=pool_size, padding=padding, use_bias=use_bias, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='deconv') self.conv1 = layers.Conv2D(filters // 2, kernel_size, padding=padding, use_bias=use_bias, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='conv1') self.conv2 = layers.Conv2D(filters // 2, kernel_size, padding=padding, use_bias=use_bias, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='conv2') self.conv3 = layers.Conv2D(filters //2, 1, padding=padding, use_bias=use_bias, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='conv3') if use_bn: self.bn_deconv = layers.BatchNormalization(momentum=0.99, name='bn_deconv') self.bn1 = layers.BatchNormalization(momentum=0.99, name='bn1') self.bn2 = layers.BatchNormalization(momentum=0.99, name='bn2') self.bn3 = layers.BatchNormalization(momentum=0.99, name='bn3') if use_res: self.res = _Residual('res')
def __init__(self, num_classes=6, L1=128, L2=256, cell_units=128, num_linear=768, p=10, time_step=100, F1=64, dropout_keep_prob=0.8): super().__init__() # strides = [1,1,1,1] self.conv1 = Conv2D(filters=L1, kernel_size=(5, 3), padding="same", use_bias=True, bias_initializer=initializers.constant(0.1)) self.dropout_keep_prob = dropout_keep_prob self.max_pool = MaxPool2D(pool_size=2, strides=2, padding='valid') self.conv2 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True, bias_initializer=initializers.constant(0.1)) self.conv3 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True, bias_initializer=initializers.constant(0.1)) self.conv4 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True, bias_initializer=initializers.constant(0.1)) self.conv5 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True, bias_initializer=initializers.constant(0.1)) self.conv6 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True, bias_initializer=initializers.constant(0.1)) self.dropout = Dropout(dropout_keep_prob) self.reshape1 = Reshape((-1, time_step, L2 * p)) self.reshape2 = Reshape((-1, L2 * p)) # self.reshape3 = Reshape((-1, time_step, num_linear)) self.flatten = Flatten() self.d1 = Dense(num_linear, use_bias=True, bias_initializer=initializers.constant(0.1), kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1)) self.bn = BatchNormalization() self.d2 = Dense(F1, use_bias=True, bias_initializer=initializers.constant(0.1), kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1)) self.d3 = Dense(num_classes, use_bias=True, bias_initializer=initializers.constant(0.1), kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1), activation='softmax') self.bilstm = Bidirectional(LSTM(cell_units, return_sequences=True), merge_mode='concat') self.attention = attention()
def __init__(self, conf, rate, nc): """ Parameters ---------- conf: Dictionary Configuration dictionary. """ # Initialize. self.conf = conf self.raw_data_path = self.conf['raw_data_path'] self.hps = self.conf['hps'] self.nn_arch = self.conf['nn_arch'] super(Module2, self).__init__() # Design layers. # Split separable conv3d. self.sep_conv3d_1 = SeparableConv3D( nc, kernel_size=3, depth_multiplier=1, dilation_rate=(rate[0] * self.nn_arch['conv_rate_multiplier'], rate[1] * self.nn_arch['conv_rate_multiplier'], rate[2] * self.nn_arch['conv_rate_multiplier']), padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal()) self.bn_1 = BatchNormalization(momentum=self.hps['bn_momentum'], scale=self.hps['bn_scale']) self.act_1 = Activation('relu') self.conv3d_2 = Conv3D( np.maximum(1, int(nc / 2)), kernel_size=3, strides=2, padding='valid', use_bias=False, kernel_initializer=initializers.TruncatedNormal(), kernel_regularizer=regularizers.l2(self.hps['weight_decay'])) self.bn_2 = BatchNormalization(momentum=self.hps['bn_momentum'], scale=self.hps['bn_scale']) self.act_2 = Activation('relu') self.sep_conv3d_3 = SeparableConv3D( nc, kernel_size=3, depth_multiplier=1, dilation_rate=(rate[0] * self.nn_arch['conv_rate_multiplier'], rate[1] * self.nn_arch['conv_rate_multiplier'], rate[2] * self.nn_arch['conv_rate_multiplier']), padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal()) self.bn_3 = BatchNormalization(momentum=self.hps['bn_momentum'], scale=self.hps['bn_scale']) self.act_3 = Activation('relu')
def create_keras_three_layer_dense_model(*, input_size, output_size, verbose=False, **kwargs ): # ................................................... # Create model "use the same parameters for 1st and seocnd layer" model = Sequential() #.. First hidden layer model.add(Dense( input_dim=input_size, units=kwargs['h1_unit_size'], activation=kwargs["h1_activation"], kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.01, seed=0) )) #.. Second hidden layer model.add(Dense( units=kwargs['h1_unit_size'], activation=kwargs["h1_activation"], kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.01, seed=0) )) #.. Output layer model.add(Dense( units=output_size, activation=kwargs["out_activation"], kernel_regularizer=tf.keras.regularizers.l2(0.001), kernel_initializer=initializers.VarianceScaling(scale=1.0, seed=0) )) # Print network summary if verbose==True: print(model.summary()) else: pass # ................................................... # Define Loss Function and Trianing Operation """ # [option]: Use only default values, model.compile( optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['acc']) """ model.compile( optimizer= kwargs["optimizer"], loss= losses.sparse_categorical_crossentropy, metrics= kwargs["metrics"] # even one arg must be in the list ) return model
def build_model(learning_rate=0.001, decay=10e-6, momentum=0.9, l2_parameter=0.1, hidden_layer_size1=256, hidden_layer_size2=64, input_dim=1024, third_layer=True): model = models.Sequential() model.add( layers.Dense(units=hidden_layer_size1, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / input_dim), seed=None), activation='relu', input_shape=(input_dim, ))) model.add(layers.BatchNormalization()) model.add( layers.Dense(units=hidden_layer_size2, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / hidden_layer_size1), seed=None), activation='relu')) model.add(layers.BatchNormalization()) if third_layer == True: model.add( layers.Dense(units=16, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / hidden_layer_size2), seed=None), activation='relu')) model.add(layers.BatchNormalization()) model.add( layers.Dense(1, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / 16), seed=None))) model.compile(optimizer=optimizers.SGD(lr=learning_rate, decay=decay, momentum=momentum, nesterov=True), loss='mse', metrics=['mse']) return model
class MLP(layers.Layer): """ MLP as used in Vision Transformer, MLP-Mixer and related networks """ k_ini = initializers.TruncatedNormal(stddev=0.02) b_ini = initializers.Zeros() def __init__(self, in_features, mlp_ratio=4.0, drop=0., name=None): super(MLP, self).__init__(name=name) self.fc1 = layers.Dense(int(in_features * mlp_ratio), name="fc1", kernel_initializer=self.k_ini, bias_initializer=self.b_ini) self.act = layers.Activation("gelu") self.fc2 = layers.Dense(in_features, name="fc2", kernel_initializer=self.k_ini, bias_initializer=self.b_ini) self.drop = layers.Dropout(drop) def call(self, x, training=None): x = self.fc1(x) x = self.act(x) x = self.drop(x, training=training) x = self.fc2(x) x = self.drop(x, training=training) return x
def __init__(self, class_num, init_stddev=0.02, **kwargs): super(PreTrainNextSentencePredictor, self).__init__(**kwargs) self.dense = layers.Dense( class_num, kernel_initializer=initializers.TruncatedNormal( stddev=init_stddev), )
def __init__(self, conf, nc): """ Parameters ---------- conf: Dictionary Configuration dictionary. """ # Initialize. self.conf = conf self.raw_data_path = self.conf['raw_data_path'] self.hps = self.conf['hps'] self.nn_arch = self.conf['nn_arch'] super(Module5, self).__init__() # Design layers. self.conv3d_1 = Conv3D( nc, kernel_size=3, strides=1, padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal(), kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
def build(self, input_shape): # define a parameter table of relative position bias # [2*Mh-1 * 2*Mw-1, nH] self.relative_position_bias_table = self.add_weight( shape=[ (2 * self.window_size[0] - 1) * (2 * self.window_size[1] - 1), self.num_heads ], initializer=initializers.TruncatedNormal(stddev=0.02), trainable=True, dtype=tf.float32, name="relative_position_bias_table") coords_h = np.arange(self.window_size[0]) coords_w = np.arange(self.window_size[1]) coords = np.stack(np.meshgrid(coords_h, coords_w, indexing="ij")) # [2, Mh, Mw] coords_flatten = np.reshape(coords, [2, -1]) # [2, Mh*Mw] # [2, Mh*Mw, 1] - [2, 1, Mh*Mw] relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # [2, Mh*Mw, Mh*Mw] relative_coords = np.transpose(relative_coords, [1, 2, 0]) # [Mh*Mw, Mh*Mw, 2] relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 relative_coords[:, :, 1] += self.window_size[1] - 1 relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 relative_position_index = relative_coords.sum(-1) # [Mh*Mw, Mh*Mw] self.relative_position_index = tf.Variable( tf.convert_to_tensor(relative_position_index), trainable=False, dtype=tf.int64, name="relative_position_index")
def __init__(self, conf, rate, nc): """ Parameters ---------- conf: Dictionary Configuration dictionary. """ # Initialize. self.conf = conf self.raw_data_path = self.conf['raw_data_path'] self.hps = self.conf['hps'] self.nn_arch = self.conf['nn_arch'] super(Module7, self).__init__() # Design layers. self.add_1 = Add() self.conv3d_1 = Conv3D( nc, kernel_size=3, dilation_rate=(rate[0] * self.nn_arch['conv_rate_multiplier'], rate[1] * self.nn_arch['conv_rate_multiplier'], rate[2] * self.nn_arch['conv_rate_multiplier']), padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal()) self.bn_1 = BatchNormalization(momentum=self.hps['bn_momentum'], scale=self.hps['bn_scale']) self.act_1 = Activation('relu')
def __init__(self, conf, nc): """ Parameters ---------- conf: Dictionary Configuration dictionary. """ # Initialize. self.conf = conf self.raw_data_path = self.conf['raw_data_path'] self.hps = self.conf['hps'] self.nn_arch = self.conf['nn_arch'] super(Module6, self).__init__() # Design layers. self.upsampling3d_1 = UpSampling3D() self.conv3d_1 = Conv3D( nc, kernel_size=3, strides=1, padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal(), kernel_regularizer=regularizers.l2(self.hps['weight_decay'])) self.bn_1 = BatchNormalization(momentum=self.hps['bn_momentum'], scale=self.hps['bn_scale']) self.act_1 = Activation('relu')
def make_original_like(conv_filters=16, conv_filter_shape=5, nodes=1024): seq = start_seq() seq.add(layers.Conv2D( conv_filters, conv_filter_shape, padding='same', kernel_initializer=initializers.TruncatedNormal(stddev=0.1))) seq.add(layers.MaxPooling2D((2, 2), (2, 2), padding='same')) seq.add(layers.Flatten()) seq.add(layers.Dense( nodes, kernel_initializer=initializers.TruncatedNormal(), bias_initializer=initializers.zeros(), activation=activations.relu)) return end_seq(seq)
def __init__(self, n_class, n_layer, root_filters, kernal_size=3, pool_size=2, use_bn=True, use_res=True, padding='SAME', concat_or_add='concat'): super().__init__() self.dw_layers = dict() self.up_layers = dict() self.max_pools = dict() self.dw1_layers = dict() for layer in range(n_layer): filters = 2 ** layer * root_filters dict_key = str(n_layer - layer - 1) if layer == 0: Dw1 = _DownSev(filters, 7, 'dw1_%d' % layer) self.dw1_layers[dict_key] = Dw1 dw = _DownSampling(filters, kernal_size, 'dw_%d' % layer, use_bn, use_res) self.dw_layers[dict_key] = dw if layer < n_layer - 1: pool = layers.MaxPool2D([2, 2], padding=padding) self.max_pools[dict_key] = pool for layer in range(n_layer - 2, -1, -1): filters = 2 ** (layer + 1) * root_filters dict_key = str(n_layer - layer - 1) up = _UpSampling(filters, kernal_size, [2, 2], concat_or_add) self.up_layers[dict_key] = up stddev = np.sqrt(2 / (kernal_size ** 2 * root_filters)) self.conv_out = layers.Conv2D(n_class, 1, padding=padding, use_bias=False, kernel_initializer=initializers.TruncatedNormal(stddev=stddev), name='conv_out')
def conv_batch_lrelu(input_tensor, numfilter, dim, strides=1): input_tensor = Conv2D(numfilter, (dim, dim), strides=strides, padding='same', kernel_regularizer=regularizers.l2(0.0005), kernel_initializer=initializers.TruncatedNormal(stddev=0.1), use_bias=False )(input_tensor) input_tensor = BatchNormalization()(input_tensor) return LeakyReLU(alpha=0.1)(input_tensor)
def build(self, input_shape): he_std = self.gain / np.sqrt(np.prod(input_shape[1:], axis=-1)) #? init_std = 1.0 / self.lrmul self.runtime_coeff = he_std * self.lrmul self.kernel_initializer = initializers.TruncatedNormal( mean=0.0, stddev=init_std) #? super(EqualizedLRDense, self).build(input_shape)
def __init__(self, patch_size=4, num_classes=1000, embed_dim=96, depths=(2, 2, 6, 2), num_heads=(3, 6, 12, 24), window_size=7, mlp_ratio=4., qkv_bias=True, drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, norm_layer=layers.LayerNormalization, name=None, **kwargs): super().__init__(name=name) self.num_classes = num_classes self.num_layers = len(depths) self.embed_dim = embed_dim self.mlp_ratio = mlp_ratio # split image into non-overlapping patches self.patch_embed = PatchEmbed(patch_size=patch_size, embed_dim=embed_dim, norm_layer=norm_layer) self.pos_drop = layers.Dropout(drop_rate) # stochastic depth decay rule dpr = [x for x in np.linspace(0, drop_path_rate, sum(depths))] # build layers self.stage_layers = [] for i_layer in range(self.num_layers): # 注意这里构建的stage和论文图中有些差异 # 这里的stage不包含该stage的patch_merging层,包含的是下个stage的 layer = BasicLayer( dim=int(embed_dim * 2**i_layer), depth=depths[i_layer], num_heads=num_heads[i_layer], window_size=window_size, mlp_ratio=self.mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, name=f"layer{i_layer}") self.stage_layers.append(layer) self.norm = norm_layer(epsilon=1e-6, name="norm") self.head = layers.Dense( num_classes, kernel_initializer=initializers.TruncatedNormal(stddev=0.02), bias_initializer=initializers.Zeros(), name="head")
def build_model(self, input_shape, dim, data_dim): input = Input(shape=input_shape, batch_size=self.batch_size) x = Dense(dim, kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.5), activation='relu')(input) x = Dense(dim * 2, activation='relu')(x) x = Dense(dim * 4, activation='relu')(x) x = Dense(data_dim)(x) return Model(inputs=input, outputs=x)
def ConvLayer(ConvIn, NumFilters, FilterSize=4, StrideLength=2, DropOutRate=False, Activation=True, BatchNormalizationON=True, Padding="same", Alpha=0.2): WeightsInitializer = initializers.TruncatedNormal(mean=0.0, stddev=0.02, seed=None) Layer = Conv2D(NumFilters, FilterSize, StrideLength, padding=Padding, kernel_initializer=WeightsInitializer)(ConvIn) if BatchNormalizationON: Layer = BatchNormalization()(Layer) if Activation: Layer = LeakyReLU(alpha=Alpha)(Layer) return Layer
def __init__(self, dim: int, norm_layer=layers.LayerNormalization, name=None): super(PatchMerging, self).__init__(name=name) self.dim = dim self.reduction = layers.Dense( 2 * dim, use_bias=False, kernel_initializer=initializers.TruncatedNormal(stddev=0.02), name="reduction") self.norm = norm_layer(epsilon=1e-6, name="norm")
def __init__(self, units, activation='sigmoid', use_bias=True, weight_initializer=None, bias_initializer='zeros', connection_initializer=None, connection_regularizer=None, connection_constraint=None, round_input=True, round_connections=True, clip_connections=True, round_bias=True, constrain_after_train=True, **kwargs): super(Tea, self).__init__(**kwargs) self.units = units self.activation = activations.get(activation) self.use_bias = use_bias if connection_initializer: self.connection_initializer = connection_initializer else: self.connection_initializer = initializers.TruncatedNormal( mean=0.5) if weight_initializer: self.weight_initializer = weight_initializer else: self.weight_initializer = tea_weight_initializer self.bias_initializer = bias_initializer self.connection_regularizer = connection_regularizer self.connection_constraint = connection_constraint self.input_width = None self.round_input = round_input self.round_connections = round_connections self.clip_connections = clip_connections self.round_bias = round_bias self.constrain_after_train = constrain_after_train self.uses_learning_phase = True
def build_model(learning_rate=0.001, decay=10e-6, momentum=0.9, l2_parameter=0.1): model = models.Sequential() model.add( layers.Dense(units=256, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / 1024), seed=None), activation='relu', input_shape=(1024, ))) model.add(layers.BatchNormalization()) model.add( layers.Dense(units=64, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / 256), seed=None), activation='relu')) model.add(layers.BatchNormalization()) model.add( layers.Dense(units=16, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / 128), seed=None), activation='relu')) model.add(layers.BatchNormalization()) model.add( layers.Dense(1, kernel_regularizer=regularizers.l2(l2_parameter), kernel_initializer=initializers.TruncatedNormal( mean=0.0, stddev=np.sqrt(2. / 64), seed=None))) model.compile(optimizer=optimizers.SGD(lr=learning_rate, decay=decay, momentum=0.9, nesterov=True), loss='mse', metrics=['mse']) return model
def CNN(inputs, filter_size, pool_size): conv_initializer = initializers.TruncatedNormal(mean=0., stddev=0.12) fc_initializer = initializers.TruncatedNormal(mean=0., stddev=0.06) x = Conv1D(filters=32, kernel_size=filter_size, padding='valid', kernel_regularizer=regularizers.l2(5e-5), bias_regularizer=regularizers.l2(5e-5), kernel_initializer=conv_initializer)(inputs) x = GroupNormalization(groups=4, axis=-1)(x) x = Activation('relu')(x) #x = LeakyReLU(alpha=0.1)(x) x = MaxPooling1D(pool_size=pool_size)(x) x = Dropout(0.25)(x) x = Flatten()(x) x = Dense(64, kernel_regularizer=regularizers.l2(5e-4), bias_regularizer=regularizers.l2(5e-4), kernel_initializer=fc_initializer)(x) x = Activation('relu')(x) #x = LeakyReLU(alpha=0.1)(x) x = Dropout(0.25)(x) return x
def ConvTransLayer(ConvTransIn, NumFilters, FilterSize=4, StrideLength=2, DropOutRate=False, convOut=None, Activation=True, BatchNormalizationON=True, Padding="same", Alpha=0.2): WeightsInitializer = initializers.TruncatedNormal(mean=0.0, stddev=0.02, seed=None) Layer = Conv2DTranspose(NumFilters, FilterSize, StrideLength, padding='same', kernel_initializer=WeightsInitializer)( concatenate([ConvTransIn, convOut]) if convOut is not None else ConvTransIn) if BatchNormalizationON: Layer = BatchNormalization()(Layer) if Activation: Layer = LeakyReLU(alpha=Alpha)(Layer) if DropOutRate: Layer = Dropout(rate=DropOutRate)(Layer) return Layer
def __init__(self, conf, nc): """ Parameters ---------- conf: Dictionary Configuration dictionary. """ # Initialize. self.conf = conf self.raw_data_path = self.conf['raw_data_path'] self.hps = self.conf['hps'] self.nn_arch = self.conf['nn_arch'] super(Module3, self).__init__() # Design layers. self.global_avg_pool3d_1 = GlobalAveragePooling3D() # data_format? self.reshape_1 = Reshape((1, 1, 1, nc)) self.conv3d_1 = Conv3D( np.maximum(1, int(nc / 2)), kernel_size=1, strides=1, padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal(), kernel_regularizer=regularizers.l2(self.hps['weight_decay'])) self.conv3d_2 = Conv3D( nc, kernel_size=1, strides=1, padding='same', use_bias=False, kernel_initializer=initializers.TruncatedNormal(), kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
def conv_block(net, style_weights, filters, kernel_size, strides, activation, depthwise_separable_conv): """ first applies padding with mode=REFLECT, then a valid conv2d, then conditional instance norm using style_weights and finally, activation TODO: add support to choose between regular and depthwise separable convolutions """ pad_0 = (kernel_size[0] - 1) // 2 pad_1 = (kernel_size[1] - 1) // 2 net = Lambda( lambda t: tf.pad(t, [[0, 0], [pad_0, pad_0], [pad_1, pad_1], [0, 0]], mode="REFLECT"))(net) weight_initializer = initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=1) if depthwise_separable_conv: net = SeparableConv2D( filters=filters, kernel_size=kernel_size, strides=strides, padding="valid", activation=None, depthwise_initializer=weight_initializer, pointwise_initializer=weight_initializer, )(net) else: net = Conv2D( filters=filters, kernel_size=kernel_size, strides=strides, padding="valid", activation=None, kernel_initializer=weight_initializer, )(net) net = ConditionalInstanceNormalization( style_weights.shape[-1])([net, style_weights]) if activation != None: net = Activation(activation)(net) return net
def build_Embedded(self, action_space=6, dueling=True): self.network_size = 256 X_input = Input(shape=(self.REM_STEP*99) ) # X_input = Input(shape=(self.REM_STEP*7,)) input_reshape=(self.REM_STEP,99) X = X_input truncatedn_init = initializers.TruncatedNormal(0, 1e-2) x_init = "he_uniform" y_init = initializers.glorot_uniform() const_init = initializers.constant(1e-2) X_reshaped = Reshape(input_reshape)(X_input) X = LayerNormalization(axis=2)(X_reshaped) X = LayerNormalization(axis=1)(X) X = Reshape((2,-1))(X) X = Dense(self.network_size*2, kernel_initializer=y_init, activation ="relu")(X) X = Dense(256, kernel_initializer=y_init, activation ="relu")(X) X = TimeDistributed(Dense(self.network_size/4, kernel_initializer=y_init,activation ="relu"))(X) X = Flatten()(X) if dueling: state_value = Dense( 1, activation ="softmax")(X) state_value = Lambda(lambda s: K.expand_dims( s[:, 0], -1), output_shape=(action_space,))(state_value) action_advantage = Dense( action_space, activation ="linear") (X) action_advantage = Lambda(lambda a: a[:, :] - K.mean( a[:, :], keepdims=True), output_shape=(action_space,))(action_advantage) X = Add()([state_value, action_advantage]) else: # Output Layer with # of actions: 2 nodes (left, right) X = Dense(action_space, activation="relu", kernel_initializer='he_uniform')(X) model = Model(inputs=X_input, outputs=X, name='build_Embedded') model.compile(loss=huber_loss, optimizer=Adam( lr=self.learning_rate), metrics=["accuracy"]) # model.compile(loss="mean_squared_error", optimizer=Adam(lr=0.00025,epsilon=0.01), metrics=["accuracy"]) model.summary() return model
def _conv_transpose_block(net, filters, kernel_size, strides, padding, activation): # https://github.com/lengstrom/fast-style-transfer/blob/master/src/transform.py#L59-L67 weights_initializer = initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=1) net = Conv2DTranspose(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, kernel_initializer=weights_initializer)(net) net = InstanceNormalization()(net) if activation != None: net = Activation(activation)(net) return net
def PolyA_CNN(length, seq_kernel_size=6, cov_kernel_size=12, pool_size=6): input_shape1 = (length, 4) input_shape2 = (length, 1) seq_input = Input(shape=input_shape1, name="seq_input") cov_input = Input(shape=input_shape2, name="cov_input") fc_initializer = initializers.TruncatedNormal(mean=0., stddev=0.06) # x = CNN(seq_input, seq_kernel_size, pool_size) y = CNN(cov_input, cov_kernel_size, pool_size) input_layers = [seq_input, cov_input] outLayer = Dense(1, kernel_regularizer=regularizers.l2(5e-4), bias_regularizer=regularizers.l2(5e-4), kernel_initializer=fc_initializer, activation='sigmoid')(concatenate([x, y])) model = Model(inputs=input_layers, outputs=outLayer) return model