def Decoder(hidden_sizes, use_BN=False, use_LN=False, use_WN=False, hidden_activation='relu'): inputs = tf.keras.Input(shape=(16, )) if use_WN: for i in range(len(hidden_sizes)): if i == 0: dec = WeightNormalization( Dense(hidden_sizes[i], activation=hidden_activation))(inputs) else: dec = WeightNormalization( Dense(hidden_sizes[i], activation=hidden_activation))(dec) if use_BN: dec = BatchNormalization()(dec) if use_LN: dec = LayerNormalization()(dec) else: for i in range(len(hidden_sizes)): if i == 0: dec = Dense(hidden_sizes[i], activation=hidden_activation)(inputs) else: dec = Dense(hidden_sizes[i], activation=hidden_activation)(dec) if use_BN: dec = BatchNormalization()(dec) if use_LN: dec = LayerNormalization()(dec) dec = Dense(8, activation='sigmoid')(dec) model = tf.keras.Model(inputs, dec) return model
def build_model(): model = Sequential([ Input(batch_input_shape=(None, 1000, 12)), BatchNormalization(), WeightNormalization( Dense(512, activation='relu', kernel_initializer='he_normal')), Dropout(.6), BatchNormalization(), WeightNormalization( Dense(512, activation='relu', kernel_initializer='he_normal')), Dropout(.4), BatchNormalization(), WeightNormalization( Dense(256, activation='relu', kernel_initializer='he_normal')), Dropout(.2), BatchNormalization(), WeightNormalization( Dense(128, activation='relu', kernel_initializer='he_normal')), Dropout(.2), Flatten(), BatchNormalization(), Dense(5, activation='sigmoid', kernel_initializer='glorot_normal') ]) model.compile(optimizer='adam', loss=tf.keras.losses.BinaryCrossentropy(), metrics=['accuracy']) return model
def build_model(): model = Sequential([ BatchNormalization(), WeightNormalization( Dense( 1028, activation='relu', kernel_initializer='he_uniform', )), Dropout(.2), BatchNormalization(), WeightNormalization( Dense( 512, activation='relu', kernel_initializer='he_uniform', )), Dropout(.2), BatchNormalization(), WeightNormalization( Dense(206, activation='sigmoid', kernel_initializer='glorot_uniform')), Dropout(.2), ]) model.compile( optimizer='adam', loss='binary_crossentropy', ) return model
def residual(inputs, n_filters, expansion, kernel_size, scaling): linear = 0.8 x = WeightNormalization( Conv2D(n_filters * expansion, (1, 1), padding='same', activation='relu'))(inputs) x = WeightNormalization( Conv2D(int(n_filters * linear), (1, 1), padding='same'))(x) x = WeightNormalization(Conv2D(n_filters, kernel_size, padding='same'))(x) if scaling: x = Lambda(lambda t: t * scaling)(x) x = Add()([inputs, x]) return x
def model1(): input_u = Input(shape = (u_fts_num,) ) layer_u = layer_BDWD(1024, activation = activation, kn_init = initializer, kn_reg = kn_reg, bias_init = bias_init_carefully) (input_u) layer_u = layer_BDWD(1024, activation = activation, kn_init = initializer, kn_reg = kn_reg, bias_init = bias_init_carefully) (layer_u) layer_u = layer_BDWD(512, activation = activation, kn_init = initializer, kn_reg = kn_reg, bias_init = bias_init_carefully) (layer_u) out_put = WeightNormalization(Dense(i_fts_num, activation = 'sigmoid' ))(layer_u) return Model(inputs=[input_u, ], outputs= [out_put])
def __init__(self, num_output_time_series, window_size, kernel_size, output_size, kernel_initializer, activation): super(DownsampleLayerWithAttention, self).__init__() self.num_output_time_series = num_output_time_series self.output_size = output_size self.kernel_initializer = kernel_initializer self.down_tcn = tf.keras.layers.Conv1D( filters=num_output_time_series, kernel_size=kernel_size, padding="causal", kernel_initializer=kernel_initializer, name=F"downsample_tcn") self.weight_norm_down_tcn = WeightNormalization( self.down_tcn, data_init=False, name=F"wn_downsample_tcn") self.query_dense_layer = tf.keras.layers.Dense(output_size) self.key_dense_layer = SepDenseLayer(num_output_time_series, window_size, window_size, kernel_initializer, activation=activation, use_bias=True, name="key") self.value_dense_layer = SepDenseLayer(num_output_time_series, window_size, window_size, kernel_initializer, activation=activation, use_bias=False, name="value") self.post_attention_layer = DotAttentionLayer(window_size)
def create_keras_model(inputShape, nClasses, scale=2, n_filters=32, depth=8, residual_expansion=6, residual_scaling=None): def residual(inputs, n_filters, expansion, kernel_size, scaling): linear = 0.8 x = WeightNormalization( Conv2D(n_filters * expansion, (1, 1), padding='same', activation='relu'))(inputs) x = WeightNormalization( Conv2D(int(n_filters * linear), (1, 1), padding='same'))(x) x = WeightNormalization(Conv2D(n_filters, kernel_size, padding='same'))(x) if scaling: x = Lambda(lambda t: t * scaling)(x) x = Add()([inputs, x]) return x inputs = Input(shape=inputShape) # main branch xm = WeightNormalization(Conv2D(n_filters, (3, 3), padding='same'))(inputs) for i in range(depth): xm = residual(xm, n_filters, residual_expansion, kernel_size=3, scaling=residual_scaling) xm = WeightNormalization( Conv2D(nClasses * scale**2, (3, 3), padding='same'))(xm) xm = Lambda(lambda x: tf.nn.depth_to_space(x, scale))(xm) # skip branch xs = WeightNormalization( Conv2D(nClasses * scale**2, (5, 5), padding='same'))(inputs) xs = Lambda(lambda x: tf.nn.depth_to_space(x, scale))(xs) outputs = Add()([xm, xs]) model = Model(inputs, outputs, name="wdsr") return model
def layer_cpl(input_layer): '''BN - DROPOUT - WEIGHTNORMAL - DENSE''' layer = BatchNormalization() (input_layer) layer = Dropout(0.25 ) (layer) dense = Dense(n_components, activation = activation, kernel_initializer = initializer, kernel_regularizer = kn_reg , bias_initializer = bias_init) layer = WeightNormalization(dense) (layer) return layer
def Encoder(hidden_sizes, use_BN=False, use_LN=False, use_WN=False, encoder_activation='tanh', hidden_activation='relu', enc_kernel_reg=None, enc_activity_reg=None, additional_layers=[]): inputs = tf.keras.Input(shape=(8, )) if use_WN: for i in range(len(hidden_sizes)): if i == 0: enc = WeightNormalization( Dense(hidden_sizes[i], activation=hidden_activation))(inputs) else: enc = WeightNormalization( Dense(hidden_sizes[i], activation=hidden_activation))(enc) if use_BN: enc = BatchNormalization()(enc) if use_LN: enc = LayerNormalization()(enc) else: for i in range(len(hidden_sizes)): if i == 0: enc = Dense(hidden_sizes[i], activation=hidden_activation)(inputs) else: enc = Dense(hidden_sizes[i], activation=hidden_activation)(enc) if use_BN: enc = BatchNormalization()(enc) if use_LN: enc = LayerNormalization()(enc) enc = Dense(16, activation=encoder_activation, kernel_regularizer=enc_kernel_reg, activity_regularizer=enc_activity_reg)(enc) for add_layer in additional_layers: enc = add_layer()(enc) model = tf.keras.Model(inputs, enc) return model
def final_pipe(): model = Sequential([ BatchNormalization(), WeightNormalization( Dense( 1028, activation='relu', kernel_initializer='he_uniform', )), Dropout(.2), BatchNormalization(), WeightNormalization( Dense( 512, activation='relu', kernel_initializer='he_uniform', )), Dropout(.2), BatchNormalization(), WeightNormalization( Dense(206, activation='sigmoid', kernel_initializer='glorot_uniform')), Dropout(.2), BatchNormalization(), WeightNormalization( Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform')), Dropout(.2), ]) model.compile( optimizer='adam', loss='mean_squared_logarithmic_error', ) return model
def DNN_model(input_size, output_size): """inputs = Input((input_size, )) outputs = BatchNormalization()(inputs) outputs = Dropout(0.20)(outputs) outputs = Dense(1024, activation="relu")(outputs) outputs = WeightNormalization()(outputs) outputs = BatchNormalization()(outputs) outputs = Dropout(0.20)(outputs) outputs = Dense(1024, activation="relu")(outputs) outputs = WeightNormalization()(outputs) outputs = BatchNormalization()(outputs) outputs = Dense(output_size, activation="sigmoid")(outputs) outputs = WeightNormalization()(outputs)""" model = Sequential([Input(input_size)]) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(WeightNormalization(Dense(1024, activation="relu"))) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(WeightNormalization(Dense(1024, activation="relu"))) model.add(BatchNormalization()) model.add(WeightNormalization(Dense(output_size, activation="sigmoid"))) #model = Model(inputs=[inputs], outputs=[outputs]) model.compile(optimizer=AdamW(lr=1e-3, weight_decay=1e-5, clipvalue=756), loss="binary_crossentropy") return model
def __init__(self, k, d, normalized=True): super(Residual, self).__init__() if normalized: self.causal1D = WeightNormalization(Conv1D(kernel_size=k, padding="causal", dilation_rate=2 ** (d - 1), activation="relu", filters=64, kernel_initializer='random_normal')) else: self.causal1D = Conv1D(kernel_size=k, padding="causal", dilation_rate=2 ** (d - 1), activation="relu", filters=64, kernel_initializer='random_normal') self.drop = SpatialDropout1D(0.1) self.matching_1D = Conv1D(kernel_size=1, padding="same", activation="relu", filters=64, kernel_initializer='random_normal')
def __init__(self, base_filters=32, lrelu_alpha=0.2, pad_type="reflect"): super(Discriminator, self).__init__(name="Discriminator") if pad_type == "reflect": self.flat_pad = ReflectionPadding2D() elif pad_type == "constant": self.flat_pad = ZeroPadding2D() else: raise ValueError(f"pad_type not recognized {pad_type}") self.flat_conv = Conv2D(base_filters, 3) self.flat_lru = LeakyReLU(lrelu_alpha) self.strided_conv1 = StridedConv(base_filters * 2, lrelu_alpha, pad_type, gp_num=32) self.strided_conv2 = StridedConv(base_filters * 4, lrelu_alpha, pad_type, gp_num=64) self.gp_norm = GroupNormalization(groups=64, axis=-1) self.conv2 = WeightNormalization(Conv2D(base_filters * 8, 3)) self.lrelu = LeakyReLU(lrelu_alpha) self.final_conv = Conv2D(1, 3)
def build(self, input_shape): with K.name_scope( self.name ): # name scope used to make sure weights get unique names self.layers = [] self.res_output_shape = input_shape for k in range(2): name = 'conv1D_{}'.format(k) with K.name_scope( name ): # name scope used to make sure weights get unique names conv = Conv1D(filters=self.nb_filters, kernel_size=self.kernel_size, dilation_rate=self.dilation_rate, padding=self.padding, name=name, kernel_initializer=self.kernel_initializer) if self.use_weight_norm: from tensorflow_addons.layers import WeightNormalization # wrap it. WeightNormalization API is different than BatchNormalization or LayerNormalization. with K.name_scope('norm_{}'.format(k)): conv = WeightNormalization(conv) self._build_layer(conv) with K.name_scope('norm_{}'.format(k)): if self.use_batch_norm: self._build_layer(BatchNormalization()) elif self.use_layer_norm: self._build_layer(LayerNormalization()) elif self.use_weight_norm: pass # done above. self._build_layer(Activation(self.activation)) self._build_layer(SpatialDropout1D(rate=self.dropout_rate)) if self.nb_filters != input_shape[-1]: # 1x1 conv to match the shapes (channel dimension). name = 'matching_conv1D' with K.name_scope(name): # make and build this layer separately because it directly uses input_shape self.shape_match_conv = Conv1D( filters=self.nb_filters, kernel_size=1, padding='same', name=name, kernel_initializer=self.kernel_initializer) else: name = 'matching_identity' self.shape_match_conv = Lambda(lambda x: x, name=name) with K.name_scope(name): self.shape_match_conv.build(input_shape) self.res_output_shape = self.shape_match_conv.compute_output_shape( input_shape) self._build_layer(Activation(self.activation)) self.final_activation = Activation(self.activation) self.final_activation.build( self.res_output_shape) # probably isn't necessary # this is done to force Keras to add the layers in the list to self._layers for layer in self.layers: self.__setattr__(layer.name, layer) self.__setattr__(self.shape_match_conv.name, self.shape_match_conv) self.__setattr__(self.final_activation.name, self.final_activation) super(ResidualBlock, self).build( input_shape) # done to make sure self.built is set True
# Hyper params NFOLDS = 7 BATCH_SIZE = 128 EPOCHS = 150 BAGGING_ALPHA = 0.75 SEEDS = [23, 228, 1488, 1998, 2208, 2077, 404] KFOLDS = 10 label_smoothing_alpha = 0.0005 P_MIN = label_smoothing_alpha P_MAX = 1 - P_MIN # Define model model = Sequential([ BatchNormalization(), WeightNormalization(Dense(1024, activation="relu")), BatchNormalization(), Dropout(0.25), WeightNormalization(Dense(512, activation="relu")), BatchNormalization(), Dropout(0.25), WeightNormalization(Dense(256, activation="relu")), BatchNormalization(), Dropout(0.25), WeightNormalization(Dense(num_labels, activation="sigmoid")) ]) def logloss(y_true, y_pred): y_pred = tf.clip_by_value(y_pred,P_MIN,P_MAX) return -K.mean(y_true*K.log(y_pred) + (1-y_true)*K.log(1-y_pred))
def __init__(self, block_num, filter_num, kernel_size, dilation_rate, window_size, use_bias, kernel_initializer, dropout_rate, dropout_format, activation, final_activation): super(BasicTCNBlock, self).__init__() self.dropout_rate = dropout_rate valid_dropout_formats = {"channel", "timestep", "all"} if dropout_format not in valid_dropout_formats: raise ValueError("Dropout format must be one of %r." % valid_dropout_formats) if dropout_format == "channel": self.noise_shape = [1, filter_num] elif dropout_format == "timestep": self.noise_shape = [window_size, 1] else: self.noise_shape = [window_size, filter_num] self.tcn_1 = tf.keras.layers.Conv1D( filters=filter_num, kernel_size=kernel_size, padding="causal", dilation_rate=dilation_rate, use_bias=use_bias, kernel_initializer=kernel_initializer, name=F"{block_num}_tcn_1") self.weight_norm_layer_1 = WeightNormalization( self.tcn_1, data_init=False, name=F"{block_num}_wn_1") self.tcn_2 = tf.keras.layers.Conv1D( filters=filter_num, kernel_size=kernel_size, padding="causal", dilation_rate=dilation_rate, use_bias=use_bias, kernel_initializer=kernel_initializer, name=F"{block_num}_tcn_2") self.weight_norm_layer_2 = WeightNormalization( self.tcn_2, data_init=False, name=F"{block_num}_wn_2") self.tcn_3 = tf.keras.layers.Conv1D( filters=filter_num, kernel_size=1, padding="causal", dilation_rate=dilation_rate, use_bias=use_bias, kernel_initializer=kernel_initializer, name=F"{block_num}_tcn_3") self.weight_norm_layer_3 = WeightNormalization( self.tcn_3, data_init=False, name=F"{block_num}_wn_3") self.dropout_layer_1 = tf.keras.layers.Dropout( rate=self.dropout_rate, noise_shape=self.noise_shape, name=F"{block_num}_dropout_1") self.dropout_layer_2 = tf.keras.layers.Dropout( rate=self.dropout_rate, noise_shape=self.noise_shape, name=F"{block_num}_dropout_2") self.activation = tf.keras.layers.Activation(activation) self.final_activation = tf.keras.layers.Activation(final_activation)