def build(self, input_shape): self.d_model = input_shape[-1] # Self multi head attention self.multi_head_attention_1 = MultiHeadAttention(self.nb_proj) self.dropout_1 = layers.Dropout(rate=self.dropout_rate) self.norm_1 = layers.LayerNormalization(epsilon=1e-6) # Multi head attention combined with encoder output self.multi_head_attention_2 = MultiHeadAttention(self.nb_proj) self.dropout_2 = layers.Dropout(rate=self.dropout_rate) self.norm_2 = layers.LayerNormalization(epsilon=1e-6) # Feed foward self.dense_1 = layers.Dense( units=self.FFN_units, activation="relu", kernel_initializer=initializers.RandomNormal(stddev=0.01, seed=3), bias_initializer=initializers.Zeros()) self.dense_2 = layers.Dense( units=self.d_model, kernel_initializer=initializers.RandomNormal(stddev=0.01, seed=3), bias_initializer=initializers.Zeros()) self.dropout_3 = layers.Dropout(rate=self.dropout_rate) self.norm_3 = layers.LayerNormalization(epsilon=1e-6)
def MyNet_tf(input_shape: Tuple[int, int, int] = (28, 28, 1), classes: int = 10) -> tf.keras.Model: """A standard DNN implementation in TensorFlow. The MyNet model has 3 dense layers. Args: input_shape: shape of the input data (height, width, channels). classes: The number of outputs the model should generate. Returns: A TensorFlow MyNet model. """ #_check_input_shape(input_shape) model = Sequential() model.add(layers.Flatten(input_shape=(28, 28))) model.add( layers.Dense(units=300, kernel_initializer=initializers.Constant(0.01), bias_initializer=initializers.Zeros(), activation='relu')) model.add( layers.Dense(units=64, kernel_initializer=initializers.Constant(0.01), bias_initializer=initializers.Zeros(), activation='relu')) model.add( layers.Dense(units=classes, kernel_initializer=initializers.Constant(0.01), bias_initializer=initializers.Zeros(), activation='softmax')) return model
def build_model( self, input_shape, number_of_layers=5, neurons_by_layers=10, bias_mu_throttle=0.5, bias_mu_steering=0.0, bias_sigma_throttle=0.03, # ~1% outside (0.5 +- 0.4) bias_sigma_steering=0.1, # ~2% outside (0 +- 0.9) dropout=0.15): """ Construct the actor network with mu and sigma as output """ print(f"Input shape of policy: {input_shape}") inputs = layers.Input(shape=input_shape) prev_layer = inputs for i in range(number_of_layers): current_layer = layers.Dense( neurons_by_layers, activation="relu", kernel_initializer=initializers.he_normal())(prev_layer) if i < number_of_layers - 1: prev_layer = current_layer layers.Dropout(dropout)(prev_layer) prev_layer = current_layer current_layer = layers.Flatten()(prev_layer) mu_throttle = layers.Dense(1, activation="linear", kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Constant( bias_mu_throttle))(current_layer) sigma_throttle = layers.Dense(1, activation="softplus", kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Constant( bias_sigma_throttle))(current_layer) mu_steering = layers.Dense(1, activation="linear", kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Constant( bias_mu_steering))(current_layer) sigma_steering = layers.Dense(1, activation="softplus", kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Constant( bias_sigma_steering))(current_layer) actor_network = keras.Model( inputs=inputs, outputs=[mu_throttle, sigma_throttle, mu_steering, sigma_steering]) return actor_network
def build(self, input_shape): if self.my_dtype.is_complex: i_kernel_dtype = self.my_dtype if isinstance( self.kernel_initializer, ComplexInitializer) else self.my_dtype.real_dtype i_bias_dtype = self.my_dtype if isinstance( self.bias_initializer, ComplexInitializer) else self.my_dtype.real_dtype i_kernel_initializer = self.kernel_initializer i_bias_initializer = self.bias_initializer if not isinstance(self.kernel_initializer, ComplexInitializer): tf.print( f"WARNING: you are using a Tensorflow Initializer for complex numbers. " f"Using {self.init_technique} method.") if self.init_technique in INIT_TECHNIQUES: if self.init_technique == 'zero_imag': # This section is done to initialize with tf initializers, making imaginary part zero i_kernel_initializer = initializers.Zeros() i_bias_initializer = initializers.Zeros() else: raise ValueError( f"Unsuported init_technique {self.init_technique}, " f"supported techniques are {INIT_TECHNIQUES}") self.w_r = tf.Variable(name='kernel_r', initial_value=self.kernel_initializer( shape=(input_shape[-1], self.units), dtype=i_kernel_dtype), trainable=True) self.w_i = tf.Variable(name='kernel_i', initial_value=i_kernel_initializer( shape=(input_shape[-1], self.units), dtype=i_kernel_dtype), trainable=True) if self.use_bias: self.b_r = tf.Variable(name='bias_r', initial_value=self.bias_initializer( shape=(self.units, ), dtype=i_bias_dtype), trainable=self.use_bias) self.b_i = tf.Variable(name='bias_i', initial_value=i_bias_initializer( shape=(self.units, ), dtype=i_bias_dtype), trainable=self.use_bias) else: # TODO: For Complex you should probably want to use MY init for real keras. DO sth! at least error message self.w = self.add_weight( 'kernel', shape=(input_shape[-1], self.units), dtype=self.my_dtype, initializer=self.kernel_initializer, trainable=True, ) if self.use_bias: self.b = self.add_weight('bias', shape=(self.units, ), dtype=self.my_dtype, initializer=self.bias_initializer, trainable=self.use_bias)
def build_model( hparams ): input_layer = Input(shape=(hparams["max_sequence_length"], )) embedding_layer_static = get_w2v('').get_keras_embedding(train_embeddings=False)(input_layer) embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=True)(input_layer) submodels = [] kernel_sizes = hparams['kernel_sizes'].split('-') for ks in kernel_sizes: model = Sequential() conv_1_d = Conv1D( activation = 'relu', filters = hparams["filters"], kernel_size = int(ks), kernel_constraint = max_norm(hparams["max_norm_value"]) ) conv_layer_static = conv_1_d(embedding_layer_static) conv_layer = conv_1_d(embedding_layer) max_pooling_static = GlobalMaxPooling1D()(conv_layer_static) max_pooling = GlobalMaxPooling1D()(conv_layer) concatenate_layer = Concatenate()([max_pooling_static, max_pooling]) submodels.append(concatenate_layer) concat = Concatenate()(submodels) dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat) hidden_layer = Dense( hparams['hidden_size'], activation = 'relu', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(len(kernel_sizes) * 2* hparams['filters']), maxval = 1 / np.sqrt(len(kernel_sizes) * 2 * hparams['filters']) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_1) dropout_layer_2 = Dropout(hparams['dropout_ratio'])(hidden_layer) output_layer = Dense( 2, activation = 'sigmoid', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(hparams['hidden_size']), maxval = 1 / np.sqrt(hparams['hidden_size']) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_2) model = Model(inputs=[input_layer], outputs=[output_layer]) model.compile( loss = dice_loss, optimizer = Adam(learning_rate = hparams["learning_rate"]), metrics = [f1_score] ) from keras.utils.vis_utils import plot_model plot_model(model, "model_cnn_multichannel.png", show_layer_names=False) return model
def __init__(self, num_channels): super().__init__() self.W1 = tf.keras.layers.Dense(units = num_channels, kernel_initializer=initializers.RandomNormal(stddev=1/num_channels**2), bias_initializer=initializers.Zeros()) self.W2 = tf.keras.layers.Dense(units = num_channels, kernel_initializer=initializers.RandomNormal(stddev=1/num_channels**2), bias_initializer=initializers.Zeros()) self.lnorm = tf.keras.layers.LayerNormalization()
def get_model(X_train, y_train, n_seq=1, units_1=100, units_2=40, epochs=100, verbose=0, x_val=None, y_val=None): model = keras.Sequential([ keras.layers.LSTM( units=units_1, return_sequences=True, input_shape=(X_train.shape[1], 1), kernel_initializer=initializers.RandomNormal(stddev=0.01), bias_initializer=initializers.Zeros()), keras.layers.Dropout(0.3), keras.layers.LSTM( units=units_1, return_sequences=True, kernel_initializer=initializers.RandomNormal(stddev=0.01), bias_initializer=initializers.Zeros()), keras.layers.Dropout(0.2), keras.layers.LSTM( units=units_1, return_sequences=True, kernel_initializer=initializers.RandomNormal(stddev=0.01), bias_initializer=initializers.Zeros()), keras.layers.Dropout(0.2), keras.layers.LSTM( units=units_2, kernel_initializer=initializers.RandomNormal(stddev=0.01), bias_initializer=initializers.Zeros()), keras.layers.Dropout(0.2), keras.layers.Dense( units=n_seq, kernel_initializer=initializers.RandomNormal(stddev=0.01), bias_initializer=initializers.Zeros()) ]) model.compile(optimizer='adam', loss='mean_squared_error') if x_val != None and y_val != None: model.fit(X_train, y_train, epochs=epochs, batch_size=32, validation_data=(x_val, y_val), verbose=verbose) elif x_val == None and y_val == None: model.fit(X_train, y_train, epochs=epochs, batch_size=32, verbose=verbose) return model
def build_model( hparams ): if hparams['word_embedding'] == 'w2v': input_layer = Input(shape=(hparams['max_sequence_length'], )) embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=hparams['train_embeddings'])(input_layer) if hparams['word_embedding'] == 'elmo': input_layer = Input(shape=(hparams['max_sequence_length'], 1024, )) embedding_layer = input_layer submodels = [] kernel_sizes = hparams['kernel_sizes'].split('-') for ks in kernel_sizes: model = Sequential() conv_layer = Conv1D( activation = 'relu', filters = hparams['filters'], kernel_size = int(ks), kernel_constraint = max_norm(hparams['max_norm_value']) )(embedding_layer) max_pooling = GlobalMaxPooling1D()(conv_layer) submodels.append(max_pooling) concat = Concatenate()(submodels) dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat) hidden_layer = Dense( hparams['hidden_size'], activation = 'relu', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(len(kernel_sizes) * hparams['filters']), maxval = 1 / np.sqrt(len(kernel_sizes) * hparams['filters']) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_1) dropout_layer_2 = Dropout(hparams['dropout_ratio'])(concat) output_layer = Dense( 2, activation = 'sigmoid', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(hparams['hidden_size']), maxval = 1 / np.sqrt(hparams['hidden_size']) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_2) model = Model(inputs=[input_layer], outputs=[output_layer]) model.compile( loss = metric.dice_loss, optimizer = Adam(learning_rate = hparams['learning_rate']), metrics = [f1_score] ) return model
def __init__(self, c_initializer=initializers.Zeros(), b_initializer=initializers.Zeros(), c_trainable=True, b_trainable=True, **kwargs): super(RadialSoftmax, self).__init__(**kwargs) self.c_initializer = c_initializer self.b_initializer = b_initializer self.supports_masking = True self.c_trainable = c_trainable self.b_trainable = b_trainable
def build(self,input_shape): self.d_model = input_shape[-1] self.multi_head_attention = MultiHeadAttention(self.nb_proj) self.dropout_1 = keras.layers.Dropout(rate=self.dropout_rate) self.norm_1 = keras.layers.LayerNormalization(epsilon=1e-6) self.dense_1 = keras.layers.Dense(units=self.FFN_units,activation="relu", kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros()) self.dense_2 = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros()) self.dropout_2 = keras.layers.Dropout(rate=self.dropout_rate) self.norm_2 = keras.layers.LayerNormalization(epsilon=1e-6)
def build_model(hparams): if hparams['word_embedding'] == 'w2v': input_layer = Input(shape=(hparams['max_sequence_length'], )) embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=hparams['train_embeddings'])(input_layer) if hparams['word_embedding'] == 'elmo': input_layer = Input(shape=(hparams['max_sequence_length'], 1024, )) embedding_layer = input_layer if hparams["word_embedding"] == "random": input_layer = Input(shape=(hparams['max_sequence_length'], )) embedding_layer = Embedding( hparams["dictionary_len"] + 2, hparams["embedding_size"], input_length = hparams["max_sequence_length"], embeddings_initializer = initializers.RandomNormal( mean=0., stddev = 2 / hparams["max_sequence_length"] ) )(input_layer) flatten_layer = Flatten()(embedding_layer) dropout_layer_1 = Dropout(hparams["dropout_ratio"])(flatten_layer) hidden_layer = Dense( hparams["hidden_size"], activation = 'relu', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(hparams["embedding_size"] * hparams["max_sequence_length"]), maxval = 1 / np.sqrt(hparams["embedding_size"] * hparams["max_sequence_length"]) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_1) dropout_layer_2 = Dropout(hparams["dropout_ratio"])(hidden_layer) output_layer = Dense( 2, activation = 'sigmoid', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(hparams["hidden_size"]), maxval = 1 / np.sqrt(hparams["hidden_size"]) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_2) model = Model(inputs=[input_layer], outputs=[output_layer]) model.compile( loss = metric.dice_loss, optimizer = Adam(learning_rate = hparams["learning_rate"]), metrics = [f1_score] ) return model
def build(self, input_shape): self.d_model = input_shape[-1] assert self.d_model % self.nb_proj == 0 self.d_proj = self.d_model // self.nb_proj self.query_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros()) self.key_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros()) self.value_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros()) self.final_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros())
def __init__(self, vocab_size_enc, vocab_size_dec, d_model, nb_layers, FFN_units, nb_proj, dropout_rate, name="transformer"): super(Transformer, self).__init__(name=name) self.encoder = Encoder(nb_layers, FFN_units, nb_proj, dropout_rate, vocab_size_enc, d_model) self.decoder = Decoder(nb_layers, FFN_units, nb_proj, dropout_rate, vocab_size_dec, d_model) self.last_linear = layers.Dense(units=vocab_size_dec, activation="softmax",name="lin_ouput", kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3), bias_initializer=initializers.Zeros())
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = { "name": "nus_wide_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "nus_wide_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param(with_label=True, output_format="dense") data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False) hetero_ftl_0 = HeteroFTL(name='hetero_ftl_0', epochs=10, alpha=1, batch_size=-1, mode='plain', communication_efficient=True, local_round=5) hetero_ftl_0.add_nn_layer( Dense(units=32, activation='sigmoid', kernel_initializer=initializers.RandomNormal(stddev=1.0), bias_initializer=initializers.Zeros())) hetero_ftl_0.compile(optimizer=optimizers.Adam(lr=0.01)) evaluation_0 = Evaluation(name='evaluation_0', eval_type="binary") pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(hetero_ftl_0, data=Data(train_data=data_transform_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_ftl_0.output.data)) pipeline.compile() pipeline.fit()
def build_model( hparams): bert_model = TFAutoModel.from_pretrained(hparams["bert_file_name"]) bert_model.trainable = True if not hparams['trainable_bert'] is None: bert_model.trainable = hparams['trainable_bert'] input_layer_ids = Input(shape = (hparams['max_sequence_length'],), dtype='int64') input_layer_masks = Input(shape = (hparams['max_sequence_length'],), dtype='int64') bert_output = bert_model([input_layer_ids,input_layer_masks]) bert_output = bert_output[1] classifier = Dense(units = 2, activation = 'sigmoid', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(bert_output.shape[1]), maxval = 1 / np.sqrt(bert_output.shape[1]) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(bert_output) model = Model(inputs=[input_layer_ids,input_layer_masks], outputs=classifier) model.compile( loss= dice_loss, optimizer = Adam(learning_rate = hparams["learning_rate"]), metrics = [f1_score] ) plot_model(model, "model_bert.png", show_layer_names=False) return model
def __init__(self, input_shape, architectures=[[1, 10, 1]], weight_clippings=[None], activation='relu', degree=1, initial_model=[None], init_null=[False], train=[True], name="ParNet", **kwargs): super().__init__(name=name, **kwargs) if len(architectures) != degree or len( weight_clippings) != degree or len( initial_model) != degree or len( init_null) != degree or len(train) != degree: logging.error( "Wrong argument legth! Check that 'architectures', 'weight_clippings', 'initial_model', 'init_null', 'train' have all legth equal to %i" % (degree)) self.a = [] for i in range(degree): initializer = None if init_null[i]: initializer = initializers.Zeros() self.a.append( BSMfinder(input_shape, architectures[i], weight_clippings[i], activation=activation, trainable=train[i], initializer=initializer)) if not (initial_model[i] == None or initial_model[i] == False): self.a[-1].load_weights(initial_model[i], by_name=True) self.degree = degree self.build(input_shape)
def __init__(self, BERT_Model, vocab_size_dec, d_model, nb_decoders, FFN_units, nb_proj, dropout_rate, name="Translator"): super(MyBertTranslator, self).__init__(name=name) self.nb_decoders = nb_decoders self.d_model = d_model self.nb_decoders = nb_decoders self.embedding = layers.Embedding( vocab_size_dec, self.d_model, embeddings_initializer=initializers.RandomNormal(stddev=0.01, seed=3)) self.pos_encoding = PositionalEncoding() self.dropout = layers.Dropout(rate=dropout_rate) self.BERT_Model = BERT_Model self.dec_layers = [ DecoderLayer(FFN_units, nb_proj, dropout_rate) for i in range(nb_decoders) ] self.last_linear = layers.Dense( units=vocab_size_dec, name="lin_output", kernel_initializer=initializers.RandomNormal(stddev=0.01, seed=3), bias_initializer=initializers.Zeros())
class MLP(layers.Layer): """ MLP as used in Vision Transformer, MLP-Mixer and related networks """ k_ini = initializers.TruncatedNormal(stddev=0.02) b_ini = initializers.Zeros() def __init__(self, in_features, mlp_ratio=4.0, drop=0., name=None): super(MLP, self).__init__(name=name) self.fc1 = layers.Dense(int(in_features * mlp_ratio), name="fc1", kernel_initializer=self.k_ini, bias_initializer=self.b_ini) self.act = layers.Activation("gelu") self.fc2 = layers.Dense(in_features, name="fc2", kernel_initializer=self.k_ini, bias_initializer=self.b_ini) self.drop = layers.Dropout(drop) def call(self, x, training=None): x = self.fc1(x) x = self.act(x) x = self.drop(x, training=training) x = self.fc2(x) x = self.drop(x, training=training) return x
def build_model(hparams): input_layer_dynamic = Input(shape=(hparams['max_sequence_length'],), name='w2v_input') input_layer_static = Input(shape=(hparams['max_sequence_length'],hparams['embedding_size']),name='ELMo_input') embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=True)(input_layer_dynamic) submodels = [] submodels.extend(build_submodels(hparams['kernel_sizes'],hparams['filters'], hparams['max_norm_value'],embedding_layer)) submodels.extend(build_submodels(hparams['kernel_sizes'],hparams['filters'], hparams['max_norm_value'],input_layer_static)) concat = Concatenate()(submodels) dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat) hidden_layer = Dense( hparams['hidden_size'], activation = 'relu', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(2 * len(hparams['kernel_sizes'])*hparams['filters']), maxval = 1 / np.sqrt(2 * len(hparams['kernel_sizes'])*hparams['filters'] ) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_1) dropout_layer_2 = Dropout(hparams['dropout_ratio'])(hidden_layer) output_layer = Dense( 2, activation = 'sigmoid', kernel_initializer = initializers.RandomUniform( minval = - 1 / np.sqrt(hparams['hidden_size']), maxval = 1 / np.sqrt(hparams['hidden_size']) ), bias_initializer = initializers.Zeros(), kernel_regularizer = regularizers.l2(hparams['l2_regularization']) )(dropout_layer_2) model = Model(inputs=[input_layer_dynamic,input_layer_static], outputs=output_layer) model.compile( loss = metric.dice_loss, optimizer = Adam(learning_rate = hparams['learning_rate']), metrics = [f1_score] ) #model.summary() return model
def build(self, input_shape): print('input_shape={}'.format(input_shape)) self.kernel = self.add_weight( shape=(1, *input_shape[1:-1], 1), initializer=initializers.Zeros(), name="kernel", trainable=True, )
def create_model(): model = models.Sequential() model.add(tf.keras.Input(shape=(28, 28, 1))) model.add( Conv2D(6, (5, 5), padding='valid', activation='sigmoid', bias_initializer=initializers.Zeros())) model.add(AveragePooling2D(pool_size=(2, 2))) model.add( Conv2D(12, (5, 5), padding='valid', activation='sigmoid', bias_initializer=initializers.Zeros())) model.add(AveragePooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(layers.Dense(10, bias_initializer=initializers.Zeros())) return model
def __init__(self, patch_size=4, num_classes=1000, embed_dim=96, depths=(2, 2, 6, 2), num_heads=(3, 6, 12, 24), window_size=7, mlp_ratio=4., qkv_bias=True, drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, norm_layer=layers.LayerNormalization, name=None, **kwargs): super().__init__(name=name) self.num_classes = num_classes self.num_layers = len(depths) self.embed_dim = embed_dim self.mlp_ratio = mlp_ratio # split image into non-overlapping patches self.patch_embed = PatchEmbed(patch_size=patch_size, embed_dim=embed_dim, norm_layer=norm_layer) self.pos_drop = layers.Dropout(drop_rate) # stochastic depth decay rule dpr = [x for x in np.linspace(0, drop_path_rate, sum(depths))] # build layers self.stage_layers = [] for i_layer in range(self.num_layers): # 注意这里构建的stage和论文图中有些差异 # 这里的stage不包含该stage的patch_merging层,包含的是下个stage的 layer = BasicLayer( dim=int(embed_dim * 2**i_layer), depth=depths[i_layer], num_heads=num_heads[i_layer], window_size=window_size, mlp_ratio=self.mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, name=f"layer{i_layer}") self.stage_layers.append(layer) self.norm = norm_layer(epsilon=1e-6, name="norm") self.head = layers.Dense( num_classes, kernel_initializer=initializers.TruncatedNormal(stddev=0.02), bias_initializer=initializers.Zeros(), name="head")
def build(self, input_shape): self.input_spec = [InputSpec(shape=input_shape)] output_shape = (int(input_shape[self.axis]),) gamma_initializer = initializers.Ones() beta_initializer = initializers.Zeros() self.gamma = K.variable(gamma_initializer(output_shape)) self.beta = K.variable(beta_initializer(output_shape)) self.trainable_weights = [self.gamma, self.beta]
def build(self, input_shape): self.gain = self.add_weight(name='gain', shape=input_shape[-1:], initializer=initializers.Ones(), trainable=True) self.bias = self.add_weight(name='bias', shape=input_shape[-1:], initializer=initializers.Zeros(), trainable=True) super().build(input_shape)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "nus_wide_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "nus_wide_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).algorithm_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).algorithm_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).algorithm_param(with_label=True, output_format="dense") dataio_0.get_party_instance(role='host', party_id=host).algorithm_param(with_label=False) hetero_ftl_0 = HeteroFTL(name='hetero_ftl_0', epochs=10, alpha=1, batch_size=-1, mode='plain') hetero_ftl_0.add_nn_layer(Dense(units=32, activation='sigmoid', kernel_initializer=initializers.RandomNormal(stddev=1.0, dtype="float32"), bias_initializer=initializers.Zeros())) hetero_ftl_0.compile(optimizer=optimizers.Adam(lr=0.01)) evaluation_0 = Evaluation(name='evaluation_0', eval_type="binary") pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(hetero_ftl_0, data=Data(train_data=dataio_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_ftl_0.output.data)) pipeline.compile() pipeline.fit(backend=backend, work_mode=work_mode) # predict # deploy required components pipeline.deploy_component([dataio_0, hetero_ftl_0]) predict_pipeline = PipeLine() # add data reader onto predict pipeline predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source predict_pipeline.add_component(pipeline, data=Data(predict_input={pipeline.dataio_0.input.data: reader_0.output.data})) # run predict model predict_pipeline.predict(backend=backend, work_mode=work_mode)
def idea1_architecture(embedding_size, model_params): initializer = tf.keras.initializers.RandomNormal(seed=1) texts_pos = Input(shape=(embedding_size, )) texts_neg = Input(shape=(embedding_size, )) sigmoid = Dense(1, activation='sigmoid', bias_initializer=initializers.Zeros(), kernel_initializer=initializer) n_out = texts_neg p_out = texts_pos drop = Dropout(model_params['dropout_first']) n_out = drop(n_out) p_out = drop(p_out) for layer_size in model_params['layers']: drop = Dropout(model_params['dropout']) if model_params['reg'] != 0: layer = Dense(layer_size, activation=tf.nn.relu, bias_initializer=initializers.Zeros(), kernel_initializer=initializer, kernel_regularizer=regularizers.l2( model_params['reg'])) else: layer = Dense(layer_size, bias_initializer=initializers.Zeros(), kernel_initializer=initializer, activation=tf.nn.relu) n_out = drop(layer(n_out)) p_out = drop(layer(p_out)) y_pos = sigmoid(p_out) y_neg = sigmoid(n_out) model = Model(inputs=[texts_pos, texts_neg], outputs=[y_pos, y_neg]) loss = custom_loss(y_pos, y_neg, model_params) # Add loss to model model.add_loss(loss) return model
def begin_insert_layer(self, layer_dim): # `self.layers[0].get_weights()` -> [weights, bias] next_units = self.layers[0].get_weights()[0].shape[0] layer = Dense( units=next_units, activation=tf.nn.relu, kernel_regularizer=regularizers.l1_l2(l1=self.l1, l2=self.l2), kernel_initializer=initializers.GlorotNormal(seed=self.seed), bias_initializer=initializers.Zeros()) layer.build(input_shape=(None, layer_dim)) self.layers.insert(0, layer)
def _create_bottom_mlp_padding(self, multiple=8): num_features = self.dataset_metadata.num_numerical_features pad_to = tf.math.ceil(num_features / multiple) * multiple pad_to = tf.cast(pad_to, dtype=tf.int32) padding_features = pad_to - num_features batch_size = self.batch_size // hvd.size() if self.data_parallel_bottom_mlp else self.batch_size padding_shape = [batch_size, padding_features] dtype=tf.float16 if self.amp else tf.float32 self.bottom_mlp_padding = self.add_weight("bottom_mlp_padding", shape=padding_shape, dtype=dtype, initializer=initializers.Zeros(), trainable=False)
def create_model(): model = models.Sequential() model.add(layers.Flatten()) model.add( layers.Dense(latent_dim, activation='relu', kernel_initializer=initializers.RandomNormal(stddev=0.01), bias_initializer=initializers.Zeros())) model.add(layers.Dense(784, activation='sigmoid')) model.add(layers.Reshape((28, 28))) model.compile(optimizer='adam', loss=losses.MeanSquaredError(), metrics=['accuracy']) return model
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', w_init=initizers.glorot_uniform(seed=None), activation='linear', b_init=initizers.Zeros(), use_bias=True, weights=None, **kwargs): super(Complex_Conv2D, self).__init__(**kwargs) self.filters = filters self.kernel_size = kernel_size self.padding = padding self.strides = strides self.w_init = w_init self.b_init = b_init self.use_bias = use_bias self.activation = activation self.w = weights if self.w is not None: self.real_layer = tf.keras.layers.Conv2D(filters, kernel_size, padding=padding, use_bias=False, strides=strides, weights=self.w[:, :, :, :, 0]) self.imag_layer = tf.keras.layers.Conv2D(filters, kernel_size, padding=padding, use_bias=False, strides=strides, weights=self.w[:, :, :, :, 1]) else: self.real_layer = tf.keras.layers.Conv2D(filters, kernel_size, padding=padding, use_bias=False, strides=strides, kernel_initializer=w_init) self.imag_layer = tf.keras.layers.Conv2D(filters, kernel_size, padding=padding, use_bias=False, strides=strides, kernel_initializer=w_init)