def build(self, input_shape):
self.d_model = input_shape[-1]
# Self multi head attention
self.multi_head_attention_1 = MultiHeadAttention(self.nb_proj)
self.dropout_1 = layers.Dropout(rate=self.dropout_rate)
self.norm_1 = layers.LayerNormalization(epsilon=1e-6)
# Multi head attention combined with encoder output
self.multi_head_attention_2 = MultiHeadAttention(self.nb_proj)
self.dropout_2 = layers.Dropout(rate=self.dropout_rate)
self.norm_2 = layers.LayerNormalization(epsilon=1e-6)
# Feed foward
self.dense_1 = layers.Dense(
units=self.FFN_units,
activation="relu",
kernel_initializer=initializers.RandomNormal(stddev=0.01, seed=3),
bias_initializer=initializers.Zeros())
self.dense_2 = layers.Dense(
units=self.d_model,
kernel_initializer=initializers.RandomNormal(stddev=0.01, seed=3),
bias_initializer=initializers.Zeros())
self.dropout_3 = layers.Dropout(rate=self.dropout_rate)
self.norm_3 = layers.LayerNormalization(epsilon=1e-6)
def MyNet_tf(input_shape: Tuple[int, int, int] = (28, 28, 1), classes: int = 10) -> tf.keras.Model:
"""A standard DNN implementation in TensorFlow.
The MyNet model has 3 dense layers.
Args:
input_shape: shape of the input data (height, width, channels).
classes: The number of outputs the model should generate.
Returns:
A TensorFlow MyNet model.
"""
#_check_input_shape(input_shape)
model = Sequential()
model.add(layers.Flatten(input_shape=(28, 28)))
model.add(
layers.Dense(units=300,
kernel_initializer=initializers.Constant(0.01),
bias_initializer=initializers.Zeros(),
activation='relu'))
model.add(
layers.Dense(units=64,
kernel_initializer=initializers.Constant(0.01),
bias_initializer=initializers.Zeros(),
activation='relu'))
model.add(
layers.Dense(units=classes,
kernel_initializer=initializers.Constant(0.01),
bias_initializer=initializers.Zeros(),
activation='softmax'))
return model
def build_model(
self,
input_shape,
number_of_layers=5,
neurons_by_layers=10,
bias_mu_throttle=0.5,
bias_mu_steering=0.0,
bias_sigma_throttle=0.03, # ~1% outside (0.5 +- 0.4)
bias_sigma_steering=0.1, # ~2% outside (0 +- 0.9)
dropout=0.15):
"""
Construct the actor network with mu and sigma as output
"""
print(f"Input shape of policy: {input_shape}")
inputs = layers.Input(shape=input_shape)
prev_layer = inputs
for i in range(number_of_layers):
current_layer = layers.Dense(
neurons_by_layers,
activation="relu",
kernel_initializer=initializers.he_normal())(prev_layer)
if i < number_of_layers - 1:
prev_layer = current_layer
layers.Dropout(dropout)(prev_layer)
prev_layer = current_layer
current_layer = layers.Flatten()(prev_layer)
mu_throttle = layers.Dense(1,
activation="linear",
kernel_initializer=initializers.Zeros(),
bias_initializer=initializers.Constant(
bias_mu_throttle))(current_layer)
sigma_throttle = layers.Dense(1,
activation="softplus",
kernel_initializer=initializers.Zeros(),
bias_initializer=initializers.Constant(
bias_sigma_throttle))(current_layer)
mu_steering = layers.Dense(1,
activation="linear",
kernel_initializer=initializers.Zeros(),
bias_initializer=initializers.Constant(
bias_mu_steering))(current_layer)
sigma_steering = layers.Dense(1,
activation="softplus",
kernel_initializer=initializers.Zeros(),
bias_initializer=initializers.Constant(
bias_sigma_steering))(current_layer)
actor_network = keras.Model(
inputs=inputs,
outputs=[mu_throttle, sigma_throttle, mu_steering, sigma_steering])
return actor_network
def build(self, input_shape):
if self.my_dtype.is_complex:
i_kernel_dtype = self.my_dtype if isinstance(
self.kernel_initializer,
ComplexInitializer) else self.my_dtype.real_dtype
i_bias_dtype = self.my_dtype if isinstance(
self.bias_initializer,
ComplexInitializer) else self.my_dtype.real_dtype
i_kernel_initializer = self.kernel_initializer
i_bias_initializer = self.bias_initializer
if not isinstance(self.kernel_initializer, ComplexInitializer):
tf.print(
f"WARNING: you are using a Tensorflow Initializer for complex numbers. "
f"Using {self.init_technique} method.")
if self.init_technique in INIT_TECHNIQUES:
if self.init_technique == 'zero_imag':
# This section is done to initialize with tf initializers, making imaginary part zero
i_kernel_initializer = initializers.Zeros()
i_bias_initializer = initializers.Zeros()
else:
raise ValueError(
f"Unsuported init_technique {self.init_technique}, "
f"supported techniques are {INIT_TECHNIQUES}")
self.w_r = tf.Variable(name='kernel_r',
initial_value=self.kernel_initializer(
shape=(input_shape[-1], self.units),
dtype=i_kernel_dtype),
trainable=True)
self.w_i = tf.Variable(name='kernel_i',
initial_value=i_kernel_initializer(
shape=(input_shape[-1], self.units),
dtype=i_kernel_dtype),
trainable=True)
if self.use_bias:
self.b_r = tf.Variable(name='bias_r',
initial_value=self.bias_initializer(
shape=(self.units, ),
dtype=i_bias_dtype),
trainable=self.use_bias)
self.b_i = tf.Variable(name='bias_i',
initial_value=i_bias_initializer(
shape=(self.units, ),
dtype=i_bias_dtype),
trainable=self.use_bias)
else:
# TODO: For Complex you should probably want to use MY init for real keras. DO sth! at least error message
self.w = self.add_weight(
'kernel',
shape=(input_shape[-1], self.units),
dtype=self.my_dtype,
initializer=self.kernel_initializer,
trainable=True,
)
if self.use_bias:
self.b = self.add_weight('bias',
shape=(self.units, ),
dtype=self.my_dtype,
initializer=self.bias_initializer,
trainable=self.use_bias)
def build_model(
hparams
):
input_layer = Input(shape=(hparams["max_sequence_length"], ))
embedding_layer_static = get_w2v('').get_keras_embedding(train_embeddings=False)(input_layer)
embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=True)(input_layer)
submodels = []
kernel_sizes = hparams['kernel_sizes'].split('-')
for ks in kernel_sizes:
model = Sequential()
conv_1_d = Conv1D(
activation = 'relu',
filters = hparams["filters"],
kernel_size = int(ks),
kernel_constraint = max_norm(hparams["max_norm_value"])
)
conv_layer_static = conv_1_d(embedding_layer_static)
conv_layer = conv_1_d(embedding_layer)
max_pooling_static = GlobalMaxPooling1D()(conv_layer_static)
max_pooling = GlobalMaxPooling1D()(conv_layer)
concatenate_layer = Concatenate()([max_pooling_static, max_pooling])
submodels.append(concatenate_layer)
concat = Concatenate()(submodels)
dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat)
hidden_layer = Dense(
hparams['hidden_size'],
activation = 'relu',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(len(kernel_sizes) * 2* hparams['filters']),
maxval = 1 / np.sqrt(len(kernel_sizes) * 2 * hparams['filters'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_1)
dropout_layer_2 = Dropout(hparams['dropout_ratio'])(hidden_layer)
output_layer = Dense(
2,
activation = 'sigmoid',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(hparams['hidden_size']),
maxval = 1 / np.sqrt(hparams['hidden_size'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_2)
model = Model(inputs=[input_layer], outputs=[output_layer])
model.compile(
loss = dice_loss,
optimizer = Adam(learning_rate = hparams["learning_rate"]),
metrics = [f1_score]
)
from keras.utils.vis_utils import plot_model
plot_model(model, "model_cnn_multichannel.png", show_layer_names=False)
return model
def __init__(self, num_channels):
super().__init__()
self.W1 = tf.keras.layers.Dense(units = num_channels,
kernel_initializer=initializers.RandomNormal(stddev=1/num_channels**2),
bias_initializer=initializers.Zeros())
self.W2 = tf.keras.layers.Dense(units = num_channels,
kernel_initializer=initializers.RandomNormal(stddev=1/num_channels**2),
bias_initializer=initializers.Zeros())
self.lnorm = tf.keras.layers.LayerNormalization()
def get_model(X_train,
y_train,
n_seq=1,
units_1=100,
units_2=40,
epochs=100,
verbose=0,
x_val=None,
y_val=None):
model = keras.Sequential([
keras.layers.LSTM(
units=units_1,
return_sequences=True,
input_shape=(X_train.shape[1], 1),
kernel_initializer=initializers.RandomNormal(stddev=0.01),
bias_initializer=initializers.Zeros()),
keras.layers.Dropout(0.3),
keras.layers.LSTM(
units=units_1,
return_sequences=True,
kernel_initializer=initializers.RandomNormal(stddev=0.01),
bias_initializer=initializers.Zeros()),
keras.layers.Dropout(0.2),
keras.layers.LSTM(
units=units_1,
return_sequences=True,
kernel_initializer=initializers.RandomNormal(stddev=0.01),
bias_initializer=initializers.Zeros()),
keras.layers.Dropout(0.2),
keras.layers.LSTM(
units=units_2,
kernel_initializer=initializers.RandomNormal(stddev=0.01),
bias_initializer=initializers.Zeros()),
keras.layers.Dropout(0.2),
keras.layers.Dense(
units=n_seq,
kernel_initializer=initializers.RandomNormal(stddev=0.01),
bias_initializer=initializers.Zeros())
])
model.compile(optimizer='adam', loss='mean_squared_error')
if x_val != None and y_val != None:
model.fit(X_train,
y_train,
epochs=epochs,
batch_size=32,
validation_data=(x_val, y_val),
verbose=verbose)
elif x_val == None and y_val == None:
model.fit(X_train,
y_train,
epochs=epochs,
batch_size=32,
verbose=verbose)
return model
def build_model(
hparams
):
if hparams['word_embedding'] == 'w2v':
input_layer = Input(shape=(hparams['max_sequence_length'], ))
embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=hparams['train_embeddings'])(input_layer)
if hparams['word_embedding'] == 'elmo':
input_layer = Input(shape=(hparams['max_sequence_length'], 1024, ))
embedding_layer = input_layer
submodels = []
kernel_sizes = hparams['kernel_sizes'].split('-')
for ks in kernel_sizes:
model = Sequential()
conv_layer = Conv1D(
activation = 'relu',
filters = hparams['filters'],
kernel_size = int(ks),
kernel_constraint = max_norm(hparams['max_norm_value'])
)(embedding_layer)
max_pooling = GlobalMaxPooling1D()(conv_layer)
submodels.append(max_pooling)
concat = Concatenate()(submodels)
dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat)
hidden_layer = Dense(
hparams['hidden_size'],
activation = 'relu',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(len(kernel_sizes) * hparams['filters']),
maxval = 1 / np.sqrt(len(kernel_sizes) * hparams['filters'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_1)
dropout_layer_2 = Dropout(hparams['dropout_ratio'])(concat)
output_layer = Dense(
2,
activation = 'sigmoid',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(hparams['hidden_size']),
maxval = 1 / np.sqrt(hparams['hidden_size'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_2)
model = Model(inputs=[input_layer], outputs=[output_layer])
model.compile(
loss = metric.dice_loss,
optimizer = Adam(learning_rate = hparams['learning_rate']),
metrics = [f1_score]
)
return model
def __init__(self,
c_initializer=initializers.Zeros(),
b_initializer=initializers.Zeros(),
c_trainable=True,
b_trainable=True,
**kwargs):
super(RadialSoftmax, self).__init__(**kwargs)
self.c_initializer = c_initializer
self.b_initializer = b_initializer
self.supports_masking = True
self.c_trainable = c_trainable
self.b_trainable = b_trainable
def build(self,input_shape):
self.d_model = input_shape[-1]
self.multi_head_attention = MultiHeadAttention(self.nb_proj)
self.dropout_1 = keras.layers.Dropout(rate=self.dropout_rate)
self.norm_1 = keras.layers.LayerNormalization(epsilon=1e-6)
self.dense_1 = keras.layers.Dense(units=self.FFN_units,activation="relu",
kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
self.dense_2 = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
self.dropout_2 = keras.layers.Dropout(rate=self.dropout_rate)
self.norm_2 = keras.layers.LayerNormalization(epsilon=1e-6)
def build_model(hparams):
if hparams['word_embedding'] == 'w2v':
input_layer = Input(shape=(hparams['max_sequence_length'], ))
embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=hparams['train_embeddings'])(input_layer)
if hparams['word_embedding'] == 'elmo':
input_layer = Input(shape=(hparams['max_sequence_length'], 1024, ))
embedding_layer = input_layer
if hparams["word_embedding"] == "random":
input_layer = Input(shape=(hparams['max_sequence_length'], ))
embedding_layer = Embedding(
hparams["dictionary_len"] + 2,
hparams["embedding_size"],
input_length = hparams["max_sequence_length"],
embeddings_initializer = initializers.RandomNormal(
mean=0.,
stddev = 2 / hparams["max_sequence_length"]
)
)(input_layer)
flatten_layer = Flatten()(embedding_layer)
dropout_layer_1 = Dropout(hparams["dropout_ratio"])(flatten_layer)
hidden_layer = Dense(
hparams["hidden_size"],
activation = 'relu',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(hparams["embedding_size"] * hparams["max_sequence_length"]),
maxval = 1 / np.sqrt(hparams["embedding_size"] * hparams["max_sequence_length"])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_1)
dropout_layer_2 = Dropout(hparams["dropout_ratio"])(hidden_layer)
output_layer = Dense(
2,
activation = 'sigmoid',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(hparams["hidden_size"]),
maxval = 1 / np.sqrt(hparams["hidden_size"])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_2)
model = Model(inputs=[input_layer], outputs=[output_layer])
model.compile(
loss = metric.dice_loss,
optimizer = Adam(learning_rate = hparams["learning_rate"]),
metrics = [f1_score]
)
return model
def build(self, input_shape):
self.d_model = input_shape[-1]
assert self.d_model % self.nb_proj == 0
self.d_proj = self.d_model // self.nb_proj
self.query_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
self.key_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
self.value_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
self.final_lin = keras.layers.Dense(units=self.d_model,kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
def __init__(self,
vocab_size_enc,
vocab_size_dec,
d_model,
nb_layers,
FFN_units,
nb_proj,
dropout_rate,
name="transformer"):
super(Transformer, self).__init__(name=name)
self.encoder = Encoder(nb_layers,
FFN_units,
nb_proj,
dropout_rate,
vocab_size_enc,
d_model)
self.decoder = Decoder(nb_layers,
FFN_units,
nb_proj,
dropout_rate,
vocab_size_dec,
d_model)
self.last_linear = layers.Dense(units=vocab_size_dec, activation="softmax",name="lin_ouput",
kernel_initializer=initializers.RandomNormal(stddev=0.01,seed=3),
bias_initializer=initializers.Zeros())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "nus_wide_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "nus_wide_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
hetero_ftl_0 = HeteroFTL(name='hetero_ftl_0',
epochs=10,
alpha=1,
batch_size=-1,
mode='plain',
communication_efficient=True,
local_round=5)
hetero_ftl_0.add_nn_layer(
Dense(units=32,
activation='sigmoid',
kernel_initializer=initializers.RandomNormal(stddev=1.0),
bias_initializer=initializers.Zeros()))
hetero_ftl_0.compile(optimizer=optimizers.Adam(lr=0.01))
evaluation_0 = Evaluation(name='evaluation_0', eval_type="binary")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(hetero_ftl_0,
data=Data(train_data=data_transform_0.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_ftl_0.output.data))
pipeline.compile()
pipeline.fit()
def build_model(
hparams):
bert_model = TFAutoModel.from_pretrained(hparams["bert_file_name"])
bert_model.trainable = True
if not hparams['trainable_bert'] is None:
bert_model.trainable = hparams['trainable_bert']
input_layer_ids = Input(shape = (hparams['max_sequence_length'],), dtype='int64')
input_layer_masks = Input(shape = (hparams['max_sequence_length'],), dtype='int64')
bert_output = bert_model([input_layer_ids,input_layer_masks])
bert_output = bert_output[1]
classifier = Dense(units = 2,
activation = 'sigmoid',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(bert_output.shape[1]),
maxval = 1 / np.sqrt(bert_output.shape[1])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(bert_output)
model = Model(inputs=[input_layer_ids,input_layer_masks], outputs=classifier)
model.compile(
loss= dice_loss,
optimizer = Adam(learning_rate = hparams["learning_rate"]),
metrics = [f1_score]
)
plot_model(model, "model_bert.png", show_layer_names=False)
return model
def __init__(self,
input_shape,
architectures=[[1, 10, 1]],
weight_clippings=[None],
activation='relu',
degree=1,
initial_model=[None],
init_null=[False],
train=[True],
name="ParNet",
**kwargs):
super().__init__(name=name, **kwargs)
if len(architectures) != degree or len(
weight_clippings) != degree or len(
initial_model) != degree or len(
init_null) != degree or len(train) != degree:
logging.error(
"Wrong argument legth! Check that 'architectures', 'weight_clippings', 'initial_model', 'init_null', 'train' have all legth equal to %i"
% (degree))
self.a = []
for i in range(degree):
initializer = None
if init_null[i]:
initializer = initializers.Zeros()
self.a.append(
BSMfinder(input_shape,
architectures[i],
weight_clippings[i],
activation=activation,
trainable=train[i],
initializer=initializer))
if not (initial_model[i] == None or initial_model[i] == False):
self.a[-1].load_weights(initial_model[i], by_name=True)
self.degree = degree
self.build(input_shape)
def __init__(self,
BERT_Model,
vocab_size_dec,
d_model,
nb_decoders,
FFN_units,
nb_proj,
dropout_rate,
name="Translator"):
super(MyBertTranslator, self).__init__(name=name)
self.nb_decoders = nb_decoders
self.d_model = d_model
self.nb_decoders = nb_decoders
self.embedding = layers.Embedding(
vocab_size_dec,
self.d_model,
embeddings_initializer=initializers.RandomNormal(stddev=0.01,
seed=3))
self.pos_encoding = PositionalEncoding()
self.dropout = layers.Dropout(rate=dropout_rate)
self.BERT_Model = BERT_Model
self.dec_layers = [
DecoderLayer(FFN_units, nb_proj, dropout_rate)
for i in range(nb_decoders)
]
self.last_linear = layers.Dense(
units=vocab_size_dec,
name="lin_output",
kernel_initializer=initializers.RandomNormal(stddev=0.01, seed=3),
bias_initializer=initializers.Zeros())
class MLP(layers.Layer):
"""
MLP as used in Vision Transformer, MLP-Mixer and related networks
"""
k_ini = initializers.TruncatedNormal(stddev=0.02)
b_ini = initializers.Zeros()
def __init__(self, in_features, mlp_ratio=4.0, drop=0., name=None):
super(MLP, self).__init__(name=name)
self.fc1 = layers.Dense(int(in_features * mlp_ratio),
name="fc1",
kernel_initializer=self.k_ini,
bias_initializer=self.b_ini)
self.act = layers.Activation("gelu")
self.fc2 = layers.Dense(in_features,
name="fc2",
kernel_initializer=self.k_ini,
bias_initializer=self.b_ini)
self.drop = layers.Dropout(drop)
def call(self, x, training=None):
x = self.fc1(x)
x = self.act(x)
x = self.drop(x, training=training)
x = self.fc2(x)
x = self.drop(x, training=training)
return x
def build_model(hparams):
input_layer_dynamic = Input(shape=(hparams['max_sequence_length'],), name='w2v_input')
input_layer_static = Input(shape=(hparams['max_sequence_length'],hparams['embedding_size']),name='ELMo_input')
embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=True)(input_layer_dynamic)
submodels = []
submodels.extend(build_submodels(hparams['kernel_sizes'],hparams['filters'],
hparams['max_norm_value'],embedding_layer))
submodels.extend(build_submodels(hparams['kernel_sizes'],hparams['filters'],
hparams['max_norm_value'],input_layer_static))
concat = Concatenate()(submodels)
dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat)
hidden_layer = Dense(
hparams['hidden_size'],
activation = 'relu',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(2 * len(hparams['kernel_sizes'])*hparams['filters']),
maxval = 1 / np.sqrt(2 * len(hparams['kernel_sizes'])*hparams['filters'] )
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_1)
dropout_layer_2 = Dropout(hparams['dropout_ratio'])(hidden_layer)
output_layer = Dense(
2,
activation = 'sigmoid',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(hparams['hidden_size']),
maxval = 1 / np.sqrt(hparams['hidden_size'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_2)
model = Model(inputs=[input_layer_dynamic,input_layer_static], outputs=output_layer)
model.compile(
loss = metric.dice_loss,
optimizer = Adam(learning_rate = hparams['learning_rate']),
metrics = [f1_score]
)
#model.summary()
return model
def build(self, input_shape):
print('input_shape={}'.format(input_shape))
self.kernel = self.add_weight(
shape=(1, *input_shape[1:-1], 1),
initializer=initializers.Zeros(),
name="kernel",
trainable=True,
)
def create_model():
model = models.Sequential()
model.add(tf.keras.Input(shape=(28, 28, 1)))
model.add(
Conv2D(6, (5, 5),
padding='valid',
activation='sigmoid',
bias_initializer=initializers.Zeros()))
model.add(AveragePooling2D(pool_size=(2, 2)))
model.add(
Conv2D(12, (5, 5),
padding='valid',
activation='sigmoid',
bias_initializer=initializers.Zeros()))
model.add(AveragePooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(layers.Dense(10, bias_initializer=initializers.Zeros()))
return model
def __init__(self,
patch_size=4,
num_classes=1000,
embed_dim=96,
depths=(2, 2, 6, 2),
num_heads=(3, 6, 12, 24),
window_size=7,
mlp_ratio=4.,
qkv_bias=True,
drop_rate=0.,
attn_drop_rate=0.,
drop_path_rate=0.1,
norm_layer=layers.LayerNormalization,
name=None,
**kwargs):
super().__init__(name=name)
self.num_classes = num_classes
self.num_layers = len(depths)
self.embed_dim = embed_dim
self.mlp_ratio = mlp_ratio
# split image into non-overlapping patches
self.patch_embed = PatchEmbed(patch_size=patch_size,
embed_dim=embed_dim,
norm_layer=norm_layer)
self.pos_drop = layers.Dropout(drop_rate)
# stochastic depth decay rule
dpr = [x for x in np.linspace(0, drop_path_rate, sum(depths))]
# build layers
self.stage_layers = []
for i_layer in range(self.num_layers):
# 注意这里构建的stage和论文图中有些差异
# 这里的stage不包含该stage的patch_merging层，包含的是下个stage的
layer = BasicLayer(
dim=int(embed_dim * 2**i_layer),
depth=depths[i_layer],
num_heads=num_heads[i_layer],
window_size=window_size,
mlp_ratio=self.mlp_ratio,
qkv_bias=qkv_bias,
drop=drop_rate,
attn_drop=attn_drop_rate,
drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
downsample=PatchMerging if
(i_layer < self.num_layers - 1) else None,
name=f"layer{i_layer}")
self.stage_layers.append(layer)
self.norm = norm_layer(epsilon=1e-6, name="norm")
self.head = layers.Dense(
num_classes,
kernel_initializer=initializers.TruncatedNormal(stddev=0.02),
bias_initializer=initializers.Zeros(),
name="head")
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
output_shape = (int(input_shape[self.axis]),)
gamma_initializer = initializers.Ones()
beta_initializer = initializers.Zeros()
self.gamma = K.variable(gamma_initializer(output_shape))
self.beta = K.variable(beta_initializer(output_shape))
self.trainable_weights = [self.gamma, self.beta]
def build(self, input_shape):
self.gain = self.add_weight(name='gain',
shape=input_shape[-1:],
initializer=initializers.Ones(),
trainable=True)
self.bias = self.add_weight(name='bias',
shape=input_shape[-1:],
initializer=initializers.Zeros(),
trainable=True)
super().build(input_shape)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "nus_wide_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "nus_wide_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).algorithm_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).algorithm_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).algorithm_param(with_label=True, output_format="dense")
dataio_0.get_party_instance(role='host', party_id=host).algorithm_param(with_label=False)
hetero_ftl_0 = HeteroFTL(name='hetero_ftl_0',
epochs=10, alpha=1, batch_size=-1, mode='plain')
hetero_ftl_0.add_nn_layer(Dense(units=32, activation='sigmoid',
kernel_initializer=initializers.RandomNormal(stddev=1.0,
dtype="float32"),
bias_initializer=initializers.Zeros()))
hetero_ftl_0.compile(optimizer=optimizers.Adam(lr=0.01))
evaluation_0 = Evaluation(name='evaluation_0', eval_type="binary")
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(hetero_ftl_0, data=Data(train_data=dataio_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=hetero_ftl_0.output.data))
pipeline.compile()
pipeline.fit(backend=backend, work_mode=work_mode)
# predict
# deploy required components
pipeline.deploy_component([dataio_0, hetero_ftl_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(pipeline,
data=Data(predict_input={pipeline.dataio_0.input.data: reader_0.output.data}))
# run predict model
predict_pipeline.predict(backend=backend, work_mode=work_mode)
def idea1_architecture(embedding_size, model_params):
initializer = tf.keras.initializers.RandomNormal(seed=1)
texts_pos = Input(shape=(embedding_size, ))
texts_neg = Input(shape=(embedding_size, ))
sigmoid = Dense(1,
activation='sigmoid',
bias_initializer=initializers.Zeros(),
kernel_initializer=initializer)
n_out = texts_neg
p_out = texts_pos
drop = Dropout(model_params['dropout_first'])
n_out = drop(n_out)
p_out = drop(p_out)
for layer_size in model_params['layers']:
drop = Dropout(model_params['dropout'])
if model_params['reg'] != 0:
layer = Dense(layer_size,
activation=tf.nn.relu,
bias_initializer=initializers.Zeros(),
kernel_initializer=initializer,
kernel_regularizer=regularizers.l2(
model_params['reg']))
else:
layer = Dense(layer_size,
bias_initializer=initializers.Zeros(),
kernel_initializer=initializer,
activation=tf.nn.relu)
n_out = drop(layer(n_out))
p_out = drop(layer(p_out))
y_pos = sigmoid(p_out)
y_neg = sigmoid(n_out)
model = Model(inputs=[texts_pos, texts_neg], outputs=[y_pos, y_neg])
loss = custom_loss(y_pos, y_neg, model_params)
# Add loss to model
model.add_loss(loss)
return model
def begin_insert_layer(self, layer_dim):
# `self.layers[0].get_weights()` -> [weights, bias]
next_units = self.layers[0].get_weights()[0].shape[0]
layer = Dense(
units=next_units,
activation=tf.nn.relu,
kernel_regularizer=regularizers.l1_l2(l1=self.l1, l2=self.l2),
kernel_initializer=initializers.GlorotNormal(seed=self.seed),
bias_initializer=initializers.Zeros())
layer.build(input_shape=(None, layer_dim))
self.layers.insert(0, layer)
def _create_bottom_mlp_padding(self, multiple=8):
num_features = self.dataset_metadata.num_numerical_features
pad_to = tf.math.ceil(num_features / multiple) * multiple
pad_to = tf.cast(pad_to, dtype=tf.int32)
padding_features = pad_to - num_features
batch_size = self.batch_size // hvd.size() if self.data_parallel_bottom_mlp else self.batch_size
padding_shape = [batch_size, padding_features]
dtype=tf.float16 if self.amp else tf.float32
self.bottom_mlp_padding = self.add_weight("bottom_mlp_padding", shape=padding_shape, dtype=dtype,
initializer=initializers.Zeros(), trainable=False)
def create_model():
model = models.Sequential()
model.add(layers.Flatten())
model.add(
layers.Dense(latent_dim,
activation='relu',
kernel_initializer=initializers.RandomNormal(stddev=0.01),
bias_initializer=initializers.Zeros()))
model.add(layers.Dense(784, activation='sigmoid'))
model.add(layers.Reshape((28, 28)))
model.compile(optimizer='adam',
loss=losses.MeanSquaredError(),
metrics=['accuracy'])
return model
def __init__(self,
filters,
kernel_size,
strides=(1, 1),
padding='valid',
w_init=initizers.glorot_uniform(seed=None),
activation='linear',
b_init=initizers.Zeros(),
use_bias=True,
weights=None,
**kwargs):
super(Complex_Conv2D, self).__init__(**kwargs)
self.filters = filters
self.kernel_size = kernel_size
self.padding = padding
self.strides = strides
self.w_init = w_init
self.b_init = b_init
self.use_bias = use_bias
self.activation = activation
self.w = weights
if self.w is not None:
self.real_layer = tf.keras.layers.Conv2D(filters,
kernel_size,
padding=padding,
use_bias=False,
strides=strides,
weights=self.w[:, :, :, :,
0])
self.imag_layer = tf.keras.layers.Conv2D(filters,
kernel_size,
padding=padding,
use_bias=False,
strides=strides,
weights=self.w[:, :, :, :,
1])
else:
self.real_layer = tf.keras.layers.Conv2D(filters,
kernel_size,
padding=padding,
use_bias=False,
strides=strides,
kernel_initializer=w_init)
self.imag_layer = tf.keras.layers.Conv2D(filters,
kernel_size,
padding=padding,
use_bias=False,
strides=strides,
kernel_initializer=w_init)