def base_model():
model = Sequential()
for i in range(layers):
neuron_number = self.neuron_numbers_var[i].get()
activation = self.activation_var[i].get()
if i == 0:
model.add(
Dense(neuron_number,
activation=activation,
input_dim=X.shape[1],
kernel_initializer=GlorotUniform(seed=0)))
model.add(Dropout(0.2))
else:
model.add(
Dense(neuron_number,
activation=activation,
kernel_initializer=GlorotUniform(seed=0)))
model.add(Dropout(0.2))
model.add(
Dense(1,
activation=self.output_activation.get(),
kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer=optimizers[self.hyperparameters[2].get()],
loss=self.hyperparameters[3].get())
return model
def conv_net(nbr_classes, img_size=28):
"""Reproduce the CNN used in the MAML paper. It was originally designed in
Vinyals and al. (2016) .
Conv layers kernels are initialized with Glorot Uniform by default."""
model = Sequential()
model.add(Conv2D(64, (3, 3), strides = (1, 1), activation='relu',\
input_shape=(img_size, img_size, 3),
kernel_initializer=GlorotUniform(seed=1234)))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(
Conv2D(64, (3, 3),
strides=(1, 1),
activation='relu',
kernel_initializer=GlorotUniform(seed=1234)))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(
Conv2D(64, (3, 3),
strides=(1, 1),
activation='relu',
kernel_initializer=GlorotUniform(seed=1234)))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Flatten())
return model
def convolutional_block_v2(X, f, filters, stage, block, s=2):
print("convolutional_block_v2_{}_{}_{}_{}_{}_{}".format(
X, f, filters, stage, block, s))
conv_name_base = "res{}{}_branch".format(stage, block)
bn_name_base = "bn{}{}_branch".format(stage, block)
F1, F2, F3 = filters
X_shortcut = X
if s == 1:
X = Conv2D(filters=F1,
kernel_size=(1, 1),
strides=(s, s),
padding="valid",
name=conv_name_base + "2a",
kernel_initializer=GlorotUniform(seed=2019))(X)
else:
X = BatchNormalization(axis=3, name=bn_name_base + "2a")(X)
X = Activation("relu")(X)
X = Conv2D(filters=F1,
kernel_size=(1, 1),
strides=(s, s),
padding="valid",
name=conv_name_base + "2a",
kernel_initializer=GlorotUniform(seed=2019))(X)
X = BatchNormalization(axis=3, name=bn_name_base + "2b")(X)
X = Activation("relu")(X)
X = Conv2D(filters=F2,
kernel_size=(f, f),
strides=(1, 1),
padding="same",
name=conv_name_base + "2b",
kernel_initializer=GlorotUniform(seed=2019))(X)
X = BatchNormalization(axis=3, name=bn_name_base + "2c")(X)
X = Activation("relu")(X)
X = Conv2D(filters=F3,
kernel_size=(1, 1),
strides=(1, 1),
padding="valid",
name=conv_name_base + "2c",
kernel_initializer=GlorotUniform(seed=2019))(X)
X_shortcut = Conv2D(
filters=F3,
kernel_size=(1, 1),
strides=(s, s),
padding="valid",
name=conv_name_base + "1",
kernel_initializer=GlorotUniform(seed=2019))(X_shortcut)
X = Add()([X, X_shortcut])
return X
def frozen_glorot_uniform():
'''
Deterministic GlorotUniform initializer.
'''
if LOCAL_SEED is not None:
initializer = GlorotUniform(LOCAL_SEED + INITIALIZER_COUNT)
globals()['INITIALIZER_COUNT'] = INITIALIZER_COUNT + 1
return initializer
else:
return GlorotUniform()
def create_implExModel(num_nodes,
num_edges,
embed_size=50,
n3_reg=1e-3,
learning_rate=5e-1,
num_negs=50,
alpha=1.,
beta=1.):
# Build complEx Model
sub_inputs = Input(shape=(), name='subject')
obj_inputs = Input(shape=(), name='object')
rel_inputs = Input(shape=(), name='relation')
cnt_inputs = Input(shape=(), name='count')
y_true_inputs = Input(shape=(), name='label')
inputs = {
"subject": sub_inputs,
"object": obj_inputs,
"relation": rel_inputs,
"count": cnt_inputs,
"label": y_true_inputs
}
node_layer = Embedding(input_dim=num_nodes,
output_dim=embed_size,
embeddings_initializer=GlorotUniform(),
name='node_embedding')
edge_layer = Embedding(input_dim=num_edges,
output_dim=embed_size,
embeddings_initializer=GlorotUniform(),
name='edge_embedding')
sub_embed = node_layer(sub_inputs)
rel_embed = edge_layer(rel_inputs)
obj_embed = node_layer(obj_inputs)
outputs = ComplExDotScore(n3_reg)([sub_embed, rel_embed, obj_embed])
model = Model(inputs, outputs, name='implEx')
# Compile implEx Model
wbce_loss = tf.nn.weighted_cross_entropy_with_logits(
y_true_inputs, outputs, num_negs) / num_negs
confidence = 1 + alpha * tf.math.log(1 + cnt_inputs / beta)
loss = K.sum(confidence * wbce_loss)
model.add_loss(loss)
model.add_metric(K.mean(wbce_loss), 'weighted_binarycrossentropy')
model.compile(optimizer=Adagrad(learning_rate))
return model
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max-n_min)/4)
model.add(Conv1D(filters=hp.Int("CNN_"+str(i), min_value=n_min, max_value=n_max, step=step), kernel_size=2, activation="relu", kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
def __init__(
self,
in_feat: int,
out_feat: int,
kernel_size: Tuple,
strides: Tuple = (1, 1),
activation: Callable = None,
name: str = 'conv',
padding: str = 'VALID',
use_batch_norm: bool = False,
initializer: Callable = GlorotUniform(seed=0)
) -> None:
# Weight parameters
self.W = tf.Variable(initializer(
[kernel_size[0], kernel_size[1], in_feat, out_feat]),
name=name + '_W')
self.b = tf.Variable(tf.zeros([out_feat]), name=name + '_b')
# params for weight updates
self.params = [self.W, self.b]
# layer attributes
self.name = name
self.stride = strides
self.padding = padding
self.activation = activation
self.use_batch_norm = use_batch_norm
def __init__(self,
num_layers=2,
num_units=32,
activation="tanh",
seed=None,
*args,
**kwargs):
layer_kws = dict(activation=activation,
kernel_initializer=GlorotUniform(seed=seed))
final_layer_kws = dict(kernel_initializer=GlorotUniform(seed=seed))
self.model = DenseSequential(1,
num_layers,
num_units,
layer_kws=layer_kws,
final_layer_kws=final_layer_kws)
def __init__(self, code_levels, code_num_in_levels, code_dims, name='hierarchical_embedding'):
super().__init__(name=name)
self.level_num = len(code_num_in_levels)
self.code_levels = code_levels # (leaf code num * level_num)
self.level_embeddings = [self.add_weight(name='hier_emb_level_%d' % level,
shape=(code_num, code_dim),
initializer=GlorotUniform(),
trainable=True)
for level, (code_num, code_dim) in enumerate(zip(code_num_in_levels, code_dims))]
def __init__(self,
n_hop,
hidden_dim,
nc=256,
penalty=1.0,
return_attention=False,
kernel_initializer=GlorotUniform(),
kernel_regularizer=None,
kernel_constraint=None,
**kwargs):
self.n_hop = n_hop
self.hidden_dim = hidden_dim
self.nc=nc
self.penalty = penalty
self.kernel_initializer = GlorotUniform() # tf.keras.initializers.get(kernel_initializer)
self.kernel_regularizer = None #tf.keras.regularizers.get(kernel_regularizer)
self.kernel_constraint = None #tf.keras.constraints.get(kernel_constraint)
self.return_attention = return_attention
super(SelfAttention, self).__init__(**kwargs)
def get_initializer(self):
if self.config.use_custom_init:
return RandomUniform(-self.config.init_scale, self.config.init_scale, seed=self.config.seed)
else:
if self.config.initializer == 'GlorotNormal':
return GlorotNormal(seed=self.config.seed)
elif self.config.initializer == 'GlorotUniform':
return GlorotUniform(seed=self.config.seed)
else:
raise NotImplementedError
def identity_block_v1(X, f, filters, stage, block):
print("identity_block_v1_{}_{}_{}_{}_{}".format(X, f, filters, stage,
block))
conv_name_base = "res{}{}_branch".format(stage, block)
bn_name_base = "bn{}{}_branch".format(stage, block)
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters=F1,
kernel_size=(1, 1),
strides=(1, 1),
padding="valid",
name=conv_name_base + "2a",
kernel_initializer=GlorotUniform(seed=2019))(X)
X = BatchNormalization(axis=3, name=bn_name_base + "2a")(X)
X = Activation("relu")(X)
X = Conv2D(filters=F2,
kernel_size=(f, f),
strides=(1, 1),
padding="same",
name=conv_name_base + "2b",
kernel_initializer=GlorotUniform(seed=2019))(X)
X = BatchNormalization(axis=3, name=bn_name_base + "2b")(X)
X = Activation("relu")(X)
X = Conv2D(filters=F3,
kernel_size=(1, 1),
strides=(1, 1),
padding="valid",
name=conv_name_base + "2c",
kernel_initializer=GlorotUniform(seed=2019))(X)
X = BatchNormalization(axis=3, name=bn_name_base + "2c")(X)
X = Activation("relu")(X)
X = Add()([X, X_shortcut])
X = Activation("relu")(X)
return X
def conv_net(nbr_classes, img_size=28):
"""Reproduces the CNN used in the MAML paper. It was originally designed in
Vinyals and al. (2016) .
Conv layers kernels are initialized with Glorot Uniform by default.
Args:
nbr_classes: Integer, the number of classes.
img_size: Integer, the width and height of the squarred images.
"""
model = Sequential()
model.add(
Conv2D(64, (3, 3),
strides=(2, 2),
activation='relu',
input_shape=(img_size, img_size, 3),
kernel_initializer=GlorotUniform(seed=1234)))
model.add(BatchNormalization())
model.add(
Conv2D(64, (3, 3),
strides=(2, 2),
activation='relu',
kernel_initializer=GlorotUniform(seed=1234)))
model.add(BatchNormalization())
model.add(
Conv2D(64, (3, 3),
strides=(2, 2),
activation='relu',
kernel_initializer=GlorotUniform(seed=1234)))
model.add(BatchNormalization())
model.add(Flatten())
model.add(
Dense(
nbr_classes,
activation='softmax',
kernel_initializer=GlorotUniform(seed=1234))) # Outputting probas
return model
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0))))
if i == layer-1:
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0))))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
def create_complExModel(num_nodes,
num_edges,
embed_size=50,
n3_reg=1e-3,
learning_rate=5e-1):
# Build complEx Model
sub_inputs = Input(shape=(), name='subject')
obj_inputs = Input(shape=(), name='object')
rel_inputs = Input(shape=(), name='relation')
inputs = {
"subject": sub_inputs,
"object": obj_inputs,
"relation": rel_inputs
}
node_layer = Embedding(input_dim=num_nodes,
output_dim=embed_size,
embeddings_initializer=GlorotUniform(),
name='node_embedding')
edge_layer = Embedding(input_dim=num_edges,
output_dim=embed_size,
embeddings_initializer=GlorotUniform(),
name='edge_embedding')
sub_embed = node_layer(sub_inputs)
rel_embed = edge_layer(rel_inputs)
obj_embed = node_layer(obj_inputs)
outputs = ComplExDotScore(n3_reg)([sub_embed, rel_embed, obj_embed])
model = Model(inputs, outputs, name='complEx')
# Compile complEx Model
loss = BinaryCrossentropy(from_logits=True, reduction='sum')
model.compile(optimizer=Adagrad(learning_rate), loss=loss, metrics=[loss])
return model
def multi_layer_classifier(number_of_classes,
hidden_size,
l2_lam=5e-4,
drop_out=0.5):
return Sequential([
Flatten(),
Dropout(drop_out),
relu_fc(size=hidden_size,
batch_norm=True,
drop_out=drop_out,
lam=l2_lam),
Dense(number_of_classes,
kernel_regularizer=l2(l2_lam),
kernel_initializer=GlorotUniform()),
Softmax()
])
def relu_fc(size, batch_norm=True, drop_out=0.5, lam=5e-4):
layer = tf.keras.Sequential()
layer.add(
Dense(size,
kernel_regularizer=l2(lam),
kernel_initializer=GlorotUniform()))
if batch_norm:
layer.add(BatchNormalization())
layer.add(ReLU())
if drop_out:
layer.add(Dropout(drop_out))
return layer
def __init__(
self,
in_feat: int,
out_feat: int,
activation: Callable = None,
name: str = 'dense',
initializer: Callable = GlorotUniform(seed=0)
) -> None:
# Weight parameters
self.W = tf.Variable(initializer([in_feat, out_feat]),
name=name + '_W')
self.b = tf.Variable(tf.zeros([out_feat]), name=name + '_b')
# params for weight updates
self.params = [self.W, self.b]
# layer attributes
self.activation = activation
def get_init(initializer_name: str) -> Initializer:
"""
Get an object in tensorflow.keras.initializers by name.
:param initializer_name:
str
Support initializer_name without case sensitive:
'glorotnormal'
'glorotuniform'
:return:
Initializer
An Initializer object.
"""
initializers = {
'glorotnormal': GlorotNormal(),
'glorotuniform': GlorotUniform(),
}
initializer_name = initializer_name.strip().lower()
try:
return initializers[initializer_name]
except KeyError as keyerr:
raise SuiValueError(f'{keyerr} is not a valid initializer name.')
def fit(self, X, y, **kwargs):
""" Fit the seq2seq model to convert sequences one to another.
Each sequence is unicode text composed from the tokens. Tokens are separated by spaces.
The Rectified Adam with Lookahead algorithm is used for training. To avoid overfitting,
you must use an early stopping criterion. This criterion is included automatically
if evaluation set is defined. You can do this in one of two ways:
1) set a `validation_split` parameter of this object, and in this case evaluation set will be selected as a
corresponded part of training set proportionally to the `validation_split` value;
2) set an `eval_set` argument of this method, and then evaluation set is defined entirely by this argument.
:param X: input texts for training.
:param y: target texts for training.
:param eval_set: optional argument containing input and target texts for evaluation during an early-stopping.
:return self
"""
self.check_params(**self.get_params(deep=False))
self.check_X(X, 'X')
self.check_X(y, 'y')
if len(X) != len(y):
raise ValueError(f'`X` does not correspond to `y`! {len(X)} != {len(y)}.')
if 'eval_set' in kwargs:
if (not isinstance(kwargs['eval_set'], tuple)) and (not isinstance(kwargs['eval_set'], list)):
raise ValueError(f'`eval_set` must be `{type((1, 2))}` or `{type([1, 2])}`, not `{type(kwargs["eval_set"])}`!')
if len(kwargs['eval_set']) != 2:
raise ValueError(f'`eval_set` must be a two-element sequence! {len(kwargs["eval_set"])} != 2')
self.check_X(kwargs['eval_set'][0], 'X_eval_set')
self.check_X(kwargs['eval_set'][1], 'y_eval_set')
if len(kwargs['eval_set'][0]) != len(kwargs['eval_set'][1]):
raise ValueError(f'`X_eval_set` does not correspond to `y_eval_set`! '
f'{len(kwargs["eval_set"][0])} != {len(kwargs["eval_set"][1])}.')
X_eval_set = kwargs['eval_set'][0]
y_eval_set = kwargs['eval_set'][1]
else:
if self.validation_split is None:
X_eval_set = None
y_eval_set = None
else:
n_eval_set = int(round(len(X) * self.validation_split))
if n_eval_set < 1:
raise ValueError('`validation_split` is too small! There are no samples for evaluation!')
if n_eval_set >= len(X):
raise ValueError('`validation_split` is too large! There are no samples for training!')
X_eval_set = X[-n_eval_set:-1]
y_eval_set = y[-n_eval_set:-1]
X = X[:-n_eval_set]
y = y[:-n_eval_set]
input_characters = set()
target_characters = set()
max_encoder_seq_length = 0
max_decoder_seq_length = 0
for sample_ind in range(len(X)):
prep = self.tokenize_text(X[sample_ind], self.lowercase)
n = len(prep)
if n == 0:
raise ValueError(f'Sample {sample_ind} of `X` is wrong! This sample is empty.')
if n > max_encoder_seq_length:
max_encoder_seq_length = n
input_characters |= set(prep)
prep = self.tokenize_text(y[sample_ind], self.lowercase)
n = len(prep)
if n == 0:
raise ValueError(f'Sample {sample_ind} of `y` is wrong! This sample is empty.')
if (n + 2) > max_decoder_seq_length:
max_decoder_seq_length = n + 2
target_characters |= set(prep)
if len(input_characters) == 0:
raise ValueError('`X` is empty!')
if len(target_characters) == 0:
raise ValueError('`y` is empty!')
input_characters_ = set()
target_characters_ = set()
if (X_eval_set is not None) and (y_eval_set is not None):
for sample_ind in range(len(X_eval_set)):
prep = self.tokenize_text(X_eval_set[sample_ind], self.lowercase)
n = len(prep)
if n == 0:
raise ValueError(f'Sample {sample_ind} of `X_eval_set` is wrong! This sample is empty.')
if n > max_encoder_seq_length:
max_encoder_seq_length = n
input_characters_ |= set(prep)
prep = self.tokenize_text(y_eval_set[sample_ind], self.lowercase)
n = len(prep)
if n == 0:
raise ValueError(f'Sample {sample_ind} of `y_eval_set` is wrong! This sample is empty.')
if (n + 2) > max_decoder_seq_length:
max_decoder_seq_length = n + 2
target_characters_ |= set(prep)
if len(input_characters_) == 0:
raise ValueError('`X_eval_set` is empty!')
if len(target_characters_) == 0:
raise ValueError('`y_eval_set` is empty!')
input_characters = sorted(list(input_characters | input_characters_))
target_characters = sorted(list(target_characters | target_characters_ | {'\t', '\n'}))
if self.verbose:
print('')
print(f'Number of samples for training: {len(X)}.')
if X_eval_set is not None:
print(f'Number of samples for evaluation and early stopping: {len(X_eval_set)}.')
print(f'Number of unique input tokens: {len(input_characters)}.')
print(f'Number of unique output tokens: {len(target_characters)}.')
print(f'Max sequence length for inputs: {max_encoder_seq_length}.')
print(f'Max sequence length for outputs: {max_decoder_seq_length}.')
print('')
self.input_token_index_ = dict([(char, i) for i, char in enumerate(input_characters)])
self.target_token_index_ = dict([(char, i) for i, char in enumerate(target_characters)])
self.max_encoder_seq_length_ = max_encoder_seq_length
self.max_decoder_seq_length_ = max_decoder_seq_length
K.clear_session()
encoder_inputs = Input(shape=(None, len(self.input_token_index_)),
name='EncoderInputs')
encoder_mask = Masking(name='EncoderMask', mask_value=0.0)(encoder_inputs)
encoder = LSTM(
self.latent_dim,
return_sequences=False, return_state=True,
kernel_initializer=GlorotUniform(seed=self.generate_random_seed()),
recurrent_initializer=Orthogonal(seed=self.generate_random_seed()),
name='EncoderLSTM'
)
encoder_outputs, state_h, state_c = encoder(encoder_mask)
encoder_states = [state_h, state_c]
decoder_inputs = Input(shape=(None, len(self.target_token_index_)),
name='DecoderInputs')
decoder_mask = Masking(name='DecoderMask', mask_value=0.0)(decoder_inputs)
decoder_lstm = LSTM(
self.latent_dim,
return_sequences=True, return_state=True,
kernel_initializer=GlorotUniform(seed=self.generate_random_seed()),
recurrent_initializer=Orthogonal(seed=self.generate_random_seed()),
name='DecoderLSTM'
)
decoder_outputs, _, _ = decoder_lstm(decoder_mask, initial_state=encoder_states)
decoder_dense = Dense(
len(self.target_token_index_), activation='softmax',
kernel_initializer=GlorotUniform(seed=self.generate_random_seed()),
name='DecoderOutput'
)
decoder_outputs = decoder_dense(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs,
name='Seq2SeqModel')
radam = RectifiedAdam(learning_rate=self.lr, weight_decay=self.weight_decay)
optimizer = Lookahead(radam, sync_period=6, slow_step_size=0.5)
model.compile(optimizer=optimizer, loss='categorical_crossentropy')
if self.verbose:
model.summary(positions=[0.23, 0.77, 0.85, 1.0])
print('')
training_set_generator = TextPairSequence(
input_texts=X, target_texts=y,
batch_size=self.batch_size,
max_encoder_seq_length=max_encoder_seq_length, max_decoder_seq_length=max_decoder_seq_length,
input_token_index=self.input_token_index_, target_token_index=self.target_token_index_,
lowercase=self.lowercase
)
if (X_eval_set is not None) and (y_eval_set is not None):
evaluation_set_generator = TextPairSequence(
input_texts=X_eval_set, target_texts=y_eval_set,
batch_size=self.batch_size,
max_encoder_seq_length=max_encoder_seq_length, max_decoder_seq_length=max_decoder_seq_length,
input_token_index=self.input_token_index_, target_token_index=self.target_token_index_,
lowercase=self.lowercase
)
callbacks = [
EarlyStopping(patience=5, verbose=(1 if self.verbose else 0), monitor='val_loss')
]
else:
evaluation_set_generator = None
callbacks = []
tmp_weights_name = self.get_temp_name()
try:
callbacks.append(
ModelCheckpoint(filepath=tmp_weights_name, verbose=(1 if self.verbose else 0), save_best_only=True,
save_weights_only=True,
monitor='loss' if evaluation_set_generator is None else 'val_loss')
)
model.fit_generator(
generator=training_set_generator,
epochs=self.epochs, verbose=(1 if self.verbose else 0),
shuffle=True,
validation_data=evaluation_set_generator,
callbacks=callbacks
)
if os.path.isfile(tmp_weights_name):
model.load_weights(tmp_weights_name)
finally:
if os.path.isfile(tmp_weights_name):
os.remove(tmp_weights_name)
self.encoder_model_ = Model(encoder_inputs, encoder_states)
decoder_state_input_h = Input(shape=(self.latent_dim,))
decoder_state_input_c = Input(shape=(self.latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c = decoder_lstm(
decoder_mask, initial_state=decoder_states_inputs)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
self.decoder_model_ = Model(
[decoder_inputs] + decoder_states_inputs,
[decoder_outputs] + decoder_states)
self.reverse_target_char_index_ = dict(
(i, char) for char, i in self.target_token_index_.items())
return self
def createModel(self):
self.model_instance += 1
clear_session()
features, label = self.getDataset()
X_train, y_train = self.createLag(features, label)
X_train = X_train[:, self.lags]
learning_rate = float(self.hyperparameters["Learning_Rate"].get())
momentum = float(self.hyperparameters["Momentum"].get())
optimizers = {
"Adam": Adam(learning_rate=learning_rate),
"SGD": SGD(learning_rate=learning_rate, momentum=momentum),
"RMSprop": RMSprop(learning_rate=learning_rate, momentum=momentum)
}
shape = (X_train.shape[1], X_train.shape[2])
model_choice = self.model_var.get()
if not self.do_optimization:
model = Sequential()
model.add(Input(shape=shape))
if model_choice == 0:
model.add(Flatten())
layers = self.no_optimization_choice_var.get()
for i in range(layers):
neuron_number = self.neuron_numbers_var[i].get()
activation_function = self.activation_var[i].get()
if model_choice == 0:
model.add(Dense(neuron_number, activation=activation_function, kernel_initializer=GlorotUniform(seed=0)))
elif model_choice == 1:
model.add(Conv1D(filters=neuron_number, kernel_size=2, activation=activation_function, kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
elif model_choice == 2:
if i == layers-1:
model.add(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
elif model_choice == 3:
if i == layers-1:
model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))))
model.add(Dropout(0.2))
else:
model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))))
model.add(Dropout(0.2))
elif model_choice == 4:
if i == layers-1:
model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
elif model_choice == 5:
if i == layers-1:
model.add(GRU(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(GRU(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
if model_choice == 1:
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, activation=self.output_activation.get(), kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1, shuffle=False)
loss = history.history["loss"][-1]
self.train_loss.set(loss)
elif self.do_optimization:
layer = self.optimization_choice_var.get()
if model_choice == 0:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
model.add(Flatten())
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Dense(units=hp.Int('MLP_'+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu'))
model.add(Dense(1))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". MLP"
elif model_choice == 1:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max-n_min)/4)
model.add(Conv1D(filters=hp.Int("CNN_"+str(i), min_value=n_min, max_value=n_max, step=step), kernel_size=2, activation="relu", kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". CNN"
elif model_choice == 2:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0)))
if i == layer-1:
model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". LSTM"
elif model_choice == 3:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0))))
if i == layer-1:
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0))))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". Bi-LSTM"
tuner = RandomSearch(build_model, objective='loss', max_trials=25, executions_per_trial=2, directory=self.runtime, project_name=name)
tuner.search(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get())
hps = tuner.get_best_hyperparameters(num_trials = 1)[0]
model = tuner.hypermodel.build(hps)
history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1)
loss = history.history["loss"][-1]
self.train_loss.set(loss)
for i in range(layer):
if model_choice == 0:
self.best_model_neurons[i].set(model.get_layer(index=i+1).get_config()["units"])
elif model_choice == 1:
self.best_model_neurons[i].set(model.get_layer(index=(2*i)).get_config()["filters"])
elif model_choice == 2:
self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["units"])
elif model_choice == 3:
self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["layer"]["config"]["units"])
model.summary()
self.model = model
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='AdamW')
sgdOpti = SGD(learning_rate=0.0001,
momentum=0.8,
nesterov=False,
name='SGD')
rmsOpti = tf.keras.optimizers.RMSprop(learning_rate=0.001,
rho=0.9,
momentum=0.8,
epsilon=1e-07,
centered=False,
name='RMSprop')
initializer = GlorotUniform()
regularizerl2 = L2(l2=0.1)
EPOCHS = 3000
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, Dropout, Flatten, MaxPooling2D, BatchNormalization
model = Sequential()
#, kernel_regularizer=regularizerl2
model.add(
Conv2D(64, (3, 3),
activation='relu',
input_shape=(8, 8, 11),
padding='same',
kernel_regularizer=regularizerl2))
model.add(BatchNormalization())
model.add(
def __init__(self,
X,
adj,
adj_n,
hidden_dim=128,
latent_dim=10,
dec_dim=None,
adj_dim=32,
decA="DBL",
layer_enc="GAT"):
super(SCGAE, self).__init__()
if dec_dim is None:
dec_dim = [64, 256, 512]
self.latent_dim = latent_dim
self.X = X
self.adj = np.float32(adj)
self.adj_n = np.float32(adj_n)
self.n_sample = X.shape[0]
self.in_dim = X.shape[1]
self.sparse = False
initializer = GlorotUniform(seed=7)
# Encoder
X_input = Input(shape=self.in_dim)
h = Dropout(0.2)(X_input)
if layer_enc == "GAT":
A_in = Input(shape=self.n_sample)
h = GraphAttention(channels=hidden_dim,
attn_heads=1,
kernel_initializer=initializer,
activation="relu")([h, A_in])
z_mean = GraphAttention(channels=latent_dim,
kernel_initializer=initializer,
attn_heads=1)([h, A_in])
elif layer_enc == "GCN":
A_in = Input(shape=self.n_sample)
h = GraphConvSkip(channels=hidden_dim,
kernel_initializer=initializer,
activation="relu")([h, A_in])
z_mean = GraphConvSkip(channels=latent_dim,
kernel_initializer=initializer)([h, A_in])
elif layer_enc == "TAG":
self.sparse = True
A_in = Input(shape=self.n_sample, sparse=True)
h = TAGConv(channels=hidden_dim,
kernel_initializer=initializer,
activation="relu")([h, A_in])
z_mean = TAGConv(channels=latent_dim,
kernel_initializer=initializer)([h, A_in])
self.encoder = Model(inputs=[X_input, A_in],
outputs=z_mean,
name="encoder")
clustering_layer = ClusteringLayer(name='clustering')(z_mean)
self.cluster_model = Model(inputs=[X_input, A_in],
outputs=clustering_layer,
name="cluster_encoder")
# Adjacency matrix decoder
if decA == "DBL":
dec_in = Input(shape=latent_dim)
h = Dense(units=adj_dim, activation=None)(dec_in)
h = Bilinear()(h)
dec_out = Lambda(lambda z: tf.nn.sigmoid(z))(h)
self.decoderA = Model(inputs=dec_in,
outputs=dec_out,
name="decoder1")
elif decA == "BL":
dec_in = Input(shape=latent_dim)
h = Bilinear()(dec_in)
dec_out = Lambda(lambda z: tf.nn.sigmoid(z))(h)
self.decoderA = Model(inputs=dec_in,
outputs=dec_out,
name="decoder1")
elif decA == "IP":
dec_in = Input(shape=latent_dim)
dec_out = Lambda(
lambda z: tf.nn.sigmoid(tf.matmul(z, tf.transpose(z))))(dec_in)
self.decoderA = Model(inputs=dec_in,
outputs=dec_out,
name="decoder1")
else:
self.decoderA = None
# Expression matrix decoder
decx_in = Input(shape=latent_dim)
h = Dense(units=dec_dim[0], activation="relu")(decx_in)
h = Dense(units=dec_dim[1], activation="relu")(h)
h = Dense(units=dec_dim[2], activation="relu")(h)
decx_out = Dense(units=self.in_dim)(h)
self.decoderX = Model(inputs=decx_in,
outputs=decx_out,
name="decoderX")
def load_weights(self, weights_as_bytes):
""" Load weights of neural model from the binary data.
:param weights_as_bytes: 2-element tuple of binary data (`bytes` or `byterray` objects) containing weights of
neural encoder and neural decoder respectively.
"""
if not isinstance(weights_as_bytes, tuple):
raise ValueError(f'`weights_as_bytes` must be a 2-element tuple, not `{type(weights_as_bytes)}`!')
if len(weights_as_bytes) != 2:
raise ValueError(f'`weights_as_bytes` must be a 2-element tuple, but it is a {len(weights_as_bytes)}-element tuple!')
if (not isinstance(weights_as_bytes[0], bytearray)) and (not isinstance(weights_as_bytes[0], bytes)):
raise ValueError(f'First element of `weights_as_bytes` must be an array of bytes, not `{type(weights_as_bytes[0])}`!')
if (not isinstance(weights_as_bytes[1], bytearray)) and (not isinstance(weights_as_bytes[1], bytes)):
raise ValueError(f'Second element of `weights_as_bytes` must be an array of bytes, not `{type(weights_as_bytes[1])}`!')
tmp_weights_name = self.get_temp_name()
try:
K.clear_session()
encoder_inputs = Input(shape=(None, len(self.input_token_index_)),
name='EncoderInputs')
encoder_mask = Masking(name='EncoderMask', mask_value=0.0)(encoder_inputs)
encoder = LSTM(
self.latent_dim,
return_sequences=False, return_state=True,
kernel_initializer=GlorotUniform(seed=self.generate_random_seed()),
recurrent_initializer=Orthogonal(seed=self.generate_random_seed()),
name='EncoderLSTM'
)
encoder_outputs, state_h, state_c = encoder(encoder_mask)
encoder_states = [state_h, state_c]
decoder_inputs = Input(shape=(None, len(self.target_token_index_)),
name='DecoderInputs')
decoder_mask = Masking(name='DecoderMask', mask_value=0.0)(decoder_inputs)
decoder_lstm = LSTM(
self.latent_dim, return_sequences=True, return_state=True,
kernel_initializer=GlorotUniform(seed=self.generate_random_seed()),
recurrent_initializer=Orthogonal(seed=self.generate_random_seed()),
name='DecoderLSTM'
)
decoder_outputs, _, _ = decoder_lstm(decoder_mask, initial_state=encoder_states)
decoder_dense = Dense(
len(self.target_token_index_), activation='softmax',
kernel_initializer=GlorotUniform(seed=self.generate_random_seed()),
name='DecoderOutput'
)
self.encoder_model_ = Model(encoder_inputs, encoder_states)
decoder_state_input_h = Input(shape=(self.latent_dim,))
decoder_state_input_c = Input(shape=(self.latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c = decoder_lstm(
decoder_mask, initial_state=decoder_states_inputs)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
self.decoder_model_ = Model(
[decoder_inputs] + decoder_states_inputs,
[decoder_outputs] + decoder_states)
with open(tmp_weights_name, 'wb') as fp:
fp.write(weights_as_bytes[0])
self.encoder_model_.load_weights(tmp_weights_name)
os.remove(tmp_weights_name)
with open(tmp_weights_name, 'wb') as fp:
fp.write(weights_as_bytes[1])
self.decoder_model_.load_weights(tmp_weights_name)
os.remove(tmp_weights_name)
finally:
if os.path.isfile(tmp_weights_name):
os.remove(tmp_weights_name)
def __init__(self, patient_num, patient_dim, name='patient_embedding'):
super().__init__(name=name)
self.patient_embeddings = self.add_weight(name='p_emb',
shape=(patient_num, patient_dim),
initializer=GlorotUniform(),
trainable=True)
def build(self, input_shape):
hidden_size = input_shape[-1]
self.w_omega = self.add_weight(name=self.name + '_w', shape=(hidden_size, self.attention_dim), initializer=GlorotUniform())
def __init__(self, attention_dim, name='attention'):
super().__init__(name=name)
self.attention_dim = attention_dim
self.u_omega = self.add_weight(name=name + '_u', shape=(attention_dim,), initializer=GlorotUniform())
self.w_omega = None
import numpy as np
import matplotlib.pyplot as plt
from sklearn.mixture import GaussianMixture
import tensorflow as tf
from tensorflow.keras import layers, models, datasets
from tensorflow.keras import backend as K
from tensorflow.keras.models import Model
from tensorflow.keras.regularizers import l2
from tensorflow import keras
from tensorflow.keras.initializers import RandomNormal, GlorotUniform
from tensorflow.keras.optimizers import Adam
from utils import load_data, utils, evaluate
import two_stage
initializer = GlorotUniform() #RandomNormal(stddev=1e-2)
reg = None #'l2'
act = 'swish' #'relu'
"""
CUSTOM LAYERS
"""
class Film(layers.Layer): #Feature-wise linear modulation
def __init__(self, c, **kwargs):
super().__init__(**kwargs)
self.c = c
self.layers = [
layers.Dense(self.c),
layers.Dense(self.c),
layers.Multiply(),
def generator(momentum=0.8):
lr_in = tf.keras.Input(shape=(None,None,3))
hr_out = Conv2D(filters = 64, kernel_size = (9,9), padding='SAME')(lr_in) #k9n64s1
hr_out = B = PReLU(shared_axes=[1, 2])(hr_out)
for i in range(16):
B_internal = B
B_internal = Conv2D(filters = 64, kernel_size = (3,3), padding='SAME')(B_internal) #k3n64s1
B_internal = BatchNormalization(momentum=momentum)(B_internal)
B_internal = PReLU(shared_axes=[1, 2])(B_internal)
B_internal = Conv2D(filters = 64, kernel_size = (3,3), padding='SAME')(B_internal) #k3n64s1
B_internal = BatchNormalization(momentum=momentum)(B_internal)
B = Add()([B, B_internal])
B = Conv2D(filters = 64, kernel_size = (3,3), padding='SAME')(B) #k3n64s1
B = BatchNormalization(momentum=momentum)(B)
hr_out = Add()([hr_out, B])
for i in range(2):
hr_out = Conv2D(filters = 256, kernel_size = (3,3), padding = "SAME", kernel_initializer=ICNR(GlorotUniform()))(hr_out) #k3n256s1
hr_out = UpSampling2D(size=2)(hr_out)
hr_out = LeakyReLU(alpha=0.2)(hr_out)
hr_out = Conv2D(filters = 3, kernel_size = (9,9), padding = "SAME")(hr_out) #k9n3s1
hr_out = tanh(hr_out)
return Model(lr_in, hr_out, name="GAN_GEN")
def build_model(version,
loss_funcs,
n_genders,
n_ages,
metrics,
loss_weights,
optimizer,
input_shape=(256, 256, 3),
stages=4,
strides=[1, 2, 2, 2],
n_identities=[2, 3, 5, 2],
initial_filter=np.array([64, 64, 256]),
bifurcation_stage=3):
if version == 1:
convolutional_block = convolutional_block_v1
identity_block = identity_block_v1
elif version == 2:
convolutional_block = convolutional_block_v2
identity_block = identity_block_v2
else:
raise NameError("Unsupported Version '{}'".format(version))
X_input = Input(shape=input_shape)
X = ZeroPadding2D((3, 3))(X_input)
X = Conv2D(64, (7, 7),
strides=(2, 2),
name="conv1",
kernel_initializer=GlorotUniform(seed=2019))(X)
X = BatchNormalization(axis=3, name="bn_conv1")(X)
X = Activation("relu")(X)
X = MaxPooling2D((3, 3), strides=(2, 2))(X)
for stage in range(stages):
s = strides[stage]
if stage < bifurcation_stage:
X = convolutional_block(X,
f=3,
filters=list(initial_filter * 2**stage),
stage=stage + 2,
block="a",
s=s)
n_identity = n_identities[stage]
for block in ascii_lowercase[1:n_identity + 1]:
X = identity_block(X,
3,
filters=list(initial_filter * 2**stage),
stage=stage + 2,
block=block)
else:
if stage == bifurcation_stage:
genders = X
ages = X
genders = convolutional_block(genders,
f=3,
filters=list(initial_filter *
2**stage),
stage=stage + 2,
block="gender_a",
s=s)
ages = convolutional_block(ages,
f=3,
filters=list(initial_filter * 2**stage),
stage=stage + 2,
block="age_a",
s=s)
n_identity = n_identities[stage]
for block in ascii_lowercase[1:n_identity + 1]:
genders = identity_block(genders,
3,
filters=list(initial_filter *
2**stage),
stage=stage + 2,
block="gender_{}".format(block))
ages = identity_block(ages,
3,
filters=list(initial_filter * 2**stage),
stage=stage + 2,
block="age_{}".format(block))
if bifurcation_stage == stages:
genders = X
ages = X
if version == 2:
genders = BatchNormalization(axis=3)(genders)
ages = BatchNormalization(axis=3)(ages)
genders = Activation("relu")(genders)
ages = Activation("relu")(ages)
genders = AveragePooling2D(pool_size=(2, 2), padding="same")(genders)
ages = AveragePooling2D(pool_size=(2, 2), padding="same")(ages)
genders = Flatten()(genders)
ages = Flatten()(ages)
genders = Dense(256,
activation="relu",
name="fc_genders_256_1",
kernel_initializer=GlorotUniform(seed=2019),
kernel_regularizer=regularizers.l1_l2(0.0001,
0.0001))(genders)
genders = Dropout(rate=.2, seed=2019)(genders)
ages = Dense(256,
activation="relu",
name="fc_ages_256_1",
kernel_initializer=GlorotUniform(seed=2019),
kernel_regularizer=regularizers.l1_l2(0.0001, 0.0001))(ages)
ages = Dropout(rate=.2, seed=2019)(ages)
genders = Dense(128,
activation="relu",
name="fc_genders_256_2",
kernel_initializer=GlorotUniform(seed=2019),
kernel_regularizer=regularizers.l1_l2(0.0001,
0.0001))(genders)
genders = Dropout(rate=.2, seed=2019)(genders)
ages = Dense(128,
activation="relu",
name="fc_ages_256_2",
kernel_initializer=GlorotUniform(seed=2019),
kernel_regularizer=regularizers.l1_l2(0.0001, 0.0001))(ages)
ages = Dropout(rate=.2, seed=2019)(ages)
genders = Dense(n_genders,
activation="softmax",
name="genders",
kernel_initializer=GlorotUniform(seed=2019))(genders)
ages = Dense(n_ages,
activation="sigmoid",
name="ages",
kernel_initializer=GlorotUniform(seed=2019))(ages)
model = Model(inputs=X_input,
outputs=[genders, ages],
name="ResNet50_mod{}{}".format(stages, bifurcation_stage))
model.compile(optimizer=optimizer,
loss=loss_funcs,
loss_weights=loss_weights,
metrics=metrics)
plot_model(model,
to_file="model_loop_{}_{}_v{}.png".format(
stages, bifurcation_stage, version))
return model