class CartpoleModel(Model):
def _build_layers_v2(self, input_dict, num_outputs, options):
self.model = Sequential()
self.model.add(layers.InputLayer(input_tensor=input_dict["obs"], input_shape=(4,)))
self.model.add(layers.Dense(4, name='l1', activation='relu'))
self.model.add(layers.Dense(10, name='l2', activation='relu'))
self.model.add(layers.Dense(10, name='l3', activation='relu'))
self.model.add(layers.Dense(10, name='l4', activation='relu'))
self.model.add(layers.Dense(2, name='l5', activation='relu'))
return self.model.get_layer("l5").output, self.model.get_layer("l4").output
def define_supernet(members, new_training_set):
model = Sequential([
Dense(num_classes, activation='softmax', input_shape=(new_training_set.shape[1],))
])
model.compile(loss='categorical_crossentropy',
optimizer="adam",
metrics=['accuracy'])
all_weights = [m.get_layer(index=-2).get_weights() for m in members]
weights = np.concatenate([w for w, _ in all_weights], axis=0)
biases = np.mean([b for _, b in all_weights], axis=0)
model.get_layer(index=-1).set_weights([weights, biases])
return model
class siamese_model:
def __init__(self, input_shape, num_classes, embedding_size):
# Convolutional Neural Network
self.model = Sequential(name="conv_net")
self.model.add(
Convolution2D(32, 3, 3, input_shape=input_shape,
activation='relu'))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
# 32 filters each. The kernel size is 3*3
self.model.add(Convolution2D(32, 3, 3, activation='relu'))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
# fully connected part
self.model.add(Flatten())
self.model.add(
Dense(embedding_size, activation='relu', name='embedding'))
self.model.add(Dense(num_classes, activation='softmax'))
# Define the tensors for the two input images
self.x1 = Input(input_shape, name="x1")
self.x2 = Input(input_shape, name="x2")
# to get the output of the embedding layer, create a sub model from input layer to embedding layer
embedding_sub_model = Model(
inputs=self.model.input,
outputs=self.model.get_layer('embedding').output)
self.embedding_x1 = embedding_sub_model(self.x1)
self.embedding_x2 = embedding_sub_model(self.x2)
# Generate model output
self.model_output_x1 = self.model(self.x1)
self.model_output_x2 = self.model(self.x2)
# Concatenate outputs (so it can be used in the same loss function)
concatted = Concatenate()([
self.model_output_x1, self.model_output_x2, self.embedding_x1,
self.embedding_x2
])
# Connect the inputs with the outputs
self.siamese = Model(inputs=[self.x1, self.x2], outputs=[concatted])
def model_remove_dropout(model_train: keras.Sequential) -> keras.Sequential:
""" Function to remove dropout layer prior to saving Sequential model """
# get layers names and classes
model_layers = model_train.get_config()['layers']
layers_info = {
layer['config']['name']: layer['class_name']
for layer in model_layers
}
if "Dropout" not in layers_info.values():
return model_train
layers_selected = [
layer_name for layer_name, layer_class in layers_info.items()
if layer_class != "Dropout"
]
model = keras.Sequential()
for layer_name in layers_selected:
model.add(model_train.get_layer(name=layer_name))
return model
output_sequence_length=sequence_length)
# Make a text-only dataset (no labels) and call adapt to build the vocabulary.
text_ds = train_ds.map(lambda x, y: x)
vectorize_layer.adapt(text_ds)
#%%
embedding_dim = 16
model = Sequential([
vectorize_layer,
Embedding(vocab_size, embedding_dim, name="embedding"),
GlobalAveragePooling1D(),
Dense(16, activation='relu'),
Dense(1)
])
#%%
# tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir="logs")
model.compile(optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
#%%
model.fit(train_ds, validation_data=val_ds, epochs=15)
#%%
vocab = vectorize_layer.get_vocabulary()
print(vocab[:10])
# Get weights matrix of layer named 'embedding'
weights = model.get_layer('embedding').get_weights()[0]
print(weights.shape)
model.summary()
# Lock the ConvNet Layers
layer_trainable = False
for layer in model.layers:
layer.trainable = layer_trainable
if layer.name == 'flatten':
layer_trainable = True
print(f"{'Layer Name':17} {'Is Trainable?'}")
for layer in model.layers:
print(f"{layer.name:17} {layer.trainable}")
# get the penultimate layer of the model
penult_layer = model.get_layer(name='penult')
# create a new output layer
output_layer = keras.layers.Dense(5, activation='softmax')(penult_layer.output)
new_model = Model(model.input, output_layer)
new_model.summary()
# Now Setup Our Training Images
# We will invert the image, scale to between 0 and 1 and resize to 28x28
new_images = []
width = 28
height = 28
dim = (width, height)
for i in range(len(train_letters)):
img = train_letters[i]
def __soft_inference(self,
x,
parent_prob,
node,
interpret=False,
background=None,
sample=None,
shaps=None):
if node.is_leaf():
return
softmax_dist = node.soft_predict(x)
#print(softmax_dist)
path_next = np.argmax(softmax_dist)
if interpret:
left_w = node.left.weight.reshape((node.left.weight.shape[0], 1))
right_w = node.right.weight.reshape(
(node.right.weight.shape[0], 1))
weights = np.concatenate((left_w, right_w), axis=1)
# create new layer with one node and set the weights
node_layer = Dense(2,
activation='softmax',
use_bias=False,
name='node_layer')
# connect to backbone
backbone = Sequential(self.neural_backbone.layers)
backbone.add(node_layer)
backbone.get_layer('node_layer').set_weights(
list(weights.reshape((1, weights.shape[0], weights.shape[1]))))
#backbone.layers[-1].set_weights(list(weights.reshape((1, weights.shape[0], weights.shape[1]))))
#print(backbone.summary())
#print("NODE OUTPUT TEST: ", backbone(sample))
# run DeepLIFT
e = shap.DeepExplainer(backbone, background)
shap_values = e.shap_values(sample, check_additivity=True)
#shap_values = shap_values[path_next]
#shaps.append(np.absolute(shap_values))
#print(np.argmax(shaps))
shaps += (np.sum(np.absolute(shap_values), axis=0) / 2.0)
#shaps.append(np.sum(shap_values, axis=0) / 2.0) # This could be a memory issue
#if len(shaps) == 0: shaps = np.sum(shap_values, axis=0) / 2.0
#else: shaps += (np.sum(shap_values, axis=0) / 2.0)
#print("TOP SNPs: (MAX: ", np.argmax(abs(shap_values[np.argmax(softmax_dist)])), ")")
# set path probabilities of each child
node.right.path_prob = softmax_dist[1] * parent_prob
node.left.path_prob = softmax_dist[0] * parent_prob
# recursive call on subtrees
self.__soft_inference(x,
node.right.path_prob,
node=node.right,
interpret=interpret,
background=background,
sample=sample,
shaps=shaps)
self.__soft_inference(x,
node.left.path_prob,
node=node.left,
interpret=interpret,
background=background,
sample=sample,
shaps=shaps)
def SATA_autoencoder(train_data,encoding_dim,k,input_dim,nodename,SATA_out,epochs):
encoded = layers.Dense(encoding_dim,
activation="linear",
input_shape=(input_dim,),
use_bias=False,
name ='layer1')
decoded =layers.Dense(input_dim,
activation="linear",
use_bias = False,
kernel_constraint=constraints.UnitNorm(axis=1),
name='layer2')
autoencoder = Sequential()
autoencoder.add(encoded)
autoencoder.add(decoded)
optimizer = optimizers.Adam(lr = 0.0001,beta_1=0.99, beta_2=0.999)
autoencoder.compile(optimizer, loss='MSE')
autoencoder.summary()
history = autoencoder.fit(train_data, train_data,
epochs=epochs,
batch_size=32,
shuffle=True)
#%matplotlib inline
import matplotlib.pyplot as plt
loss = history.history['loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'b', label='Training loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
encoder = autoencoder.get_layer('layer1')
encoder = models.Model(inputs = autoencoder.input, outputs = encoder.output)
encoded_imgs = encoder.predict(train_data)
print(encoded_imgs.shape)
decoded_imgs = autoencoder.predict(train_data)
print(decoded_imgs.shape)
# how many digits will display
n_nodes = encoding_dim
n = encoded_imgs.shape[0]
print(n)
dict_encode = {}
for i in range(encoding_dim):
y = []
for j in range(n):
y.append(encoded_imgs[j][i])
dict_encode[i] = y
squence = np.arange(1,train_data.shape[0]+1,1)
squence = squence.tolist()
fig, ax = plt.subplots(1,n_nodes,figsize=(20,10))
if n_nodes < 2:
x =range(0,n)
plt.plot(x, dict_encode[i], 'r', label='N'+str(i+1))
dataframe = pd.DataFrame({'1':squence,'2':dict_encode[i]})
dataframe.to_csv(SATA_out +".csv",header = 0,index =0, date_format = '%d %.12f')
plt.legend()
return
def build_deep_supervised_autoencoder_model_batchnorm():
leakyReLuLayer = tf.keras.layers.LeakyReLU()
encoder = Sequential()
l = 1e-7
units = []
n_units = len(units)
# input = layers.Input(shape=(14,))
# encoder.add(input)
if not isleakyrelu:
encoder.add(
layers.Dense(units=6,
activation=ACTIVATION,
input_dim=14,
kernel_regularizer=regularizers.l2(l)))
else:
encoder.add(
layers.Dense(units=6,
input_dim=14,
kernel_regularizer=regularizers.l2(l)))
encoder.add(leakyReLuLayer())
encoder.add(layers.Dropout(DROP_OUT_PROB))
encoder.add(layers.BatchNormalization())
for i in range(n_units):
if not isleakyrelu:
encoder.add(
layers.Dense(units=units[i],
activation=ACTIVATION,
kernel_regularizer=regularizers.l2(l)))
else:
encoder.add(
layers.Dense(units=units[i],
kernel_regularizer=regularizers.l2(l)))
encoder.add(leakyReLuLayer())
encoder.add(layers.Dropout(DROP_OUT_PROB)())
encoder.add(layers.BatchNormalization())
if not isleakyrelu:
encoder.add(
layers.Dense(units=4,
activation=ACTIVATION,
kernel_regularizer=regularizers.l2(l)))
else:
encoder.add(
layers.Dense(units=4, kernel_regularizer=regularizers.l2(l)))
encoder.add(leakyReLuLayer())
encoder.add(layers.Dropout(DROP_OUT_PROB))
encoder.add(layers.BatchNormalization(name='latent'))
# decoder
decoder = Sequential()
decoder.add(encoder.get_layer('latent'))
for i in range(n_units):
if not isleakyrelu:
decoder.add(
layers.Dense(units=units[n_units - i - 1],
activation=ACTIVATION,
kernel_regularizer=regularizers.l2(l)))
else:
decoder.add(
layers.Dense(units=units[n_units - i - 1],
kernel_regularizer=regularizers.l2(l)))
decoder.add(leakyReLuLayer())
decoder.add(layers.Dropout(DROP_OUT_PROB))
decoder.add(layers.BatchNormalization())
if not isleakyrelu:
decoder.add(
layers.Dense(units=6,
activation=ACTIVATION,
kernel_regularizer=regularizers.l2(l)))
else:
decoder.add(
layers.Dense(units=6, kernel_regularizer=regularizers.l2(l)))
decoder.add(leakyReLuLayer())
decoder.add(layers.Dropout(DROP_OUT_PROB))
decoder.add(layers.BatchNormalization())
decoder.add(layers.Dense(units=14, name=DECODER))
classifier = Sequential()
classifier.add(encoder.get_layer('latent'))
classifier.add(layers.Dense(units=1, activation='sigmoid',
name=CLASSIFIER))
model = tf.keras.models.Model(inputs=encoder.inputs,
outputs=[decoder.output, classifier.output],
name='deep_supervised_autoencoder')
losses = {DECODER: 'mse', CLASSIFIER: custom_focal_loss}
loss_weights = {DECODER: 0.2, CLASSIFIER: 1.0}
init_lr = 0.0001
adam = Adam(lr=init_lr, decay=init_lr / EPOCHS)
model.compile(optimizer=adam, loss=losses, loss_weights=loss_weights)
plot_model(model, show_shapes=True, to_file='model_graph.png')
model.summary()
return model
class BiLSTM(BaseModel):
_model_name = 'BiLSTMClf'
def __init__(self,
output_dim=None,
input_length=None,
run_time=None,
X_train=None,
y_train=None,
save_path=None,
epochs=None,
batch_size=None,
validation_data=None,
validation_split=None,
verbose=1,
embedding_path=None,
n_words=10000):
super().__init__()
self._output_dim = output_dim
self._input_length = input_length
self._run_time = run_time
self._X_train = X_train
self._y_train = y_train
self._save_path = save_path
self._epochs = epochs
self._batch_size = batch_size
self._validation_data = validation_data
self._validation_split = validation_split
self._verbose = verbose
self._embedding_path = embedding_path
self._n_words = n_words
self._input_dim = ''
self._weights = ''
self._tokenizer = ''
self._model = None
self._history = None
def _create_model(self):
self._model = Sequential([
Embedding(
input_dim=self._input_dim, # vocab_size
output_dim=self._output_dim,
weights=[self._weights], # embedding_matrix
input_length=self._input_length,
name='embeddings')
]) # max_len
self._model.add(Bidirectional(LSTM(64, return_sequences=True)))
self._model.add(GlobalMaxPooling1D())
self._model.add(Dense(16, activation='relu'))
self._model.add(Dropout(0.30))
self._model.add(Dense(6, activation='sigmoid'))
self._model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[
'accuracy'
]) # TODO: change to correct metrics once all ok
def _preprocess_data(self):
self._tokenizer = Tokenizer(num_words=self._n_words, oov_token='<oov>')
self._tokenizer.fit_on_texts(self._X_train)
self._input_length = max(
[len(row) for row in self._X_train]
) if self._input_length is None or self._input_length == 'None' else int(
self._input_length)
self._X_train = self._tokenize_and_pad(self._X_train, self._tokenizer,
self._input_length)
self._input_dim = len(self._tokenizer.word_index) + 1
def _tokenize_and_pad(self,
data,
tokenizer,
maxlen,
padding='post',
truncating='post'):
print('tokenizing data')
data = tokenizer.texts_to_sequences(data)
print('padding tokens')
return pad_sequences(data,
maxlen=maxlen,
padding=padding,
truncating=truncating)
def _get_embeddings(self):
embeddings_index = {}
print('reading pre-trained embeddings')
glove = open(self._embedding_path, 'r', encoding='utf-8')
for line in tqdm(glove):
values = line.split(" ")
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
glove.close()
print('Found %s word vectors.' % len(embeddings_index))
# creating embedding matrix for words dataset
print('creating embedding matrix')
self._weights = np.zeros(
(len(self._tokenizer.word_index) + 1, self._output_dim))
for word, index in tqdm(self._tokenizer.word_index.items()):
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
self._weights[index] = embedding_vector
def get_summary(self):
return self._model.summary()
def train(self):
# preprocess data
print('preprocessing data')
self._preprocess_data()
# get embeddings
print('getting embeddings weights')
self._get_embeddings()
# create model architecture
self._create_model()
summary = self.get_summary()
print('mode summary:', summary)
self._save_path = os.path.join(self._save_path, 'checkpoints',
f'{self._model_name}_{self._run_time}')
create_folder(self._save_path)
cp_callback = ModelCheckpoint(
filepath=self._save_path,
save_weights_only=False,
# verbose=verbose,
save_best_only=True,
monitor='val_loss',
mode='min')
self._history = self._model.fit(
self._X_train,
self._y_train,
epochs=self._epochs,
validation_data=self._validation_data,
validation_split=self._validation_split,
batch_size=self._batch_size,
callbacks=[cp_callback])
def predict(self, X, threshhold=0.5):
pred = self._model.predict(X)
return (pred > threshhold).astype(np.int)
def evaluate(self, X, y):
X = self._tokenize_and_pad(X, self._tokenizer, self._input_length)
return self._model.evaluate(X, y)
def load_model(self, path):
self._model = load_model(path)
def save_model(self, mlflow, path):
path = os.path.join(path, 'saved_models',
f'{self._model_name}_{self._run_time}')
create_folder(path)
self._save_tokenizer(path)
self._save_model(mlflow, path)
self._save_embeddings(path)
def _save_embeddings(self, path):
embeddings = {}
model_embeddings = self._model.get_layer('embeddings').get_weights()[0]
for word, index in self._tokenizer.word_index.items():
embeddings[word] = model_embeddings[index]
with open(os.path.join(path, f'embeddings.pkl'), 'wb') as handle:
pickle.dump(embeddings, handle, protocol=pickle.HIGHEST_PROTOCOL)
def _save_tokenizer(self, path):
tokenizer_json = self._tokenizer.to_json()
with io.open(os.path.join(path, f'tokenizer.json'),
'w',
encoding='utf-8') as f:
f.write(json.dumps(tokenizer_json, ensure_ascii=False))
def _save_model(self, mlflow, path):
mlflow.keras.save_model(self._model, os.path.join(path, f'model'))
