class CartpoleModel(Model): def _build_layers_v2(self, input_dict, num_outputs, options): self.model = Sequential() self.model.add(layers.InputLayer(input_tensor=input_dict["obs"], input_shape=(4,))) self.model.add(layers.Dense(4, name='l1', activation='relu')) self.model.add(layers.Dense(10, name='l2', activation='relu')) self.model.add(layers.Dense(10, name='l3', activation='relu')) self.model.add(layers.Dense(10, name='l4', activation='relu')) self.model.add(layers.Dense(2, name='l5', activation='relu')) return self.model.get_layer("l5").output, self.model.get_layer("l4").output
def define_supernet(members, new_training_set): model = Sequential([ Dense(num_classes, activation='softmax', input_shape=(new_training_set.shape[1],)) ]) model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['accuracy']) all_weights = [m.get_layer(index=-2).get_weights() for m in members] weights = np.concatenate([w for w, _ in all_weights], axis=0) biases = np.mean([b for _, b in all_weights], axis=0) model.get_layer(index=-1).set_weights([weights, biases]) return model
class siamese_model: def __init__(self, input_shape, num_classes, embedding_size): # Convolutional Neural Network self.model = Sequential(name="conv_net") self.model.add( Convolution2D(32, 3, 3, input_shape=input_shape, activation='relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) # 32 filters each. The kernel size is 3*3 self.model.add(Convolution2D(32, 3, 3, activation='relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) # fully connected part self.model.add(Flatten()) self.model.add( Dense(embedding_size, activation='relu', name='embedding')) self.model.add(Dense(num_classes, activation='softmax')) # Define the tensors for the two input images self.x1 = Input(input_shape, name="x1") self.x2 = Input(input_shape, name="x2") # to get the output of the embedding layer, create a sub model from input layer to embedding layer embedding_sub_model = Model( inputs=self.model.input, outputs=self.model.get_layer('embedding').output) self.embedding_x1 = embedding_sub_model(self.x1) self.embedding_x2 = embedding_sub_model(self.x2) # Generate model output self.model_output_x1 = self.model(self.x1) self.model_output_x2 = self.model(self.x2) # Concatenate outputs (so it can be used in the same loss function) concatted = Concatenate()([ self.model_output_x1, self.model_output_x2, self.embedding_x1, self.embedding_x2 ]) # Connect the inputs with the outputs self.siamese = Model(inputs=[self.x1, self.x2], outputs=[concatted])
def model_remove_dropout(model_train: keras.Sequential) -> keras.Sequential: """ Function to remove dropout layer prior to saving Sequential model """ # get layers names and classes model_layers = model_train.get_config()['layers'] layers_info = { layer['config']['name']: layer['class_name'] for layer in model_layers } if "Dropout" not in layers_info.values(): return model_train layers_selected = [ layer_name for layer_name, layer_class in layers_info.items() if layer_class != "Dropout" ] model = keras.Sequential() for layer_name in layers_selected: model.add(model_train.get_layer(name=layer_name)) return model
output_sequence_length=sequence_length) # Make a text-only dataset (no labels) and call adapt to build the vocabulary. text_ds = train_ds.map(lambda x, y: x) vectorize_layer.adapt(text_ds) #%% embedding_dim = 16 model = Sequential([ vectorize_layer, Embedding(vocab_size, embedding_dim, name="embedding"), GlobalAveragePooling1D(), Dense(16, activation='relu'), Dense(1) ]) #%% # tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir="logs") model.compile(optimizer='adam', loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), metrics=['accuracy']) model.summary() #%% model.fit(train_ds, validation_data=val_ds, epochs=15) #%% vocab = vectorize_layer.get_vocabulary() print(vocab[:10]) # Get weights matrix of layer named 'embedding' weights = model.get_layer('embedding').get_weights()[0] print(weights.shape)
model.summary() # Lock the ConvNet Layers layer_trainable = False for layer in model.layers: layer.trainable = layer_trainable if layer.name == 'flatten': layer_trainable = True print(f"{'Layer Name':17} {'Is Trainable?'}") for layer in model.layers: print(f"{layer.name:17} {layer.trainable}") # get the penultimate layer of the model penult_layer = model.get_layer(name='penult') # create a new output layer output_layer = keras.layers.Dense(5, activation='softmax')(penult_layer.output) new_model = Model(model.input, output_layer) new_model.summary() # Now Setup Our Training Images # We will invert the image, scale to between 0 and 1 and resize to 28x28 new_images = [] width = 28 height = 28 dim = (width, height) for i in range(len(train_letters)): img = train_letters[i]
def __soft_inference(self, x, parent_prob, node, interpret=False, background=None, sample=None, shaps=None): if node.is_leaf(): return softmax_dist = node.soft_predict(x) #print(softmax_dist) path_next = np.argmax(softmax_dist) if interpret: left_w = node.left.weight.reshape((node.left.weight.shape[0], 1)) right_w = node.right.weight.reshape( (node.right.weight.shape[0], 1)) weights = np.concatenate((left_w, right_w), axis=1) # create new layer with one node and set the weights node_layer = Dense(2, activation='softmax', use_bias=False, name='node_layer') # connect to backbone backbone = Sequential(self.neural_backbone.layers) backbone.add(node_layer) backbone.get_layer('node_layer').set_weights( list(weights.reshape((1, weights.shape[0], weights.shape[1])))) #backbone.layers[-1].set_weights(list(weights.reshape((1, weights.shape[0], weights.shape[1])))) #print(backbone.summary()) #print("NODE OUTPUT TEST: ", backbone(sample)) # run DeepLIFT e = shap.DeepExplainer(backbone, background) shap_values = e.shap_values(sample, check_additivity=True) #shap_values = shap_values[path_next] #shaps.append(np.absolute(shap_values)) #print(np.argmax(shaps)) shaps += (np.sum(np.absolute(shap_values), axis=0) / 2.0) #shaps.append(np.sum(shap_values, axis=0) / 2.0) # This could be a memory issue #if len(shaps) == 0: shaps = np.sum(shap_values, axis=0) / 2.0 #else: shaps += (np.sum(shap_values, axis=0) / 2.0) #print("TOP SNPs: (MAX: ", np.argmax(abs(shap_values[np.argmax(softmax_dist)])), ")") # set path probabilities of each child node.right.path_prob = softmax_dist[1] * parent_prob node.left.path_prob = softmax_dist[0] * parent_prob # recursive call on subtrees self.__soft_inference(x, node.right.path_prob, node=node.right, interpret=interpret, background=background, sample=sample, shaps=shaps) self.__soft_inference(x, node.left.path_prob, node=node.left, interpret=interpret, background=background, sample=sample, shaps=shaps)
def SATA_autoencoder(train_data,encoding_dim,k,input_dim,nodename,SATA_out,epochs): encoded = layers.Dense(encoding_dim, activation="linear", input_shape=(input_dim,), use_bias=False, name ='layer1') decoded =layers.Dense(input_dim, activation="linear", use_bias = False, kernel_constraint=constraints.UnitNorm(axis=1), name='layer2') autoencoder = Sequential() autoencoder.add(encoded) autoencoder.add(decoded) optimizer = optimizers.Adam(lr = 0.0001,beta_1=0.99, beta_2=0.999) autoencoder.compile(optimizer, loss='MSE') autoencoder.summary() history = autoencoder.fit(train_data, train_data, epochs=epochs, batch_size=32, shuffle=True) #%matplotlib inline import matplotlib.pyplot as plt loss = history.history['loss'] epochs = range(1, len(loss) + 1) plt.plot(epochs, loss, 'b', label='Training loss') plt.xlabel('Epochs') plt.ylabel('Loss') plt.legend() plt.show() encoder = autoencoder.get_layer('layer1') encoder = models.Model(inputs = autoencoder.input, outputs = encoder.output) encoded_imgs = encoder.predict(train_data) print(encoded_imgs.shape) decoded_imgs = autoencoder.predict(train_data) print(decoded_imgs.shape) # how many digits will display n_nodes = encoding_dim n = encoded_imgs.shape[0] print(n) dict_encode = {} for i in range(encoding_dim): y = [] for j in range(n): y.append(encoded_imgs[j][i]) dict_encode[i] = y squence = np.arange(1,train_data.shape[0]+1,1) squence = squence.tolist() fig, ax = plt.subplots(1,n_nodes,figsize=(20,10)) if n_nodes < 2: x =range(0,n) plt.plot(x, dict_encode[i], 'r', label='N'+str(i+1)) dataframe = pd.DataFrame({'1':squence,'2':dict_encode[i]}) dataframe.to_csv(SATA_out +".csv",header = 0,index =0, date_format = '%d %.12f') plt.legend() return
def build_deep_supervised_autoencoder_model_batchnorm(): leakyReLuLayer = tf.keras.layers.LeakyReLU() encoder = Sequential() l = 1e-7 units = [] n_units = len(units) # input = layers.Input(shape=(14,)) # encoder.add(input) if not isleakyrelu: encoder.add( layers.Dense(units=6, activation=ACTIVATION, input_dim=14, kernel_regularizer=regularizers.l2(l))) else: encoder.add( layers.Dense(units=6, input_dim=14, kernel_regularizer=regularizers.l2(l))) encoder.add(leakyReLuLayer()) encoder.add(layers.Dropout(DROP_OUT_PROB)) encoder.add(layers.BatchNormalization()) for i in range(n_units): if not isleakyrelu: encoder.add( layers.Dense(units=units[i], activation=ACTIVATION, kernel_regularizer=regularizers.l2(l))) else: encoder.add( layers.Dense(units=units[i], kernel_regularizer=regularizers.l2(l))) encoder.add(leakyReLuLayer()) encoder.add(layers.Dropout(DROP_OUT_PROB)()) encoder.add(layers.BatchNormalization()) if not isleakyrelu: encoder.add( layers.Dense(units=4, activation=ACTIVATION, kernel_regularizer=regularizers.l2(l))) else: encoder.add( layers.Dense(units=4, kernel_regularizer=regularizers.l2(l))) encoder.add(leakyReLuLayer()) encoder.add(layers.Dropout(DROP_OUT_PROB)) encoder.add(layers.BatchNormalization(name='latent')) # decoder decoder = Sequential() decoder.add(encoder.get_layer('latent')) for i in range(n_units): if not isleakyrelu: decoder.add( layers.Dense(units=units[n_units - i - 1], activation=ACTIVATION, kernel_regularizer=regularizers.l2(l))) else: decoder.add( layers.Dense(units=units[n_units - i - 1], kernel_regularizer=regularizers.l2(l))) decoder.add(leakyReLuLayer()) decoder.add(layers.Dropout(DROP_OUT_PROB)) decoder.add(layers.BatchNormalization()) if not isleakyrelu: decoder.add( layers.Dense(units=6, activation=ACTIVATION, kernel_regularizer=regularizers.l2(l))) else: decoder.add( layers.Dense(units=6, kernel_regularizer=regularizers.l2(l))) decoder.add(leakyReLuLayer()) decoder.add(layers.Dropout(DROP_OUT_PROB)) decoder.add(layers.BatchNormalization()) decoder.add(layers.Dense(units=14, name=DECODER)) classifier = Sequential() classifier.add(encoder.get_layer('latent')) classifier.add(layers.Dense(units=1, activation='sigmoid', name=CLASSIFIER)) model = tf.keras.models.Model(inputs=encoder.inputs, outputs=[decoder.output, classifier.output], name='deep_supervised_autoencoder') losses = {DECODER: 'mse', CLASSIFIER: custom_focal_loss} loss_weights = {DECODER: 0.2, CLASSIFIER: 1.0} init_lr = 0.0001 adam = Adam(lr=init_lr, decay=init_lr / EPOCHS) model.compile(optimizer=adam, loss=losses, loss_weights=loss_weights) plot_model(model, show_shapes=True, to_file='model_graph.png') model.summary() return model
class BiLSTM(BaseModel): _model_name = 'BiLSTMClf' def __init__(self, output_dim=None, input_length=None, run_time=None, X_train=None, y_train=None, save_path=None, epochs=None, batch_size=None, validation_data=None, validation_split=None, verbose=1, embedding_path=None, n_words=10000): super().__init__() self._output_dim = output_dim self._input_length = input_length self._run_time = run_time self._X_train = X_train self._y_train = y_train self._save_path = save_path self._epochs = epochs self._batch_size = batch_size self._validation_data = validation_data self._validation_split = validation_split self._verbose = verbose self._embedding_path = embedding_path self._n_words = n_words self._input_dim = '' self._weights = '' self._tokenizer = '' self._model = None self._history = None def _create_model(self): self._model = Sequential([ Embedding( input_dim=self._input_dim, # vocab_size output_dim=self._output_dim, weights=[self._weights], # embedding_matrix input_length=self._input_length, name='embeddings') ]) # max_len self._model.add(Bidirectional(LSTM(64, return_sequences=True))) self._model.add(GlobalMaxPooling1D()) self._model.add(Dense(16, activation='relu')) self._model.add(Dropout(0.30)) self._model.add(Dense(6, activation='sigmoid')) self._model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[ 'accuracy' ]) # TODO: change to correct metrics once all ok def _preprocess_data(self): self._tokenizer = Tokenizer(num_words=self._n_words, oov_token='<oov>') self._tokenizer.fit_on_texts(self._X_train) self._input_length = max( [len(row) for row in self._X_train] ) if self._input_length is None or self._input_length == 'None' else int( self._input_length) self._X_train = self._tokenize_and_pad(self._X_train, self._tokenizer, self._input_length) self._input_dim = len(self._tokenizer.word_index) + 1 def _tokenize_and_pad(self, data, tokenizer, maxlen, padding='post', truncating='post'): print('tokenizing data') data = tokenizer.texts_to_sequences(data) print('padding tokens') return pad_sequences(data, maxlen=maxlen, padding=padding, truncating=truncating) def _get_embeddings(self): embeddings_index = {} print('reading pre-trained embeddings') glove = open(self._embedding_path, 'r', encoding='utf-8') for line in tqdm(glove): values = line.split(" ") word = values[0] coefs = np.asarray(values[1:], dtype='float32') embeddings_index[word] = coefs glove.close() print('Found %s word vectors.' % len(embeddings_index)) # creating embedding matrix for words dataset print('creating embedding matrix') self._weights = np.zeros( (len(self._tokenizer.word_index) + 1, self._output_dim)) for word, index in tqdm(self._tokenizer.word_index.items()): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: self._weights[index] = embedding_vector def get_summary(self): return self._model.summary() def train(self): # preprocess data print('preprocessing data') self._preprocess_data() # get embeddings print('getting embeddings weights') self._get_embeddings() # create model architecture self._create_model() summary = self.get_summary() print('mode summary:', summary) self._save_path = os.path.join(self._save_path, 'checkpoints', f'{self._model_name}_{self._run_time}') create_folder(self._save_path) cp_callback = ModelCheckpoint( filepath=self._save_path, save_weights_only=False, # verbose=verbose, save_best_only=True, monitor='val_loss', mode='min') self._history = self._model.fit( self._X_train, self._y_train, epochs=self._epochs, validation_data=self._validation_data, validation_split=self._validation_split, batch_size=self._batch_size, callbacks=[cp_callback]) def predict(self, X, threshhold=0.5): pred = self._model.predict(X) return (pred > threshhold).astype(np.int) def evaluate(self, X, y): X = self._tokenize_and_pad(X, self._tokenizer, self._input_length) return self._model.evaluate(X, y) def load_model(self, path): self._model = load_model(path) def save_model(self, mlflow, path): path = os.path.join(path, 'saved_models', f'{self._model_name}_{self._run_time}') create_folder(path) self._save_tokenizer(path) self._save_model(mlflow, path) self._save_embeddings(path) def _save_embeddings(self, path): embeddings = {} model_embeddings = self._model.get_layer('embeddings').get_weights()[0] for word, index in self._tokenizer.word_index.items(): embeddings[word] = model_embeddings[index] with open(os.path.join(path, f'embeddings.pkl'), 'wb') as handle: pickle.dump(embeddings, handle, protocol=pickle.HIGHEST_PROTOCOL) def _save_tokenizer(self, path): tokenizer_json = self._tokenizer.to_json() with io.open(os.path.join(path, f'tokenizer.json'), 'w', encoding='utf-8') as f: f.write(json.dumps(tokenizer_json, ensure_ascii=False)) def _save_model(self, mlflow, path): mlflow.keras.save_model(self._model, os.path.join(path, f'model'))