shear_range=0.15, horizontal_flip=True, fill_mode="nearest") # load the MobileNetV2 network, classifying layer off mobile_net = MobileNetV2(weights="imagenet", include_top=False, input_tensor=Input(shape=(224, 224, 3))) custom_net = mobile_net.output custom_net = AveragePooling2D(pool_size=(7, 7))(custom_net) custom_net = Flatten(name="flatten")(custom_net) custom_net = Dense(128, activation="relu")(custom_net) custom_net = Dropout(0.5)(custom_net) custom_net = Dense(2, activation="softmax")(custom_net) # sequencing the network model = Model(inputs=mobile_net.input, outputs=custom_net) # Only train the custom section of the network for layer in mobile_net.layers: layer.trainable = False # compile our model model.compile(loss="binary_crossentropy", optimizer=Adam(lr=init_lr, decay=init_lr / epochs), metrics=["accuracy"]) # train the head of the network train_custom = model.fit( image_data_generator.flow(train_x, train_y, batch_size=batch_size), steps_per_epoch=len(train_x) // batch_size, validation_data=(test_x, test_y), validation_steps=len(test_x) // batch_size, epochs=epochs)
def compile_model(network):
""" Compile a sequential model.
Args:
network (dict): the parameters of the network
Returns:
a compiled network.
Note the input shape here is considered to be (1, 5) and loss function is set to MSE.
Modify if necessary
"""
# Get the network parameters.
n_layers = network[
'n_layers'] # Note n_layers is the number of hidden layers.
layer_info = network['layer_info']
optimizer = network['optimizer']
final_act = network['final_act']
# Set the number of input and output features and time step.
input_features = 5
output_features = 1
time_step = 1
# Add input layer
inputs = Input(shape=(time_step, input_features))
# Add each layer
if n_layers == 0:
# If n_layers == 0, flatten and jump straight to the output layer.
hidden_layer = Reshape((input_features, ))(inputs)
elif n_layers > 0:
# If n_layers > 0, loop through layer_info.
for i in range(n_layers):
if i == 0:
# For the first hidden layer, specify the layer input as 'inputs'
if layer_info[i][0] == 'Dense':
hidden_layer = TimeDistributed(
Dense(layer_info[i][1],
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01),
use_bias=False))(inputs)
hidden_layer = Activation(layer_info[i][2])(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'LSTM':
hidden_layer = LSTM(layer_info[i][1],
return_sequences=True,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01),
use_bias=False)(inputs)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'GRU':
hidden_layer = GRU(layer_info[i][1],
return_sequences=True,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01),
use_bias=False)(inputs)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif i > 0:
# For the next hidden layers, simply add them along with the batch normalization and dropout.
if layer_info[i][0] == 'Dense':
hidden_layer = TimeDistributed(
Dense(layer_info[i][1],
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01)))(hidden_layer)
hidden_layer = Activation(layer_info[i][2])(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'LSTM':
hidden_layer = LSTM(
layer_info[i][1],
return_sequences=True,
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01))(hidden_layer)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'GRU':
hidden_layer = GRU(
layer_info[i][1],
return_sequences=True,
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01))(hidden_layer)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
# Add the flattening layer
hidden_layer = Flatten()(hidden_layer)
hidden_layer = Dense(output_features,
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01))(hidden_layer)
outputs = Activation(final_act)(hidden_layer)
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss='mean_squared_error', optimizer=optimizer)
print(network_arch(network))
return model
def build_mdl_lst(org_mdl, prev_out_shape, indices_to_tls):
"""
New
"""
import tensorflow as tf
from tensorflow.keras.models import Model
import numpy as np
import tensorflow.keras.backend as K
from collections.abc import Iterable
mdl = tf.keras.models.clone_model(org_mdl)
# dictionary to describe the network graph
network_dict = {'input_layers_of': {}, 'new_output_tensor_of': {}}
# Set the input layers of each layer
for layer in mdl.layers:
for node in layer._outbound_nodes: # the output node of the current layer (layer)
layer_name = node.outbound_layer.name # the layer that take node as input
# layer_name takes layer.name as input
if layer_name not in network_dict['input_layers_of']:
network_dict['input_layers_of'].update({layer_name: [layer.name]})
else:
network_dict['input_layers_of'][layer_name].append(layer.name)
min_idx_to_tl = np.min([idx if not isinstance(idx, Iterable) else idx[0] for idx in indices_to_tls])
num_layers = len(mdl.layers)
model_input = tf.keras.Input(shape = prev_out_shape)
# Iterate over all layers after the input
if min_idx_to_tl == 0:
layer_name = mdl.layers[0].name
# if the previous layer (layer_name) is an input layer
if model_util.is_Input(type(mdl.layers[0]).__name__):
# set model_input as the output of this input layer
network_dict['new_output_tensor_of'].update({layer_name: model_input})
else: # if it is not (happen when using Sequential())
_input_layer_name = 'input_layer' # -> insert one addiitonal input layer
# x is the output of _input_layer_name
network_dict['new_output_tensor_of'].update({_input_layer_name: model_input})
# the input's of layer_name is _input_layer_name
network_dict['input_layers_of'].update({layer_name: [_input_layer_name]})
else:
network_dict['new_output_tensor_of'].update({mdl.layers[min_idx_to_tl-1].name: model_input})
for idx_to_l in range(min_idx_to_tl, num_layers):
layer = mdl.layers[idx_to_l]
layer_name = layer.name
# Determine input tensors
layer_input = []
for layer_aux in network_dict['input_layers_of'][layer.name]:
layer_input.append(network_dict['new_output_tensor_of'][layer_aux])
if len(layer_input) == 1:
layer_input = layer_input[0]
x = layer(layer_input)
# Set new output tensor (the original one, or the one of the replaced layer)
# x is the output of the layer "layer_name" (current one)
network_dict['new_output_tensor_of'].update({layer_name: x})
last_layer_name = mdl.layers[-1].name
mdl = Model(inputs = model_input, # this is a constant
outputs = network_dict['new_output_tensor_of'][last_layer_name])
return mdl
def cae_model():
## Encoder
encoder_inputs = Input(shape=(1024, 1024, 1), name='Field')
# Encode
x = Conv2D(30, kernel_size=(3, 3), activation='relu',
padding='same')(encoder_inputs)
enc_l2 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(25, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l2)
enc_l3 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(20, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l3)
enc_l4 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(15, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l4)
enc_l5 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(10, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l5)
enc_l6 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(5, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l6)
enc_l7 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(3, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l7)
enc_l8 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(2, kernel_size=(3, 3), activation='relu',
padding='same')(enc_l8)
enc_l9 = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
x = Conv2D(1, kernel_size=(3, 3), activation=None, padding='same')(enc_l9)
encoded = MaxPooling2D(pool_size=(2, 2), padding='same')(x)
encoder = Model(inputs=encoder_inputs, outputs=encoded)
## Decoder
decoder_inputs = Input(shape=(2, 2, 1), name='decoded')
x = Conv2D(1, kernel_size=(3, 3), activation=None,
padding='same')(decoder_inputs)
dec_l0 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(2, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l0)
dec_l1 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(3, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l1)
dec_l2 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(5, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l2)
dec_l3 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(10, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l3)
dec_l4 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(15, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l4)
dec_l5 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(20, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l5)
dec_l6 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(25, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l6)
dec_l7 = UpSampling2D(size=(2, 2))(x)
x = Conv2D(30, kernel_size=(3, 3), activation='relu',
padding='same')(dec_l7)
dec_l8 = UpSampling2D(size=(2, 2))(x)
decoded = Conv2D(1, kernel_size=(3, 3), activation=None,
padding='same')(dec_l8)
decoder = Model(inputs=decoder_inputs, outputs=decoded)
## Autoencoder
ae_outputs = decoder(encoder(encoder_inputs))
model = Model(inputs=encoder_inputs, outputs=ae_outputs, name='CAE')
return model, encoder, decoder
kernel_size=2,
padding="same",
activation=gelu,
strides=1)(x)
x = Conv1D(filters=128,
kernel_size=3,
padding="same",
activation=gelu,
strides=1)(x)
x = AttentionPooling1D(hdims=128)(x, mask=mask)
x = Dropout(0.2)(x)
x = Dense(128)(x)
x = gelu(x)
outputs = Dense(num_classes, activation="softmax")(x)
model = Model(inputs, outputs)
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.summary()
# 训练
batch_size = 32
epochs = 5
callbacks = [SaveBestModelOnMemory()]
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=callbacks,
validation_split=0.1)
def __init__(self, layer='block5_pool'):
self._base = VGG19(include_top=False, weights='imagenet')
self._model = Model(inputs=self._base.input,
outputs=self._base.get_layer(layer).output)
self._seed = np.random.seed(2501) # important! must extract the same positions for each image
self._indices = np.random.permutation(7*7*512) # shuffle indices
#2 모델구성
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Input, Dense
input1 = Input(shape=(13, ))
dense1 = Dense(120, activation='relu')(input1)
dense1 = Dense(80)(dense1)
dense1 = Dense(60)(dense1)
dense1 = Dense(30)(dense1)
dense1 = Dense(7)(dense1)
dense1 = Dense(7)(dense1)
dense1 = Dense(5)(dense1)
dense1 = Dense(4)(dense1)
output1 = Dense(1)(dense1)
model = Model(inputs=input1, outputs=output1)
#3 컴파일 훈련
model.compile(loss='mse', optimizer='adam', metrics=['mae'])
model.fit(x_train,
y_train,
epochs=1000,
batch_size=4,
validation_split=0.2,
verbose=1)
#4평가 예측
loss, mae = model.evaluate(x_test, y_test, batch_size=4)
print('loss,mae : ', loss, mae)
y_predict = model.predict(x_test)
X = []
for f in tqdm(os.listdir(ROOT)):
im = imread(os.path.join(ROOT, f))
if im.shape[-1] > 3:
im = rgba2rgb(im)
im = resize(im, (224, 224)) # NOQA: E912
X.append(im)
X = np.array(X)
with h5py.File('thumbnails.h5', 'w') as fout:
fout.create_dataset('thumbnails', data=X)
model = load_model('vgg16-validated-five-classes.h5')
encoder = Model(model.layers[1].input, model.layers[1].output)
x = imread('/tmp/choropleth.png')
x = rgba2rgb(x) # if n_channels is > 3
x = resize(x, (224, 224)) # NOQA: E912
xAct = encoder(x)
# Get nearest neighbors
thumbnailsEncoded = encoder(X)
nn = NearestNeighbors(n_neighbors=10, n_jobs=-1)
nn.fit(thumbnailsEncoded)
y = nn.kneighbors(xAct, return_distance=False)
tsne = TSNE(n_components=2)
thumb_reduced = tsne.fit_transform(thumbnailsEncoded.T)
thumb_reduced = tsne.fit_transform(tf.transpose(thumbnailsEncoded))
x = layers.Dense(40)(x)
x = layers.Dropout(.11)(x)
x = layers.Dense(40)(x)
x = layers.Dropout(.159)(x)
output = layers.Dense(1, activation='sigmoid', name='output')(x)
checkpoint = ModelCheckpoint("./CNN_saves/cnn.ckpt",
monitor='loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
model = Model(inputs=[title_inputs, body_inputs],
outputs=[output],
name='CNN_model')
model.compile(loss=BinaryCrossentropy(),
optimizer='Adam',
metrics=['accuracy'])
print("begin training... \n")
history = model.fit([train_title, train_body],
train_labels,
batch_size=32,
epochs=4,
validation_split=0.3,
shuffle=True,
callbacks=[checkpoint])
def auto_encoder(width=None,
height=None,
depth=None,
filters=None,
latentDim=None):
def conv_2d(x, f, transpose, chanDim):
if transpose:
x = Conv2DTranspose(f, (3, 3), strides=2, padding="same")(x)
else:
x = Conv2D(f, (3, 3), strides=2, padding="same")(x)
x = LeakyReLU(alpha=0.2)(x)
x = BatchNormalization(axis=chanDim)(x)
return x
# initialize the input shape to be "channels last" along with
# the channels dimension itself
# channels dimension itself
base_model = DenseNet121(input_shape=(224, 224, 3),
weights='imagenet',
include_top=True)
for layer in base_model.layers:
layer.trainable = False
feature = base_model.layers[-2].output
#feature = Dense(512, activation='relu')(feature)
inputShape = (height, width, depth)
chanDim = -1
# define the input to the encoder
inputs = base_model.input
x = inputs
# loop over the number of filters
x = conv_2d(x, 8, 0, chanDim)
x1 = x
x = conv_2d(x, 16, 0, chanDim)
x2 = x
x = conv_2d(x, 32, 0, chanDim)
x3 = x
# flatten the network and then construct our latent vector
volumeSize = K.int_shape(x)
x = Flatten()(x)
latent = Dense(1024, name="encoded")(x)
latent = add([feature, latent])
x = Dense(np.prod(volumeSize[1:]))(latent)
x = Reshape((volumeSize[1], volumeSize[2], volumeSize[3]))(x)
# loop over our number of filters again, but this time in
# reverse order
x = concatenate([x, x3])
x = conv_2d(x, 32, 1, chanDim)
x = concatenate([x, x2])
x = conv_2d(x, 16, 1, chanDim)
x = concatenate([x, x1])
x = conv_2d(x, 3, 1, chanDim)
#x = conv_2d(x,3,1,chanDim)
#x = concatenate([x,inputs])
outputs = Conv2DTranspose(3, (3, 3), padding="same")(x)
outputs = Activation("sigmoid", name="decoded")(x)
# construct our autoencoder model
autoencoder = Model(inputs, outputs, name="autoencoder")
# return the autoencoder model
autoencoder.compile(optimizer='adam', loss='mse')
return autoencoder
def train_model(self,
sentences_pair,
is_similar,
embedding_meta_data,
model_save_directory='./'):
"""
Train Siamese network to find similarity between sentences in `sentences_pair`
Steps Involved:
1. Pass the each from sentences_pairs to bidirectional LSTM encoder.
2. Merge the vectors from LSTM encodes and passed to dense layer.
3. Pass the dense layer vectors to sigmoid output layer.
4. Use cross entropy loss to train weights
Args:
sentences_pair (list): list of tuple of sentence pairs
is_similar (list): target value 1 if same sentences pair are similar otherwise 0
embedding_meta_data (dict): dict containing tokenizer and word embedding matrix
model_save_directory (str): working directory for where to save models
Returns:
return (best_model_path): path of best model
"""
tokenizer, embedding_matrix = embedding_meta_data[
'tokenizer'], embedding_meta_data['embedding_matrix']
train_data_x1, train_data_x2, train_labels, leaks_train, \
val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, sentences_pair,
is_similar, self.max_sequence_length,
self.validation_split_ratio)
if train_data_x1 is None:
print("++++ !! Failure: Unable to train model ++++")
return None
nb_words = len(tokenizer.word_index) + 1
# Creating word embedding layer
embedding_layer = Embedding(nb_words,
self.embedding_dim,
weights=[embedding_matrix],
input_length=self.max_sequence_length,
trainable=False)
# Creating LSTM Encoder
lstm_layer = Bidirectional(
LSTM(self.number_lstm_units,
dropout=self.rate_drop_lstm,
recurrent_dropout=self.rate_drop_lstm))
# Creating LSTM Encoder layer for First Sentence
sequence_1_input = Input(shape=(self.max_sequence_length, ),
dtype='int32')
embedded_sequences_1 = embedding_layer(sequence_1_input)
x1 = lstm_layer(embedded_sequences_1)
# Creating LSTM Encoder layer for Second Sentence
sequence_2_input = Input(shape=(self.max_sequence_length, ),
dtype='int32')
embedded_sequences_2 = embedding_layer(sequence_2_input)
x2 = lstm_layer(embedded_sequences_2)
# Creating leaks input
leaks_input = Input(shape=(leaks_train.shape[1], ))
leaks_dense = Dense(int(self.number_dense_units / 2),
activation=self.activation_function)(leaks_input)
# Merging two LSTM encodes vectors from sentences to
# pass it to dense layer applying dropout and batch normalisation
merged = concatenate([x1, x2, leaks_dense])
merged = BatchNormalization()(merged)
merged = Dropout(self.rate_drop_dense)(merged)
merged = Dense(self.number_dense_units,
activation=self.activation_function)(merged)
merged = BatchNormalization()(merged)
merged = Dropout(self.rate_drop_dense)(merged)
preds = Dense(1, activation='sigmoid')(merged)
model = Model(inputs=[sequence_1_input, sequence_2_input, leaks_input],
outputs=preds)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['acc'])
early_stopping = EarlyStopping(monitor='val_loss', patience=3)
STAMP = 'lstm_%d_%d_%.2f_%.2f' % (
self.number_lstm_units, self.number_dense_units,
self.rate_drop_lstm, self.rate_drop_dense)
checkpoint_dir = model_save_directory + 'checkpoints/' + str(
int(time.time())) + '/'
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
bst_model_path = checkpoint_dir + STAMP + '.h5'
model_checkpoint = ModelCheckpoint(bst_model_path,
save_best_only=True,
save_weights_only=False)
tensorboard = TensorBoard(log_dir=checkpoint_dir +
"logs/{}".format(time.time()))
model.fit([train_data_x1, train_data_x2, leaks_train],
train_labels,
validation_data=([val_data_x1, val_data_x2,
leaks_val], val_labels),
epochs=200,
batch_size=64,
shuffle=True,
callbacks=[early_stopping, model_checkpoint, tensorboard])
return bst_model_path
def __init__(self, batch_size=200):
base_model = InceptionV3(weights='imagenet')
self.model = Model(inputs=base_model.input, outputs=base_model.get_layer('avg_pool').output)
self.batch_size = batch_size
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
rct, thr = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY)
contours, hierachy = cv2.findContours(thr, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
x, y, w, h = cv2.boundingRect(contours[0])
len = w if w > h else h
dst = img[y : y+len, x: x+len]
return dst
print(' [STEP 2] 모델 불러오기')
inputs = Input(shape=(FLG.HEIGHT, FLG.WIDTH, FLG.DEPTH))
defectBranch = SmallerVGGNetBuilder.vggNet_output(inputs=inputs, classes=5, finalAct="sigmoid", output_names='defect')
lacunaBranch = SmallerVGGNetBuilder.vggNet_output(inputs=inputs, classes=5, finalAct="sigmoid", output_names='lacuna')
spokeBranch = SmallerVGGNetBuilder.vggNet_output(inputs=inputs, classes=5, finalAct="sigmoid", output_names='spoke')
spotBranch = SmallerVGGNetBuilder.vggNet_output(inputs=inputs, classes=5, finalAct="sigmoid", output_names='spot')
model = Model(inputs=inputs, outputs=[defectBranch, lacunaBranch, spokeBranch, spotBranch])
# h5_weights_path = './output/iris_pattern_300_32/modelsaved/h5/iris_pattern_300_32_weights.h5'
h5_weights_path ='./output/multi_outout_smallvgg_L2_aug_hh_300_16/modelsaved/h5/model_weights.h5'
model.load_weights(h5_weights_path)
losses = {"defect": sparse_categorical_crossentropy, "lacuna": sparse_categorical_crossentropy, "spoke": sparse_categorical_crossentropy, "spot": sparse_categorical_crossentropy}
lossWeights = {"defect": 1.0, "lacuna": 1.0, "spoke": 1.0, "spot": 1.0}
print('# 엮기(compile)')
model.compile(loss=losses, optimizer='rmsprop', metrics=['accuracy'], loss_weights=lossWeights)
print('[STEP 1] 이미지 불러오기')
datas = []
origs = []
def create_model(
self,
lr,
type="vanilla",
rescale_value=255.0,
):
""" Builds the DQN Agent architecture.
Source:https://cs.corp.google.com/piper///depot/google3/third_party/py/
dopamine/agents/dqn/dqn_agent.py?q=DQN&dr=CSs&l=15
This initializes the model as per the specifications mentioned in the
DQN paper by Deepmind. This is a sequential model implemention of
tf.keras. The compiled model is returned by the Method.
Args:
Returns:
Model: Compiled Model
"""
#with tf.device('/gpu:0'):
self.image_frames = Input(shape=(self.height, self.width, self.channels))
#self.normalize = Lambda(lambda input: input/255.0)
self.conv1 = Conv2D(
filters=32,
kernel_size=(8, 8),
strides=(4, 4),
activation="relu",
name="conv1")(
Lambda(lambda input: input / float(rescale_value))(
self.image_frames))
self.conv2 = Conv2D(
filters=64,
kernel_size=(4, 4),
strides=(2, 2),
activation="relu",
name="conv2")(
self.conv1)
self.conv3 = Conv2D(
filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation="relu",
name="conv3")(
self.conv2)
self.flattened = Flatten(name="flattened")(self.conv3)
self.fully_connected_1 = Dense(
units=512, activation="relu", name="fully_connected_1")(
self.flattened)
self.q_values = Dense(
units=self.actions, activation="linear", name="q_values")(
self.fully_connected_1)
self.model = Model(inputs=[self.image_frames], outputs=[self.q_values])
self.optimizer = Adam(lr=lr)
if self.loss == "huber":
self.loss = huber_loss
K.get_session().run(tf.global_variables_initializer())
def reward(y_true, y_pred):
return self.reward_tensor
self.model.compile(
optimizer=self.optimizer, loss=self.loss, metrics=["mse", reward])
return self.model
baseModel = MobileNetV2(weights="imagenet",
include_top=False,
input_tensor=Input(shape=(224, 224, 3)))
# construct the head of the model that will be placed on top of the
# the base model
headModel = baseModel.output
headModel = AveragePooling2D(pool_size=(7, 7))(headModel)
headModel = Flatten(name="flatten")(headModel)
headModel = Dense(128, activation="relu")(headModel)
headModel = Dropout(0.5)(headModel)
headModel = Dense(2, activation="softmax")(headModel)
# place the head FC model on top of the base model (this will become
# the actual model we will train)
model = Model(inputs=baseModel.input, outputs=headModel)
# loop over all layers in the base model and freeze them so they will
# *not* be updated during the first training process
for layer in baseModel.layers:
layer.trainable = False
# compile our model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"])
# train the head of the network
print("[INFO] training head...")
H = model.fit(aug.flow(trainX, trainY, batch_size=BS),
steps_per_epoch=len(trainX) // BS,
window_length = data.shape[1]
#TODO: Normalize Data
#Encoder
input_window = Input(shape=(window_length, 3))
x = Conv1D(16, 3, activation="relu",
padding="same")(input_window) # Full Dimension
x = BatchNormalization()(x)
x = MaxPooling1D(3, padding="same")(x)
x = Conv1D(1, 3, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
encoded = MaxPooling1D(2, padding="same")(
x) # 3 dims... I'm not super convinced this is actually 3 dimensions
encoder = Model(input_window, encoded)
# 3 dimensions in the encoded layer
x = Conv1D(1, 3, activation="relu", padding="same")(encoded) # Latent space
x = BatchNormalization()(x)
x = UpSampling1D(2)(x) # 6 dims
x = Conv1D(16, 3, activation='relu', padding='same')(x) # 5 dims
x = BatchNormalization()(x)
x = UpSampling1D(3)(x) # 10 dims
decoded = Conv1D(3, 3, activation='sigmoid', padding='same')(x) # 10 dims
autoencoder = Model(input_window, decoded)
autoencoder.summary()
x_train = data
epochs = 2
def region_model():
region, region_input = MobileDetectNetModel.region()
return Model(inputs=region_input, outputs=region)
def getModel():
# Input head of the model
Input1 = InputLayer(input_shape=(
30,
30,
5,
))
Input2 = InputLayer(input_shape=(
30,
30,
17,
))
Input3 = InputLayer(input_shape=(
30,
30,
5,
))
Input4 = InputLayer(input_shape=(30, 30, 5))
# Convolution segment for VIIRS input
Part1Conv1 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Input1.output)
Part1Conv1 = AveragePooling2D()(Part1Conv1)
Part1Conv2 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Part1Conv1)
Part1Flat = Flatten()(Part1Conv2)
# Convolution segment for Geographical input
Part2Conv1 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Input2.output)
Part2Conv1 = AveragePooling2D()(Part2Conv1)
Part2Conv2 = Conv2D(128, 3, activation='relu',
data_format='channels_last')(Part2Conv1)
Part2Flat = Flatten()(Part2Conv2)
# Convolution segment for Weather (T=0) input
Part3Conv1 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Input3.output)
Part3Conv1 = AveragePooling2D()(Part3Conv1)
Part3Conv2 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Part3Conv1)
Part3Flat = Flatten()(Part3Conv2)
# Convolution segment for Weather (T=12) input
Part4Conv1 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Input4.output)
Part4Conv1 = AveragePooling2D()(Part4Conv1)
Part4Conv2 = Conv2D(64, 3, activation='relu',
data_format='channels_last')(Part4Conv1)
Part4Flat = Flatten()(Part4Conv2)
# Projecting all features into a common embedding space for T=12 prediction
merge_layer = concatenate([Part1Flat, Part2Flat, Part3Flat])
Dense1 = Dense(units=2048, activation='relu')(merge_layer)
Dense1 = Dropout(rate=0.2)(Dense1)
Dense2 = Dense(units=1024, activation='relu')(Dense1)
Dense2 = Dropout(rate=0.2)(Dense2)
output = Dense(units=900, activation='sigmoid')(Dense2)
output_1 = Reshape((30, 30), name="t12_output")(output)
# Projecting all features into common embedding space for T=24 prediction
merge_layer_2 = concatenate([merge_layer, Part4Flat])
Dense_1 = Dense(units=2048, activation='relu')(merge_layer_2)
Dense_1 = Dropout(rate=0.2)(Dense_1)
Dense_2 = Dense(units=1024, activation='relu')(Dense_1)
Dense_2 = Dropout(rate=0.2)(Dense_2)
output_2 = Dense(units=900, activation='sigmoid')(Dense_2)
output_2 = Reshape((30, 30), name="t24_output")(output_2)
model = Model(
inputs=[Input1.input, Input2.input, Input3.input, Input4.input],
outputs=[output_1, output_2])
model.summary()
return model
class CurrencyDetector:
"""
1. Class for making and training the model on a given ```Image Classification``` dataset.
2. It takes in train path,test path,input shape and labels as params.(__init__)
3. Builds the model(make_vgg16_model) and trains(train_model) it.
4. Also provides functionality for visualization of dataset(explore_dataset) and training(plot_train)
5. Can be used for any type of dataset or anytype of pretrained model.
"""
def __init__(self,
train_path: str,
test_path: str,
shape: list = [196, 196],
labels: int = 7):
"""
Constructor for class.
Params:
train_path(str): path to the training directory
test_path(str): path to the testing directory
shape(list): input shape for the model
labels: no of labels in the dataset
"""
self.shape = shape
self.train_path = train_path # path to training data
self.test_path = test_path # path to testing/validation data
self.labels = labels # no of labels
self.train_gen = ImageDataGenerator(
preprocessing_function=preprocess_input,
zoom_range=0.2,
shear_range=0.2,
horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.4,
height_shift_range=0.4,
brightness_range=[0.4, 1.0],
rotation_range=40)
# Image generator object for training data
self.val_gen = ImageDataGenerator(
preprocessing_function=preprocess_input)
# Image generator object for validation data
self.train_data_gen = self.train_gen.flow_from_directory(
self.train_path,
target_size=self.shape,
class_mode='categorical',
batch_size=32)
self.val_data_gen = self.train_gen.flow_from_directory(
self.test_path,
target_size=self.shape,
class_mode='categorical',
batch_size=32)
self.make_vgg16_model() # will store model object
def explore_dataset(self):
"""
Method for exploratory analysis of the data.
Params:
None
"""
pass
def make_vgg16_model(self):
"""
Method to intialize vgg16 model with custom layers for fine tuning it.
params:
None
"""
print("inside model building function")
vgg = VGG16(weights='imagenet',
include_top=False,
input_shape=tuple(self.shape + [3]))
for layer in vgg.layers:
layer.trainable = False #making all the layers non-trainable
x = Flatten()(vgg.output) #flattening out the last layer
predictions = Dense(self.labels, activation='softmax')(
x) #Dense layer to predict wether their is pneumonia or not
self.model = Model(inputs=vgg.input,
outputs=predictions) # making the model
self.model.summary() #model summary
def train_model(self, epochs: int = 20):
"""
Method to train the model.
params:
epochs(int): no of epochs to train the model
"""
self.model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = self.model.fit_generator(self.train_data_gen,
steps_per_epoch=200,
epochs=epochs,
validation_data=self.val_data_gen,
validation_steps=100)
self.plot_train(hist=history)
self.model.save('model.h5')
def plot_train(self, hist):
"""
Method to plot the training and validation performance curves of the model.
Params:
hist: keras history object for parsing the values.
"""
plt.style.use("ggplot")
plt.figure()
plt.plot(hist.history["loss"], label="train_loss")
plt.plot(hist.history["val_loss"], label="val_loss")
plt.plot(hist.history["accuracy"], label="train_acc")
plt.plot(hist.history["val_accuracy"], label="val_acc")
plt.title("Model Training")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend()
plt.savefig("epochs.png")
def resnet_v2(input_shape, depth, num_classes=10):
if (depth - 2) % 9 != 0:
raise ValueError('depth should be 9n+2 (e.g. 20, 101, 164, ...)')
num_filters_in = 16
num_res_blocks = int((depth - 2) / 9)
inputs = Input(shape=input_shape)
# input conv
x = resnet_layer(inputs=inputs,
num_filters=num_filters_in,
conv_first=True)
# downsample feature map 1/2 times for every 2n iterations
# expand output filter size 2 times for every 2n iterations
for stage in range(3):
# residual block
for res_block in range(num_res_blocks):
strides = 1
activation = 'relu'
batch_normalization = True
# downsample the feature map with 1/2 for every 2n iterations
if stage == 0:
num_filters_out = num_filters_in * 4
# if res_block == 0:
# activation = None
# batch_normalization = False
else:
num_filters_out = num_filters_in * 2
if res_block == 0:
strides = 2
y = resnet_layer(inputs=x,
num_filters=num_filters_in,
kernel_size=1,
strides=strides,
activation=activation,
batch_normalization=batch_normalization,
conv_first=False)
y = resnet_layer(inputs=y,
num_filters=num_filters_in,
kernel_size=3,
strides=1,
activation='relu',
conv_first=False)
y = resnet_layer(inputs=y,
num_filters=num_filters_out,
kernel_size=1,
strides=1,
activation='relu',
conv_first=False)
# downsampling the input x is required for skip-connection
# note that activation and BN are not applied! (only resizing the feature map)
if res_block == 0:
x = resnet_layer(inputs=x,
num_filters=num_filters_out,
kernel_size=1,
strides=strides,
activation=None,
batch_normalization=False)
# shortcut-connection
x = add([x, y])
# expand filter size for every 2n iterations
num_filters_in = num_filters_out
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = AveragePooling2D(pool_size=8, padding='same')(x)
y = Flatten()(x)
outputs = Dense(num_classes,
activation='softmax',
kernel_initializer='he_normal')(y)
model = Model(inputs=inputs, outputs=outputs)
return model
def test_shared_instancenorm():
'''Test that a IN layer can be shared
across different data streams.
'''
# Test single layer reuse
bn = InstanceNormalization(input_shape=(10, ))
x1 = Input(shape=(10, ))
bn(x1)
x2 = Input(shape=(10, ))
y2 = bn(x2)
x = np.random.normal(loc=5.0, scale=10.0, size=(2, 10))
model = Model(x2, y2)
model.compile('sgd', 'mse')
model.train_on_batch(x, x)
# Test model-level reuse
x3 = Input(shape=(10, ))
y3 = model(x3)
new_model = Model(x3, y3)
new_model.compile('sgd', 'mse')
new_model.train_on_batch(x, x)
# Flatten images
X_train = X_train.reshape((len(X_train), np.prod(X_train.shape[1:])))
X_test = X_test.reshape((len(X_test), np.prod(X_test.shape[1:])))
# Create Vanilla Autoencoder
input_size = 784
hidden_size = 64
output_size = 784
x = Input(shape=(input_size, ))
h = Dense(hidden_size, activation='relu')(x)
r = Dense(output_size, activation='sigmoid')(h)
autoencoder = Model(inputs=x, outputs=r)
autoencoder.compile(optimizer='adam', loss='mse')
SVG(model_to_dot(autoencoder).create(prog='dot', format='svg'))
# Train
epochs = 5
batch_size = 128
history = autoencoder.fit(X_train,
X_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_test, X_test))
class BaseConvNet(object):
"""
Convolutional Neural Network consisting of sequential convolution and pooling layers. The class is built on
Tensorflow 2/Keras under the hood but uses the scikit-learn design paradigm to enable greater flexibility in
evaluating hyperparameters.
Attributes:
min_filters (int): Minimum number of convolutional filters
filter_growth_rate (float): Factor to scale filter count after each layer.
filter_width (int): Width of square convolutional filter.
min_data_width (int): Output of conv->pooling layer combo is flattened after the data width reaches this
threshold.
pooling_width (int): Factor by which pooling should reduce the spatial dimensions of the input.
hidden_activation (str): Activation function used after each convolution and dense hidden layer. leaky produces leaky relu
output_type: (str): Either linear (regression), sigmoid (binary classification), or softmax (multiclass)
pooling (str): Either max or mean
use_dropout (bool): If True or 1, include SpatialDropout and regular Dropout layers
dropoout_alpha (float): Dropout relative frequency between 0 and 1.
dense_neurons (int): Number of neurons in dense hidden layer. Used as information bottleneck for interpretation.
data_format (str): channels_last (default) or channels_first.
optimizer (str): Supports adam or sgd
loss (str): Supports mse, mae, or binary_crossentropy
leaky_alpha (float): If leaky activation is used, this controls the scaling factor for the leaky ReLU
metrics (list): List of additional metrics to calculate during training.
learning_rate (float): Learning rate for optimizer
batch_size (int): Number of examples per batch
verbose (int): Level of verbosity in fit loop. 1 results in a progress bar and 2 prints loss for each batch.
l2_alpha (float): if l2_alpha > 0 then l2 regularization with strength l2_alpha is used.
early_stopping (int): If > 0, then early stopping of training is triggered when validation loss does not change
for early_stopping epochs.
covariance_scale (float): if > 0, then a covariance regularizer is applied to the activation of the
last hidden layer to promote more independent activations.
"""
def __init__(self, min_filters=16, filter_growth_rate=2, filter_width=5, min_data_width=4, pooling_width=2,
hidden_activation="relu", output_type="linear",
pooling="mean", use_dropout=False, dropout_alpha=0.0, dense_neurons=64,
data_format="channels_last", optimizer="adam", loss="mse", leaky_alpha=0.1, metrics=None,
learning_rate=0.0001, batch_size=1024, epochs=10, verbose=0, l2_alpha=0, early_stopping=0,
covariance_scale=0, **kwargs):
self.min_filters = min_filters
self.filter_width = filter_width
self.filter_growth_rate = filter_growth_rate
self.pooling_width = pooling_width
self.min_data_width = min_data_width
self.hidden_activation = hidden_activation
self.output_type = output_type
self.use_dropout = use_dropout
self.pooling = pooling
self.dropout_alpha = dropout_alpha
self.data_format = data_format
self.optimizer = optimizer
self.learning_rate = learning_rate
self.loss = loss
self.dense_neurons = dense_neurons
self.metrics = metrics
self.leaky_alpha = leaky_alpha
self.batch_size = batch_size
self.epochs = epochs
self.l2_alpha = l2_alpha
self.covariance_scale = covariance_scale
if l2_alpha > 0:
self.use_l2 = 1
else:
self.use_l2 = 0
self.verbose = verbose
self.early_stopping = early_stopping
self.model_ = None
def build_network(self, conv_input_shape, output_size):
"""
Create a keras model with the hyperparameters specified in the constructor.
Args:
conv_input_shape (tuple of shape [variable, y, x]): The shape of the input data
output_size: Number of neurons in output layer.
"""
if self.use_l2:
reg = l2(self.l2_alpha)
else:
reg = None
if self.covariance_scale > 0:
cov_reg = CovarianceRegularizer(self.covariance_scale)
else:
cov_reg = None
conv_input_layer = Input(shape=conv_input_shape, name="conv_input")
num_conv_layers = int(np.round((np.log(conv_input_shape[1]) - np.log(self.min_data_width))
/ np.log(self.pooling_width)))
num_filters = self.min_filters
scn_model = conv_input_layer
for c in range(num_conv_layers):
scn_model = Conv2D(num_filters, (self.filter_width, self.filter_width),
data_format=self.data_format, kernel_regularizer=reg,
padding="same", name="conv_{0:02d}".format(c))(scn_model)
if self.hidden_activation == "leaky":
scn_model = LeakyReLU(self.leaky_alpha, name="hidden_activation_{0:02d}".format(c))(scn_model)
else:
scn_model = Activation(self.hidden_activation, name="hidden_activation_{0:02d}".format(c))(scn_model)
if self.use_dropout:
scn_model = SpatialDropout2D(rate=self.dropout_alpha)(scn_model)
num_filters = int(num_filters * self.filter_growth_rate)
if self.pooling.lower() == "max":
scn_model = MaxPool2D(pool_size=(self.pooling_width, self.pooling_width),
data_format=self.data_format, name="pooling_{0:02d}".format(c))(scn_model)
else:
scn_model = AveragePooling2D(pool_size=(self.pooling_width, self.pooling_width),
data_format=self.data_format, name="pooling_{0:02d}".format(c))(scn_model)
scn_model = Flatten(name="flatten")(scn_model)
if self.use_dropout:
scn_model = Dropout(rate=self.dropout_alpha)(scn_model)
scn_model = Dense(self.dense_neurons, name="dense_hidden",
kernel_regularizer=reg, activity_regularizer=cov_reg)(scn_model)
if self.hidden_activation == "leaky":
scn_model = LeakyReLU(self.leaky_alpha, name="hidden_dense_activation")(scn_model)
else:
scn_model = Activation(self.hidden_activation, name="hidden_dense_activation")(scn_model)
scn_model = Dense(output_size, name="dense_output")(scn_model)
scn_model = Activation(self.output_type, name="activation_output")(scn_model)
self.model_ = Model(conv_input_layer, scn_model)
def compile_model(self):
"""
Compile the model in tensorflow with the right optimizer and loss function.
"""
if self.optimizer.lower() == "adam":
opt = Adam(lr=self.learning_rate)
else:
opt = SGD(lr=self.learning_rate, momentum=0.99)
self.model_.compile(opt, losses[self.loss], metrics=self.metrics)
@staticmethod
def get_data_shapes(x, y):
"""
Extract the input and output data shapes in order to construct the neural network.
"""
if len(x.shape) != 4:
raise ValueError("Input data does not have dimensions (examples, y, x, predictor)")
if len(y.shape) == 1:
output_size = 1
else:
output_size = y.shape[1]
return x.shape[1:], output_size
def fit(self, x, y, val_x=None, val_y=None, build=True, callbacks=None, **kwargs):
"""
Train the neural network.
"""
if build:
x_conv_shape, y_size = self.get_data_shapes(x, y)
self.build_network(x_conv_shape, y_size)
self.compile_model()
if val_x is None:
val_data = None
else:
val_data = (val_x, val_y)
if callbacks is None:
callbacks = []
if self.early_stopping > 0:
callbacks.append(EarlyStopping(patience=self.early_stopping))
self.model_.fit(x, y, batch_size=self.batch_size, epochs=self.epochs, verbose=self.verbose,
validation_data=val_data, callbacks=callbacks, **kwargs)
def predict(self, x):
preds = self.model_.predict(x, batch_size=self.batch_size)
if len(preds.shape) == 2:
if preds.shape[1] == 1:
preds = preds.ravel()
return preds
def output_hidden_layer(self, x, layer_index=-3):
"""
Chop the end off the neural network and capture the output from the specified layer index
Args:
x: input data
layer_index (int): list index of the layer being output.
Returns:
output: array containing output of that layer for each example.
"""
sub_model = Model(self.model_.input, self.model_.layers[layer_index].output)
output = sub_model.predict(x, batch_size=self.batch_size)
return output
def saliency(self, x, layer_index=-3, ref_activation=10):
"""
Output the gradient of input field with respect to each neuron in the specified layer.
Args:
x:
layer_index:
ref_activation: Reference activation value for loss function.
Returns:
"""
saliency_values = np.zeros((self.model_.layers[layer_index].output.shape[-1],
x.shape[0], x.shape[1],
x.shape[2], x.shape[3]),
dtype=np.float32)
for s in trange(self.model_.layers[layer_index].output.shape[-1], desc="neurons"):
sub_model = Model(self.model_.input, self.model_.layers[layer_index].output[:, s])
batch_indices = np.append(np.arange(0, x.shape[0], self.batch_size), x.shape[0])
for b, batch_index in enumerate(tqdm(batch_indices[:-1], desc="batch examples", leave=False)):
x_case = tf.Variable(x.values[batch_index:batch_indices[b + 1]])
with tf.GradientTape() as tape:
tape.watch(x_case)
act_out = sub_model(x_case)
loss = (ref_activation - act_out) ** 2
saliency_values[s, batch_index:batch_indices[b + 1]] = tape.gradient(loss, x_case)
saliency_da = xr.DataArray(saliency_values, dims=("neuron", "p", "row", "col", "var_name"),
coords=x.coords, name="saliency")
return saliency_da
def model_config(self):
all_model_attrs = pd.Series(list(self.__dict__.keys()))
config_attrs = all_model_attrs[all_model_attrs.str[-1] != "_"]
model_config_dict = {}
for attr in config_attrs:
model_config_dict[attr] = self.__dict__[attr]
return model_config_dict
def save_model(self, out_path, model_name):
model_config_dict = self.model_config()
model_config_file = join(out_path, "config_" + model_name + ".yml")
with open(model_config_file, "w") as mcf:
yaml.dump(model_config_dict, mcf, Dumper=yaml.Dumper)
if self.model_ is not None:
model_filename = join(out_path, model_name + ".h5")
self.model_.save(model_filename, save_format="h5")
return
def build_k_frame_model(mdl, X, indices_to_tls, act_func = None):
"""
"""
import tensorflow as tf
from tensorflow.keras.models import Model
import tensorflow.keras.backend as K
import re
import numpy as np
from collections.abc import Iterable
targeting_clname_pattns = ['Dense*', 'Conv*']#, 'LSTM*'] #if not target_all else None
is_target = lambda clname, targets: (targets is None) or any([bool(re.match(t,clname)) for t in targets])
# dictionary to describe the network graph
network_dict = {'input_layers_of': {}, 'new_output_tensor_of': {}}
# Set the input layers of each layer
for layer in mdl.layers:
for node in layer._outbound_nodes: # the output node of the current layer (layer)
layer_name = node.outbound_layer.name # the layer that take node as input
# layer_name takes layer.name as input
if layer_name not in network_dict['input_layers_of']:
network_dict['input_layers_of'].update({layer_name: [layer.name]})
else:
network_dict['input_layers_of'][layer_name].append(layer.name)
min_idx_to_tl = np.min([idx if not isinstance(idx, Iterable) else idx[0] for idx in indices_to_tls])
num_layers = len(mdl.layers)
# Iterate over all layers after the input
# Set the output tensor of the input layer (or more exactly, our starting point)
if min_idx_to_tl == 0 or min_idx_to_tl - 1 == 0:
x = tf.constant(X, dtype = tf.float32) #X.dtype)
layer_name = mdl.layers[0].name
network_dict['new_output_tensor_of'].update({layer_name: x})
else:
t_mdl = Model(inputs = mdl.input, outputs = mdl.layers[min_idx_to_tl-1].output)
prev_output = t_mdl.predict(X)
dtype = mdl.layers[min_idx_to_tl-1].output.dtype
x = tf.constant(prev_output, dtype = dtype)
network_dict['new_output_tensor_of'].update({mdl.layers[min_idx_to_tl-1].name: x})
t_ws = []
for idx_to_l in range(min_idx_to_tl, num_layers):
layer = mdl.layers[idx_to_l]
layer_name = layer.name
# Determine input tensors
layer_input = [network_dict['new_output_tensor_of'][layer_aux]
for layer_aux in network_dict['input_layers_of'][layer.name]]
if len(layer_input) == 1:
layer_input = layer_input[0]
# Insert layer if name matches the regular expression
if idx_to_l in indices_to_tls:
l_class_name = type(layer).__name__
if is_target(l_class_name, targeting_clname_pattns):
if model_util.is_FC(l_class_name):
w,b = layer.get_weights()
t_w = tf.placeholder(dtype = w.dtype, shape = w.shape)
t_b = tf.constant(b)
# this is a neat way, but the probelm is memory explosion... -> process by batches
x = tf.add(tf.tensordot(
layer_input, t_w, [[len(layer_input.shape)-1],[0]]), t_b, name = layer_name)
if act_func is not None:
x = act_func(x)
t_ws.append(t_w)
else: # model_util.is_C2D(l_class_name):
# should be conv2d, if not, then something is wrong
w,b = layer.get_weights()
t_w = tf.placeholder(dtype = w.dtype, shape = w.shape)
t_b = tf.constant(b, dtype = b.dtype)
if layer.get_config()['data_format'] == 'channels_first':
data_format = 'NCHW'
else: # channels_last
data_format = 'NHWC'
x = tf.nn.conv2d(layer_input,
t_w,
strides = list(layer.get_config()['strides'])*2,
padding = layer.get_config()['padding'].upper(),
data_format = data_format,
name = layer_name)
x = tf.nn.bias_add(x, t_b, data_format = data_format)
if act_func is not None: # tf.nn.relu
x = act_func(x)
t_ws.append(t_w)
else:
msg = "{}th layer {}({}) is not our target".format(idx_to_l, layer_name, l_class_name)
assert False, msg
else:
x = layer(layer_input)
# Set new output tensor (the original one, or the one of the replaced layer)
# x is the output of the layer "layer_name" (current one)
network_dict['new_output_tensor_of'].update({layer_name: x})
last_layer_name = mdl.layers[-1].name
num_label = int(mdl.layers[-1].output.shape[-1])
ys = tf.placeholder(dtype = tf.float32, shape = (None, num_label))
pred_probs = tf.math.softmax(network_dict['new_output_tensor_of'][last_layer_name]) # softmax
if len(ys.shape) != len(pred_probs.shape):
if pred_probs.shape[1] == 1:
pred_probs = tf.squeeze(pred_probs, axis = 1)
loss_op = tf.keras.metrics.categorical_crossentropy(ys, pred_probs)
fn = K.function(t_ws + [ys], [network_dict['new_output_tensor_of'][last_layer_name], loss_op])
return fn, t_ws, ys
model2 = BatchNormalization()(model1)
model2 = Dense(7)(model2)
model2 = LeakyReLU(alpha=0.2)(model2)
model3 = Concatenate(axis=-1,activity_regularizer=keras.regularizers.l2(1e-5))([input_layer,model2])
model3 = Dropout(0.5)(model3)
model3 = BatchNormalization(renorm=True)(model3)
prediction1 = Dense(3,activation='linear',kernel_regularizer = keras.regularizers.l2(1e-5))(model3)
model = Model(inputs=input_layer,outputs=prediction1)
opt = keras.optimizers.Adam(decay=1e-5)
model.compile(optimizer=opt , loss = 'mse')
history = model.fit(X_train1,y_train[:,0:3],
epochs=500,
shuffle=True,
validation_data = (X_validation1,y_validation[:,0:3]),
callbacks=[best_model,es])
#sample_weight=waveform_weight)
# ## Loss curve to observe how the network performs
def ResNet18(include_top=True,
weights='cifar100_coarse',
input_tensor=None,
input_shape=None,
pooling=None,
classes=20,
**kwargs):
global backend, layers, models, keras_utils
backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs)
if not (weights in {'cifar100_coarse', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `cifar100_coarse` '
'(pre-training on cifar100 coarse (super) classes), '
'or the path to the weights file to be loaded.')
if weights == 'cifar100_coarse' and include_top and classes != 20:
raise ValueError(
'If using `weights` as `"cifar100_coarse"` with `include_top`'
' as true, `classes` should be 20')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=32,
data_format=backend.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not tf.keras.backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input)
x = Convolution2D(64, (7, 7),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = identity_block(x, 3, [64, 64], stage=2, block='a')
x = identity_block(x, 3, [64, 64], stage=2, block='b')
x = conv_block(x, 3, [128, 128], stage=3, block='a')
x = identity_block(x, 3, [128, 128], stage=3, block='b')
x = conv_block(x, 3, [256, 256], stage=4, block='a')
x = identity_block(x, 3, [256, 256], stage=4, block='b')
x = conv_block(x, 3, [512, 512], stage=5, block='a')
x = identity_block(x, 3, [512, 512], stage=5, block='b')
if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='fc20')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D()(x)
'''
else:
warnings.warn('The output shape of `ResNet18(include_top=False)` '
'has been changed since Keras 2.2.0.')
'''
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet18')
# Load weights.
if weights == 'cifar100_coarse':
if include_top:
weights_path = keras_utils.get_file(
'resnet18_cifar100_top.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='e0798dd90ac7e0498cbdea853bd3ed7f')
else:
weights_path = keras_utils.get_file(
'resnet18_cifar100_no_top.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='bfeace78cec55f2b0401c1f41c81e1dd')
model.load_weights(weights_path)
return model
name='softmax_output')(gru2_out)
# Total number of pixels entering the network
input_length_processed = Lambda(lambda x, num_windows=None: x * num_windows,
arguments={'num_windows': num_windows})(input_length)
# Use tensorflow to calculate CTC loss
ctc_loss_output = Lambda(lambda x: K.ctc_batch_cost(x[0], x[1], x[2], x[3]), name='ctc_loss')(
[y_true, softmax_out, input_length_processed, label_length])
# Out decoded CTC output
ctc_decoded_output = Lambda(lambda x: ctc_decode(x[0], x[1], output_length), name='ctc_decoded')(
[softmax_out, input_length_processed])
# Build the complete model
complete_model = Model(inputs=[image_input, y_true, input_length, label_length],
outputs=[ctc_loss_output, ctc_decoded_output])
# Define loss as our custom ctc_loss output layer
complete_model.compile(
loss={'ctc_loss': lambda y_true, y_pred: y_pred}, optimizer="adam")
complete_model.summary()
def decode_examples(images, labels):
"""Transform our output for example logging"""
return complete_model.predict(format_batch_ctc(images, labels)[0])[1]
# Initialize generators and train
train = Generator(dataset.x_train, dataset.y_train,
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Reshape, Flatten, Dense
from unit_test.helper import tf_random_seed, clean_name
model_list = []
for layer_item, units in zip([Dense] * 3, [128, 64, 32]):
tf_random_seed()
placeholder = Input((32, 32, 3), name='data')
x = Flatten()(placeholder)
x = Dense(units, activation='relu')(x)
model = Model(
placeholder,
x,
name=
f'model_single_layer_{layer_item.__name__.lower()}_{clean_name(str(units))}'
)
model_list.append(model)
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None)(x)
x = Conv2D(64, (3, 3), activation='relu', trainable=False)(x)
x = AveragePooling2D((2, 2), trainable=False)(x)
x = Conv2D(128, (3, 3), activation='relu', trainable=False)(x)
x = Conv2D(128, (3, 3), activation='relu', trainable=False)(x)
x = Reshape((-1, 128))(x)
x = Capsule(32, 8, 3, True)(x)
x = Capsule(32, 8, 3, True)(x)
capsule = Capsule(5, 16, 3, True)(x)
output = Lambda(lambda z: K.sqrt(K.sum(K.square(z), 2)))(capsule)
model = Model(inputs=[input_image], outputs=[output])
model.load_weights('pre-train.h5')
capsule2 = Capsule(2, 16, 3, True)(model.layers[-3].output)
output2 = Lambda(lambda z: K.sqrt(K.sum(K.square(z), 2)))(capsule2)
model2 = Model(inputs=[input_image], outputs=[output2])
adam = optimizers.Adam(lr=0.001)
model2.compile(loss=margin_loss, optimizer=adam, metrics=['accuracy'])
model2.summary()
data_augmentation = False
class Agent():
""" Agent object which initalizes and trains the keras model.
"""
def __init__(self,
actions,
height=80,
width=80,
channels=1,
discount=0.95,
loss="huber",
env="Breakout-v0",
model_dir=None):
""" Initializes the parameters of the model.
Args:
height: Height of the image
width: Width of the image
channels: Number of channels, history of past frame
discount: Discount_Factor for Q Learning update
"""
self.height = height
self.width = width
self.channels = channels
self.discount = discount
self.actions = actions
self.env = env
self.loss = loss
self.epoch_num = 0
self.model_dir = model_dir
self.max_reward = 0
self.cur_reward = 0
self.reward_tensor = K.variable(value=0)
if model_dir is not None:
self.tbCallBack = TensorBoard(
log_dir=model_dir,
histogram_freq=0,
write_graph=True,
write_images=True)
def create_model(
self,
lr,
type="vanilla",
rescale_value=255.0,
):
""" Builds the DQN Agent architecture.
Source:https://cs.corp.google.com/piper///depot/google3/third_party/py/
dopamine/agents/dqn/dqn_agent.py?q=DQN&dr=CSs&l=15
This initializes the model as per the specifications mentioned in the
DQN paper by Deepmind. This is a sequential model implemention of
tf.keras. The compiled model is returned by the Method.
Args:
Returns:
Model: Compiled Model
"""
#with tf.device('/gpu:0'):
self.image_frames = Input(shape=(self.height, self.width, self.channels))
#self.normalize = Lambda(lambda input: input/255.0)
self.conv1 = Conv2D(
filters=32,
kernel_size=(8, 8),
strides=(4, 4),
activation="relu",
name="conv1")(
Lambda(lambda input: input / float(rescale_value))(
self.image_frames))
self.conv2 = Conv2D(
filters=64,
kernel_size=(4, 4),
strides=(2, 2),
activation="relu",
name="conv2")(
self.conv1)
self.conv3 = Conv2D(
filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation="relu",
name="conv3")(
self.conv2)
self.flattened = Flatten(name="flattened")(self.conv3)
self.fully_connected_1 = Dense(
units=512, activation="relu", name="fully_connected_1")(
self.flattened)
self.q_values = Dense(
units=self.actions, activation="linear", name="q_values")(
self.fully_connected_1)
self.model = Model(inputs=[self.image_frames], outputs=[self.q_values])
self.optimizer = Adam(lr=lr)
if self.loss == "huber":
self.loss = huber_loss
K.get_session().run(tf.global_variables_initializer())
def reward(y_true, y_pred):
return self.reward_tensor
self.model.compile(
optimizer=self.optimizer, loss=self.loss, metrics=["mse", reward])
return self.model
def batch_train(self,
curr_state,
next_state,
immediate_reward,
action,
done,
target,
type="Double"):
""" Computes the TD Error for a given batch of tuples.
Here, we randomly sample episodes from the Experience buffer and use
this to train our model. This method computes this for a batch and
trains the model.
Args:
curr_state(array): Numpy array representing an array of current
states of game
next_state(array): Numpy array for immediate next state of
the game
action(array): List of actions taken to go from current state
to the next
reward(array): List of rewards for the given transition
done(bool): if this is a terminal state or not.
target(keras.model object): Target network for computing TD error
"""
if type == "Double":
forward_action = np.argmax(self.model.predict(next_state), axis=1)
predicted_qvalue = target.predict(next_state) # BxN matrix
B = forward_action.size
forward_qvalue = predicted_qvalue[np.arange(B), forward_action] # Bx1 vec
elif type == "Vanilla":
forward_qvalue = np.max(target.predict(next_state), axis=1)
discounted_reward = (self.discount * forward_qvalue * (1 - done))
Q_value = immediate_reward + discounted_reward
target_values = self.model.predict(curr_state)
target_values[range(target_values.shape[0]), action] = Q_value
"""
for i, target in enumerate(target_values):
target_values[i, action[i]] = Q_value[i]
"""
callbacks = []
# Update epoch number for TensorBoard.
K.set_value(self.reward_tensor, self.cur_reward)
if self.model_dir is not None and self.epoch_num % TB_LOGGING_EPOCHS == 0:
callbacks.append(self.tbCallBack)
self.model.fit(
curr_state,
target_values,
verbose=0,
initial_epoch=self.epoch_num,
callbacks=callbacks,
epochs=self.epoch_num + 1)
self.epoch_num += 1
def predict_action(self, state):
""" Predict the action for a given state.
Args:
state(float): Numpy array
Return:
action(int): Discrete action to sample
"""
#state = downsample_state(convert_greyscale(state))
#state = np.expand_dims(state, axis=0)
if np.ndim(state) == 3:
state = np.expand_dims(state, axis=0)
return np.argmax(self.model.predict(state))
def play(self, env, directory, mode):
""" Returns the total reward for an episode of the game."""
steps = []
state = env.reset()
done = False
tot_reward = 0
actions = [0] * self.actions
while not done:
if mode != "Train":
s = env.render("rgb_array")
steps.append(s)
action = self.predict_action(state)
actions[action] += 1
state, reward, done, _ = env.step(action)
tot_reward += reward
self.cur_reward = tot_reward
if mode != "Train" and tot_reward > self.max_reward:
print("New high reward: ", tot_reward)
clip = ImageSequenceClip(steps, fps=30)
clip.write_gif("~/breakout.gif", fps=30)
self.max_reward = tot_reward
print("ACTIONS TAKEN", actions)
return tot_reward
def train_SHAME_Model(numpy2dTimeArray, numpy2dSizeArray, numpy2dIatArray,
numpy2dFeatArray, INPUT_SHAPE, NUM_CLASSES, time_weights,
size_weights, iat_weights, y):
if not isdir("./time_weights"):
makedirs("./time_weights")
if not isdir("./size_weights"):
makedirs("./size_weights")
if not isdir("./iat_weights"):
makedirs("./iat_weights")
if not isdir("./ensemble_weights"):
makedirs("./ensemble_weights")
if not isdir("./pics"):
makedirs("./pics")
early_stopping = EarlyStopping(monitor="val_accuracy",
patience=100,
restore_best_weights=True)
checkpoint1 = ModelCheckpoint(
'./time_weights/time-model-{epoch:03d}-{accuracy:03f}-{val_accuracy:03f}.h5',
verbose=1,
monitor='val_accuracy',
save_best_only=True,
mode='auto')
checkpoint2 = ModelCheckpoint(
'./size_weights/size-model-{epoch:03d}-{accuracy:03f}-{val_accuracy:03f}.h5',
verbose=1,
monitor='val_accuracy',
save_best_only=True,
mode='auto')
checkpoint3 = ModelCheckpoint(
'./iat_weights/iat-model-{epoch:03d}-{accuracy:03f}-{val_accuracy:03f}.h5',
verbose=1,
monitor='val_accuracy',
save_best_only=True,
mode='auto')
checkpoint4 = ModelCheckpoint(
'./ensemble_weights/ensemble-model-{epoch:03d}-{accuracy:03f}-{val_accuracy:03f}.h5',
verbose=1,
monitor='val_accuracy',
save_best_only=True,
mode='auto')
tensorboard_callback = TensorBoard(log_dir="./logs")
# Create two DF models
model = build_deep_fingerprinting_model(INPUT_SHAPE, NUM_CLASSES)
model2 = build_deep_fingerprinting_model(INPUT_SHAPE, NUM_CLASSES)
model3 = build_deep_fingerprinting_model(INPUT_SHAPE, NUM_CLASSES)
te_cut = int(len(y) * .9)
# Check to see if pre-trained weights were passed or not
batch_size = 256
if time_weights is None:
df_time_history = model.fit(numpy2dTimeArray[:te_cut],
to_categorical(y[:te_cut]),
batch_size=batch_size,
epochs=600,
validation_split=0.10,
verbose=False,
callbacks=[checkpoint1, early_stopping])
else:
model.load_weights(time_weights)
if size_weights is None:
df_size_history = model2.fit(numpy2dSizeArray[:te_cut],
to_categorical(y[:te_cut]),
batch_size=batch_size,
epochs=600,
validation_split=0.10,
verbose=False,
callbacks=[checkpoint2, early_stopping])
else:
model2.load_weights(size_weights)
if iat_weights is None:
df_iat_history = model3.fit(numpy2dIatArray[:te_cut],
to_categorical(y[:te_cut]),
batch_size=batch_size,
epochs=600,
validation_split=0.10,
verbose=False,
callbacks=[checkpoint3, early_stopping])
else: # Load weights
model3.load_weights(iat_weights)
# Make sure to not train either model any further
model.trainable = False
model2.trainable = False
model3.trainable = False
from tensorflow.keras.utils import plot_model
plot_model(model, to_file='./pics/time_model.png', show_shapes='True')
plot_model(model2, to_file='./pics/size_model2.png', show_shapes='True')
plot_model(model3, to_file='./pics/iat_model3.png', show_shapes='True')
print("Getting Flatten layer using the time array")
# Create a new model that takes in (MAX_SIZE, 1) and outputs the flatten layers for time
flatten_model1 = Model(inputs=model.input,
outputs=model.get_layer('flatten').output)
outputs1 = flatten_model1.predict(numpy2dTimeArray, verbose=1) # (N, 1024)
print("Getting Flatten layer using the size array")
# Create a new model that takes in (MAX_SIZE, 1) and outputs the flatten layers for size
flatten_model2 = Model(inputs=model2.input,
outputs=model2.get_layer('flatten').output)
outputs2 = flatten_model2.predict(numpy2dSizeArray, verbose=1) # (N, 1024)
print("Getting Flatten layer using the iat array")
# Create a new model that takes in (MAX_SIZE, 1) and outputs the flatten layers for size
flatten_model3 = Model(inputs=model3.input,
outputs=model3.get_layer('flatten').output)
outputs3 = flatten_model3.predict(numpy2dIatArray, verbose=1) # (N, 1024)
from sklearn.preprocessing import normalize
numpy2dFeatArray = normalize(numpy2dFeatArray, norm='l2')
#print(numpy2dFeatArray[0])
ensemble_input = np.concatenate(
(numpy2dFeatArray, outputs1, outputs2, outputs3),
axis=1) # (N, FEATURES+3072)
print(ensemble_input.shape)
model4 = build_ensemble_model((ensemble_input.shape[1], ), NUM_CLASSES)
model4.summary()
#early_stopping = EarlyStopping(monitor="val_loss", patience=20, restore_best_weights=True)
model4.fit(x=ensemble_input[:te_cut],
y=to_categorical(y[:te_cut]),
batch_size=256,
epochs=600,
validation_split=0.10,
verbose=False,
callbacks=[checkpoint4, early_stopping])
plot_model(model4,
to_file='./pics/ensemble_model3.png',
show_shapes='True')
print(
model.evaluate(x=numpy2dTimeArray[te_cut:],
y=to_categorical(y[te_cut:]),
verbose=0))
print(
model2.evaluate(x=numpy2dSizeArray[te_cut:],
y=to_categorical(y[te_cut:]),
verbose=0))
print(
model3.evaluate(x=numpy2dIatArray[te_cut:],
y=to_categorical(y[te_cut:]),
verbose=0))
print(
model4.evaluate(x=ensemble_input[te_cut:],
y=to_categorical(y[te_cut:]),
verbose=0))
# Headline input: meant to receive sequences of 100 integers, between 1 and 10000.
# Note that we can name any layer by passing it a "name" argument.
main_input = Input(shape=(100,), dtype='float32', name='main_input')
# This embedding layer will encode the input sequence
# into a sequence of dense 512-dimensional vectors.
x = Embedding(output_dim=512, input_dim=10000, input_length=100)(main_input)
# A LSTM will transform the vector sequence into a single vector,
# containing information about the entire sequence
lstm_out = LSTM(32)(x)
auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
auxiliary_input = Input(shape=(5,), name='aux_input')
x = keras.layers.concatenate([lstm_out, auxiliary_input])
# We stack a deep densely-connected network on top
x = Dense(64, activation='relu')(x)
x = Dense(64, activation='relu')(x)
x = Dense(64, activation='relu')(x)
# And finally we add the main logistic regression layer
main_output = Dense(1, activation='sigmoid', name='main_output')(x)
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output, auxiliary_output])
X = np.random.randn(12, 100)
Z = np.random.randn(1, 5, 1)
model.predict({'main_input': X, 'aux_input': Z})
def build(input_shape, outputShape, block_fn, repetitions, reg_factor):
"""Instantiate a vanilla ResNet3D keras model.
# Arguments
input_shape: Tuple of input shape in the format
(conv_dim1, conv_dim2, conv_dim3, channels) if dim_ordering='tf'
(filter, conv_dim1, conv_dim2, conv_dim3) if dim_ordering='th'
num_outputs: The number of outputs at the final softmax layer
block_fn: Unit block to use
repetitions: Repetitions of unit blocks
# Returns
model: a 3D ResNet model that takes a 5D array (volumetric images
in batch) as input and returns a 3D array (prediction) as output.
"""
nVes, nVal = outputShape
num_outputs = nVes * nVal
_handle_data_format()
if len(input_shape) != 4:
raise ValueError("Input shape should be a tuple "
"(conv_dim1, conv_dim2, conv_dim3, channels) "
"for tensorflow as backend or "
"(channels, conv_dim1, conv_dim2, conv_dim3) "
"for theano as backend")
block_fn = _get_block(block_fn)
input = Input(shape=input_shape)
# # first conv
# conv1 = _conv_bn_relu3D(filters=64, kernel_size=(7, 7, 7),
# strides=(2, 2, 2),
# kernel_regularizer=l2(reg_factor)
# )(input)
# pool2 = MaxPooling3D(pool_size=(3, 3, 3), strides=(2, 2, 2),
# padding="same")(conv1)
# repeat blocks
block = input
filters = 64
for i, r in enumerate(repetitions):
block = _residual_block3d(block_fn,
filters=filters,
kernel_regularizer=l2(reg_factor),
repetitions=r,
is_first_layer=(i == 0))(block)
filters *= 2
# last activation
block_output = _bn_relu(block)
# average pool and classification
pool2 = AveragePooling3D(pool_size=(block.shape[1], block.shape[2],
block.shape[3]),
strides=(1, 1, 1))(block_output)
flatten1 = Flatten()(pool2)
dense = Dropout(0.5)(flatten1)
numFCLayers = 3
for dNum in range(numFCLayers):
dense = Dense(units=num_outputs * 2**(numFCLayers - dNum - 1),
kernel_initializer="he_normal",
kernel_regularizer=l2(reg_factor))(dense)
if dNum == (numFCLayers - 1):
dense = Activation("sigmoid")(dense)
else:
dense = Activation("relu")(dense)
outputs = tf.keras.layers.Reshape(outputShape)(dense)
return Model(inputs=input, outputs=outputs)
