def test_adapt_preprocessing_stage_with_single_input_output(self):
x = Input(shape=(3, ))
l0 = PL()
y = l0(x)
l1 = PL()
z = l1(y)
stage = preprocessing_stage.FunctionalPreprocessingStage(x, z)
stage.compile()
# Test with NumPy array
one_array = np.ones((4, 3), dtype="float32")
stage.adapt(one_array)
self.assertEqual(l0.adapt_count, 1)
self.assertEqual(l1.adapt_count, 1)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
# Check call
z = stage(tf.ones((4, 3), dtype="float32"))
self.assertAllClose(z, np.ones((4, 3), dtype="float32") + 2.0)
# Test with dataset
adapt_data = tf.data.Dataset.from_tensor_slices(one_array)
adapt_data = adapt_data.batch(2) # 5 batches of 2 samples
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 2)
self.assertEqual(l1.adapt_count, 2)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
# Test error with bad data
with self.assertRaisesRegex(ValueError, "requires a "):
stage.adapt(None)
# Disallow calling fit
with self.assertRaisesRegex(ValueError, "Preprocessing stage"):
stage.fit(None)
def chargen_model(num_of_classes,
cfg,
weights_filepath=None,
dropout=0.3,
optimizer=RMSprop(lr=4e-3, rho=0.99)):
"""Builds the neural network model architecture for CharGen and loads the weights specified for the model.
:param num_of_classes: total number of classes
:param cfg: configuration of CharGen
:param weights_filepath: path of the weights file
:param dropout: fraction of neurons to be ignored in a single forward and backward pass (default 0.05)
:param optimizer: specify which optimization algorithm to use (Default RMSprop)
:return: model to be used for training
"""
inp = Input(shape=(cfg['input_length'], ), name='input')
embedded = Embedding(num_of_classes,
cfg['embedding_dims'],
input_length=cfg['input_length'],
name='embedding')(inp)
if dropout > 0.0:
embedded = SpatialDropout1D(dropout, name='dropout')(embedded)
rnn_layer_list = []
for i in range(cfg['rnn_layers']):
prev_layer = embedded if i is 0 else rnn_layer_list[-1]
rnn_layer_list.append(new_rnn_layer(cfg, i + 1)(prev_layer))
seq_concat = concatenate([embedded] + rnn_layer_list, name='rnn_concat')
attention = WeightedAttentionAverage(name='attention')(seq_concat)
output = Dense(num_of_classes, name='output',
activation='softmax')(attention)
model = Model(inputs=[inp], outputs=[output])
if weights_filepath is not None:
model.load_weights(weights_filepath, by_name=True)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
def build(self):
base_model = MobileNetV2(input_tensor=Input(shape=(360,640,3)), \
alpha=.75, include_top=False, weights='imagenet', pooling='avg')
#freeze_weights(base_model)
#for layer in base_model.layers[:154]:
# layer.trainable=False
#for i,layer in enumerate(base_model.layers):
# print(i,layer.name)
x = base_model.output
#testing
x = Dense(200, activation='relu')(x)
#x = Conv2D(8, (1, 1), padding='same')(x)
#x = Flatten()(x)
#x = Dropout(0.2, name='Dropout1')(x)
x = Dense(self.num_outputs, activation='linear')(x)
x = Reshape((self.num_outputs,))(x)
model = Model(inputs=base_model.inputs, outputs=x)
return model
def _setup_generic_input_layer(self, input_params):
encoder_name = input_params["encoder"]
encoder_params = self.args["encoders"][encoder_name]
scenario = self.args["features"]["scenarios"][input_params["scenario"]]
emb_input = scenario["input"]
emb_field = input_params["field"]
max_seq_len = encoder_params["max_seq_len_{}".format(emb_input)]
layer = Input(shape=(max_seq_len, ),
name="{}_encoder".format(emb_field))
self.inputs.append(layer)
embedding = self.setup_embedding(input_params)
layer = embedding.apply_to_input_layer(layer,
max_seq_len,
apply_mask=True)
encoder = self._setup_encoder(encoder_name)
layer = encoder(layer)
layer = self._setup_attention(encoder_params, layer, encoder_name)
return layer
def create_model_duel(output_shape, input_shape):
inputs = Input(shape=input_shape)
net = Conv2D(32,
8,
strides=(4, 4),
data_format='channels_first',
activation='relu')(inputs)
net = Conv2D(64,
4,
strides=(2, 2),
data_format='channels_first',
activation='relu')(net)
net = Flatten()(net)
advt = Dense(256, activation='relu')(net)
advt = Dense(output_shape)(advt)
value = Dense(256, activation='relu')(net)
value = Dense(1)(value)
advt = Lambda(lambda advt: advt - tf.reduce_mean(
advt, axis=-1, keep_dims=True))(advt)
value = Lambda(lambda value: tf.tile(value, [1, output_shape]))(value)
final = Add()([value, advt])
model = Model(inputs=inputs, outputs=final)
model.compile(optimizer='Adam', loss='mse')
return model
def adv_output(c_bits, k_bits):
einput = Input(shape=(c_bits,)) # ciphertext only
e_dense1 = Dense(units=(c_bits + k_bits))(einput)
e_dense1a = Activation('tanh')(e_dense1)
e_dense2 = Dense(units=(c_bits + k_bits))(e_dense1a)
e_dense2a = Activation('tanh')(e_dense2)
e_reshape = Reshape((c_bits + k_bits, 1,))(e_dense2a)
e_conv1 = Conv1D(filters=2, kernel_size=4, strides=1, padding=pad)(e_reshape)
e_conv1a = Activation('tanh')(e_conv1)
e_conv2 = Conv1D(filters=4, kernel_size=2, strides=2, padding=pad)(e_conv1a)
e_conv2a = Activation('tanh')(e_conv2)
e_conv3 = Conv1D(filters=4, kernel_size=1, strides=1, padding=pad)(e_conv2a)
e_conv3a = Activation('tanh')(e_conv3)
e_conv4 = Conv1D(filters=1, kernel_size=1, strides=1, padding=pad)(e_conv3a)
e_conv4a = Activation('sigmoid')(e_conv4)
# adversary attempt at guessing the plaintext
attacker_output = Flatten()(e_conv4a)
attacker_model = Model(einput, attacker_output, name='attacker')
return attacker_output, attacker_model
def build_model(X_train, X_test, Y_train, noLSTM, train_labels):
model = Sequential()
model.reset_states()
with codecs.open(rootFolder + "training.csv", 'a') as logfile:
fieldnames = ['lstms', 'outpts']
writer = csv.DictWriter(logfile, fieldnames=fieldnames)
writer.writerow({'lstms': noLSTM[0], 'outpts': noLSTM[1]})
print(noLSTM[0], " >> ", noLSTM[1])
model.add(
Input(shape=(X_train.shape[1], ),
batch_shape=(slidingWindowSize, X_train.shape[1]),
use_bias=True))
for p in range(noLSTM[0]):
model.add(Dense(noLSTM[1], activation='tanh', use_bias=True))
model.add(Dropout(0.5))
for p in range(noLSTM[0]):
model.add(LSTM(noLSTM[1], activation='tanh', recurrent_activation='hard_sigmoid', \
use_bias=True, kernel_initializer='glorot_uniform', \
recurrent_initializer='orthogonal', \
unit_forget_bias=True, kernel_regularizer=None, \
recurrent_regularizer=None, \
bias_regularizer=None, activity_regularizer=None, \
kernel_constraint=None, recurrent_constraint=None, \
bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, \
implementation=1, return_sequences=True, return_state=False, \
go_backwards=False, stateful=False, unroll=False, \
input_shape=(slidingWindowSize, X_train.shape[1])))
model.add(Dropout(0.5))
for p in range(noLSTM[0]):
model.add(Dense(noLSTM[1], activation='tanh', use_bias=True))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
fnametmp = rootFolder + "plot-{}-{}-{}.png".format("Model", noLSTM[0],
noLSTM[1])
plot_model(model,
to_file=fnametmp,
show_shapes=True,
show_layer_names=True,
rankdir='LR')
return
csv_logger = CSVLogger(rootFolder + 'training.csv', append=True)
early_stop = EarlyStopping(monitor='val_acc',
patience=2,
verbose=2,
mode='auto')
history = model.fit(X_train,
Y_train,
batch_size=1,
epochs=5,
callbacks=[csv_logger, early_stop],
validation_split=0.2,
shuffle=True)
with open(rootFolder + "history-" + noLSTM[0] + "-" + noLSTM[1] + ".file",
"wb") as f:
pickle.dump([history], f, pickle.HIGHEST_PROTOCOL)
# ['acc', 'loss', 'val_acc', 'val_loss']
fnametmp = "plot-{}-{}-{}.png".format("model-accuracy", noLSTM[0],
noLSTM[1])
drawMe(yVal=history.history['acc'],
xVal=history.history['val_acc'],
title='model accuracy',
xlabel='epoch',
ylabel='accuracy',
legend=['train', 'test'],
save=True,
fileName=fnametmp,
show=False)
fnametmp = "plot-{}-{}-{}.png".format("model-loss", noLSTM[0], noLSTM[1])
drawMe(yVal=history.history['loss'],
xVal=history.history['val_loss'],
title='model loss',
xlabel='epoch',
ylabel='loss',
legend=['train', 'test'],
save=True,
fileName=fnametmp,
show=False)
pred = model.predict(X_test)
compute_accuracy(pred, train_labels)
def CNN_GRU(n_timesteps):
#15 submodels, a CNN is trained for each feature vector
visible1 = Input(shape=(n_timesteps, 1))
cnn1 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible1)
#cnn1 = MaxPooling1D(pool_size=2)(cnn1)
#cnn1 = Flatten(cnn1)
visible2 = Input(shape=(n_timesteps, 1))
cnn2 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible2)
#cnn2 = MaxPooling1D(pool_size=2)(cnn2)
#cnn2 = Flatten(cnn2)
visible3 = Input(shape=(n_timesteps, 1))
cnn3 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible3)
#cnn3 = MaxPooling1D(pool_size=2)(cnn3)
#cnn3 = Flatten(cnn3)
visible4 = Input(shape=(n_timesteps, 1))
cnn4 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible4)
#cnn4 = MaxPooling1D(pool_size=2)(cnn4)
#cnn4 = Flatten(cnn4)
visible5 = Input(shape=(n_timesteps, 1))
cnn5 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible5)
#cnn5 = MaxPooling1D(pool_size=2)(cnn5)
#cnn5 = Flatten(cnn5)
visible6 = Input(shape=(n_timesteps, 1))
cnn6 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible6)
#cnn6 = MaxPooling1D(pool_size=2)(cnn6)
#cnn6 = Flatten(cnn1)
visible7 = Input(shape=(n_timesteps, 1))
cnn7 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible7)
#cnn7 = MaxPooling1D(pool_size=2)(cnn7)
#cnn7 = Flatten(cnn7)
visible8 = Input(shape=(n_timesteps, 1))
cnn8 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible8)
#cnn8 = MaxPooling1D(pool_size=2)(cnn8)
#cnn8 = Flatten(cnn8)
visible9 = Input(shape=(n_timesteps, 1))
cnn9 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible9)
#cnn9 = MaxPooling1D(pool_size=2)(cnn9)
#cnn9 = Flatten(cnn9)
visible10 = Input(shape=(n_timesteps, 1))
cnn10 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible10)
#cnn10 = MaxPooling1D(pool_size=2)(cnn10)
#cnn10 = Flatten(cnn1)
visible11 = Input(shape=(n_timesteps, 1))
cnn11 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible11)
#cnn11 = MaxPooling1D(pool_size=2)(cnn11)
#cnn11 = Flatten(cnn11)
visible12 = Input(shape=(n_timesteps, 1))
cnn12 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible12)
#cnn12 = MaxPooling1D(pool_size=2)(cnn12)
#cnn12 = Flatten(cnn12)
visible13 = Input(shape=(n_timesteps, 1))
cnn13 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible13)
#cnn13 = MaxPooling1D(pool_size=2)(cnn13)
#cnn13 = Flatten(cnn13)
visible14 = Input(shape=(n_timesteps, 1))
cnn14 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible14)
#cnn13 = MaxPooling1D(pool_size=2)(cnn13)
#cnn13 = Flatten(cnn13)
visible15 = Input(shape=(n_timesteps, 1))
cnn15 = Conv1D(filters=64, kernel_size=2, strides=1,
activation='relu')(visible15)
merged = concatenate([
cnn1, cnn2, cnn3, cnn4, cnn5, cnn6, cnn7, cnn8, cnn9, cnn10, cnn11,
cnn12, cnn13, cnn14, cnn15
]) # 3* (64*13)
merged = GRU(50, activation='relu', return_sequences=True)(
merged) # 3*50 # initially filters were 64, now 1
merged = GRU(50, activation='relu', return_sequences=True)(merged) # 3*50
merged = GRU(50, activation='relu', return_sequences=True)(merged)
merged = GRU(50, activation='relu')(merged) # 1*50
merged = Dense(25)(merged)
output = Dense(1)(merged) # 1*1
model = Model(inputs=[
visible1, visible2, visible3, visible4, visible5, visible6, visible7,
visible8, visible9, visible10, visible11, visible12, visible13,
visible14, visible15
],
outputs=output)
model.compile(optimizer='adam', loss='mse')
model.summary()
return model
def create_base_network(inputs):
# Base of the Neural Network
x = Conv3D(64, kernel_size=(1, 3, 3), activation='relu')(inputs)
x = Conv3D(64, (1, 3, 3), activation='relu')(x)
x = Conv3D(128, (1, 3, 3), activation='relu')(x)
x = Conv3D(128, (1, 3, 3), activation='relu')(x)
x = MaxPooling3D(pool_size=(1, 2, 2))(x)
x = Conv3D(256, (3, 3, 3), activation='relu')(x)
x = Conv3D(512, (3, 3, 3), activation='relu')(x)
x = Flatten()(x)
x = Dense(64, activation='relu')(x)
return x
input_shape = (5, 60, 60, 4)
input1 = Input(shape=input_shape)
input2 = Input(shape=(1,))
mod1 = create_base_network(input1)
x = keras.layers.concatenate([mod1, input2])
output = Dense(2, activation='softmax')(mod1)
model = Model(inputs=[input1], outputs=[output])
sgd = optimizers.SGD(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer= sgd, metrics=['accuracy'])
print(" ===== 3D CNN Model Built and Compiled.Ready For USE ===== \n")
print("\n ===== ** ==== Starting the Training of the 3D CNN Model.This May Take a While.Hang Tight !! ==== ** =====")
MODEL = 'currentCNNmodel.h5'
checkpoint = ModelCheckpoint(MODEL, monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1)
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=100, verbose=1, mode='auto')
BATCH_SIZE = 32
# Starting Training
if c3 == 0:
return 0
precision = c1 / c2
recall = c1 / c3
f1_score = 2 * (precision * recall) / (precision + recall)
return f1_score
x_train_emw, y_train_emw = get_padded_dataset(emw_train_dataset)
x_dev_emw, y_dev_emw = get_padded_dataset(emw_dev_dataset)
# Modeling Our Network
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedding_layer = Embedding(len(embeddings_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
embedded_sequences = embedding_layer(sequence_input)
x = Bidirectional(CuDNNGRU(128, return_sequences=True))(embedded_sequences)
x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
x = AttentionWithContext()(x)
x = Dense(64, activation="relu")(x)
preds = Dense(1, activation='sigmoid')(x)
model = Model(sequence_input, preds)
def create(self, res: Resolution, classes: Optional[int]) -> Model:
"""
Creates a keras.models.Model object.
"""
if type(classes) is not int:
raise ValueError(classes)
img_input = Input(res.hwc())
# Block 1
x = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1')(img_input)
x = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1')(x)
x = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1')(x)
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2')(x)
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
model = Model(img_input, x, name='vgg16')
return model
def _clone_sequential_model(model, input_tensors=None, layer_fn=_clone_layer):
"""Clone a `Sequential` model instance.
Model cloning is similar to calling a model on new inputs,
except that it creates new layers (and thus new weights) instead
of sharing the weights of the existing layers.
Args:
model: Instance of `Sequential`.
input_tensors: optional list of input tensors
to build the model upon. If not provided,
placeholders will be created.
layer_fn: callable to be applied on non-input layers in the model. By
default it clones the layer. Another example is to preserve the layer
to share the weights. This is required when we create a per-replica
copy of the model with distribution strategy; we want the weights to
be shared but still feed inputs separately so we create new input
layers.
Returns:
An instance of `Sequential` reproducing the behavior
of the original model, on top of new inputs tensors,
using newly instantiated weights.
Raises:
ValueError: in case of invalid `model` argument value or `layer_fn`
argument value.
"""
if not isinstance(model, Sequential):
raise ValueError(
'Expected `model` argument '
'to be a `Sequential` model instance, '
'but got:', model)
if not callable(layer_fn):
raise ValueError('Expected `layer_fn` argument to be a callable.')
layers = [] # Layers needed to compute the model's outputs.
layer_map = {}
# Ensure that all layers are cloned. The model's layers
# property will exclude the initial InputLayer (if it exists) in the model,
# resulting in a different Sequential model structure.
for layer in model._flatten_layers(include_self=False, recursive=False):
if isinstance(layer, InputLayer) and input_tensors is not None:
# If input tensors are provided, the original model's InputLayer is
# overwritten with a different InputLayer.
continue
cloned_layer = (_clone_layer(layer)
if isinstance(layer, InputLayer) else layer_fn(layer))
layers.append(cloned_layer)
layer_map[layer] = cloned_layer
layers, ancillary_layers = _remove_ancillary_layers(
model, layer_map, layers)
if input_tensors is None:
cloned_model = Sequential(layers=layers, name=model.name)
elif len(generic_utils.to_list(input_tensors)) != 1:
raise ValueError('To clone a `Sequential` model, we expect '
' at most one tensor '
'as part of `input_tensors`.')
else:
# Overwrite the original model's input layer.
if isinstance(input_tensors, tuple):
input_tensors = list(input_tensors)
x = generic_utils.to_list(input_tensors)[0]
if backend.is_keras_tensor(x):
origin_layer = x._keras_history.layer
if isinstance(origin_layer, InputLayer):
cloned_model = Sequential(layers=[origin_layer] + layers,
name=model.name)
else:
raise ValueError('Cannot clone a `Sequential` model on top '
'of a tensor that comes from a Keras layer '
'other than an `InputLayer`. '
'Use the functional API instead.')
else:
input_tensor = Input(tensor=x,
name='input_wrapper_for_' + str(x.name))
input_layer = input_tensor._keras_history.layer
cloned_model = Sequential(layers=[input_layer] + layers,
name=model.name)
if not ancillary_layers:
return cloned_model
tensor_map = {} # Maps tensors from `model` to those in `cloned_model`.
for depth, cloned_nodes in cloned_model._nodes_by_depth.items():
nodes = model._nodes_by_depth[depth]
# This should be safe in a Sequential model. In an arbitrary network, you
# need to sort using the outbound layer of the node as a key.
for cloned_node, node in zip(cloned_nodes, nodes):
if isinstance(cloned_node.output_tensors, list):
for j, output_tensor in enumerate(cloned_node.output_tensors):
tensor_map[node.output_tensors[j]] = output_tensor
else:
tensor_map[node.output_tensors] = cloned_node.output_tensors
# Ancillary nodes have negative depth.
new_nodes = _make_new_nodes(
{
depth: nodes
for depth, nodes in model._nodes_by_depth.items() if depth < 0
}, layer_fn, layer_map, tensor_map)
_insert_ancillary_layers(cloned_model, ancillary_layers,
model.metrics_names, new_nodes)
return cloned_model
def ResNet50(input_shape=None, classes=10, **kwargs):
# Define the input as a tensor with shape input_shape
x_input = Input(input_shape)
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
#stage 1
with tf.name_scope('stage1'):
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(x_input)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = layers.Activation('relu')(x)
print("stage1:" + str(x.shape))
#stage 2
with tf.name_scope('stage2'):
x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
print("stage2:" + str(x.shape))
#stage 3
with tf.name_scope('stage3'):
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
print("stage3:" + str(x.shape))
#stage 4
with tf.name_scope('stage4'):
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
print("stage4:" + str(x.shape))
#stage 5
with tf.name_scope('stage5'):
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
print("stage5:" + str(x.shape))
#full-connected layer
with tf.name_scope('fc'):
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='fc10')(x)
# Create model.
model = models.Model(x_input, x, name='resnet50')
return model
# In[15]:
# Get images
X = []
for filename in os.listdir('dataset/images/train/'):
X.append(img_to_array(load_img('dataset/images/train/'+filename)))
X = np.array(X, dtype=float)
Xtrain = 1.0/255*X
Xtrain, XValid = train_test_split(Xtrain, test_size=0.15)
#Load weights
# inception = InceptionResNetV2(weights=None, include_top=True)
# inception.load_weights('dataset/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5')
# inception.graph = tf.get_default_graph()
embed_input = Input(shape=(5000,))
# In[16]:
#Encoder
encoder_input = Input(shape=(256, 256, 1,))
encoder_output = Conv2D(64, (3,3), activation='relu', padding='same', strides=2)(encoder_input)
encoder_output = Conv2D(128, (3,3), activation='relu', padding='same')(encoder_output)
encoder_output = Conv2D(128, (3,3), activation='relu', padding='same', strides=2)(encoder_output)
encoder_output = Conv2D(256, (3,3), activation='relu', padding='same')(encoder_output)
encoder_output = Conv2D(256, (3,3), activation='relu', padding='same', strides=2)(encoder_output)
encoder_output = Conv2D(512, (3,3), activation='relu', padding='same')(encoder_output)
encoder_output = Conv2D(512, (3,3), activation='relu', padding='same')(encoder_output)
encoder_output = Conv2D(256, (3,3), activation='relu', padding='same')(encoder_output)
def ResNet(input_shape, l1_coef, depth=20, dropout=0.2, num_classes=10):
def resnet_layer(inputs,
num_filters=16,
kernel_size=3,
strides=1,
activation="relu",
batch_normalization=True,
conv_first=True):
conv = Conv2D(num_filters,
kernel_size=kernel_size,
strides=strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))
x = inputs
if conv_first:
x = conv(x)
if batch_normalization:
x = BatchNormalization()(x)
if activation is not None:
x = Activation(activation)(x)
else:
if batch_normalization:
x = BatchNormalization()(x)
if activation is not None:
x = Activation(activation)(x)
x = conv(x)
return x
if (depth - 2) % 6 != 0:
raise ValueError("depth should be 6n+2 (eg 20, 32, 44 in [a])")
num_filters = 16
num_res_blocks = int((depth - 2) / 6)
inputs = Input(shape=input_shape)
x = resnet_layer(inputs=inputs)
# Instantiate the stack of residual units
for stack in range(3):
for res_block in range(num_res_blocks):
strides = 1
if stack > 0 and res_block == 0: # First layer but not first stack
strides = 2 # Downsample
y = resnet_layer(inputs=x,
num_filters=num_filters,
strides=strides)
y = Dropout(dropout)(y)
y = resnet_layer(inputs=y,
num_filters=num_filters,
activation=None)
if stack > 0 and res_block == 0: # First layer but not first stack
# Linear projection residual shortcut connection to match
# changed dims
x = resnet_layer(inputs=x,
num_filters=num_filters,
kernel_size=1,
strides=strides,
activation=None,
batch_normalization=False)
x = Add()([x, y])
x = Activation("relu")(x)
num_filters *= 2
x = AveragePooling2D(pool_size=(int(x.shape[1]), int(x.shape[2])))(x)
y = Flatten()(x)
outputs = Dense(num_classes,
activation="softmax",
kernel_initializer="he_normal")(y)
# Instantiate model
model = Model(inputs=inputs, outputs=outputs)
return model
regularizers.serialize(self.kernel_regularizer),
'bias_regularizer':
regularizers.serialize(self.bias_regularizer),
'activity_regularizer':
regularizers.serialize(self.activity_regularizer),
'kernel_constraint':
constraints.serialize(self.kernel_constraint),
'bias_constraint':
constraints.serialize(self.bias_constraint)
}
base_config = super(Sampling, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
if __name__ == "__main__":
inputs = Input(shape=(4, ))
target = Input(shape=(1, ),
dtype=tf.int32) # sparse format, e.g. [1, 3, 2, 6, ...]
net = Dense(8)(inputs)
net = Sampling(units=128,
num_sampled=32,
type='sampled_softmax',
num_true=3)([net, target])
model = Model(inputs=[inputs, target], outputs=net)
model.compile(optimizer='adam', loss=None)
x = np.random.rand(1000, 4)
y = np.random.randint(128, size=1000)
history = model.fit([x, y], None)
for key in history.history.keys():
print(key)
print(len(y))
for j in range(min(args.mc_top_n, len(d))):
for k, feat in enumerate(all_features):
if feat in d[j]:
# TODO log transform..
X[j * len(all_features) + k] = d[j][feat]
all_x_test.append(X)
all_x_test = np.stack(all_x_test, axis=0)
print(np.shape(all_x_test))
dropout_prob = 0.3
batch_size = 32
hidden_dims = 128
input_shape = ((len(all_features) * args.mc_top_n), )
model_input = Input(shape=input_shape)
z = Dense(hidden_dims, activation="relu")(model_input)
z = Dropout(dropout_prob)(z)
z = Dense(int(hidden_dims / 2), activation="relu")(z)
z = Dropout(dropout_prob)(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
# Training parameters
num_epochs = 25
learning_rate = 0.0001
def my_cuystom_loss(y_true, y_pred):
return keras.losses.binary_crossentropy(y_true,
y_pred,
def ResNet50(input_shape=(30000, 1),
max_pool_s=10,
max_strides=5,
kernel_size=3,
strides=2,
f=3,
ave_pool_size=5,
n_out=1):
"""
Implementation of the popular ResNet50 the following architecture:
CONV1D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3
-> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER
Arguments:
input_shape -- shape of the 1D data
n_out -- integer, number of classes or output
Returns:
model -- a Model() instance in Keras
params here were used in one of my projects.
"""
# Define the input as a tensor with shape input_shape
X_input = Input(input_shape)
X = MaxPooling1D(max_pool_s, max_strides)(X_input)
# Zero-Padding
X = ZeroPadding1D(3)(X)
# stage 1, 64 filters, kernel_size=7
X = Conv1D(64,
kernel_size,
strides=1,
name='conv1',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=2, name='bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling1D(3, strides=2)(X)
# Stage 2
X = convolutional_block(X,
f,
filters=[16, 16, 64],
stage=2,
block='a',
s=1)
X = identity_block(X, f, [16, 16, 64], stage=2, block='b')
X = identity_block(X, f, [16, 16, 64], stage=2, block='c')
### START CODE HERE ###
X = convolutional_block(X,
f,
filters=[32, 32, 128],
stage=3,
block='a',
s=2)
X = identity_block(X, f, [32, 32, 128], stage=3, block='b')
X = identity_block(X, f, [32, 32, 128], stage=3, block='c')
X = identity_block(X, f, [32, 32, 128], stage=3, block='d')
# Stage 4 (≈6 lines)
X = convolutional_block(X,
f,
filters=[64, 64, 256],
stage=4,
block='a',
s=2)
X = identity_block(X, f, [64, 64, 256], stage=4, block='b')
X = identity_block(X, f, [64, 64, 256], stage=4, block='c')
X = identity_block(X, f, [64, 64, 256], stage=4, block='d')
X = identity_block(X, f, [64, 64, 256], stage=4, block='e')
X = identity_block(X, f, [64, 64, 256], stage=4, block='f')
# Stage 5
X = convolutional_block(X,
f,
filters=[64, 64, 256],
stage=5,
block='a',
s=2)
X = identity_block(X, f, [64, 64, 256], stage=5, block='b')
X = identity_block(X, f, [64, 64, 256], stage=5, block='c')
X = AveragePooling1D(ave_pool_size)(X)
# output layer
X = Flatten()(X)
X = Dropout(0.5)(X)
# For regression
X = Dense(n_out,
name='fc-dense',
kernel_initializer=glorot_uniform(seed=0),
kernel_regularizer=regularizers.l2(0.2),
bias_regularizer=regularizers.l2(0.2))(X)
# for classification, if n_out =1, add:
# X = Activation('sigmoid')(X)
# for classification, if n_out > 1, add:
# X = Activation('softmax')(X)
# Create model
model = Model(inputs=X_input, outputs=X, name='ResNet50_1d')
return model
def test_adapt_preprocessing_stage_with_dict_input(self):
x0 = Input(shape=(3, ), name='x0')
x1 = Input(shape=(4, ), name='x1')
x2 = Input(shape=(3, 5), name='x2')
# dimension will mismatch if x1 incorrectly placed.
x1_sum = core.Lambda(
lambda x: tf.reduce_sum(x, axis=-1, keepdims=True))(x1)
x2_sum = core.Lambda(lambda x: tf.reduce_sum(x, axis=-1))(x2)
l0 = PLMerge()
y = l0([x0, x1_sum])
l1 = PLMerge()
y = l1([y, x2_sum])
l2 = PLSplit()
z, y = l2(y)
stage = preprocessing_stage.FunctionalPreprocessingStage(
{
'x2': x2,
'x0': x0,
'x1': x1
}, [y, z])
stage.compile()
# Test with dict of NumPy array
one_array0 = np.ones((4, 3), dtype='float32')
one_array1 = np.ones((4, 4), dtype='float32')
one_array2 = np.ones((4, 3, 5), dtype='float32')
adapt_data = {'x1': one_array1, 'x0': one_array0, 'x2': one_array2}
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 1)
self.assertEqual(l1.adapt_count, 1)
self.assertEqual(l2.adapt_count, 1)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
self.assertLessEqual(l1.adapt_time, l2.adapt_time)
# Check call
y, z = stage({
'x1': tf.constant(one_array1),
'x2': tf.constant(one_array2),
'x0': tf.constant(one_array0)
})
self.assertAllClose(y, np.zeros((4, 3), dtype='float32') + 9.)
self.assertAllClose(z, np.zeros((4, 3), dtype='float32') + 11.)
# Test with list of NumPy array
adapt_data = [one_array0, one_array1, one_array2]
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 2)
self.assertEqual(l1.adapt_count, 2)
self.assertEqual(l2.adapt_count, 2)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
self.assertLessEqual(l1.adapt_time, l2.adapt_time)
# Test with flattened dataset
adapt_data = tf.data.Dataset.from_tensor_slices(
(one_array0, one_array1, one_array2))
adapt_data = adapt_data.batch(2) # 5 batches of 2 samples
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 3)
self.assertEqual(l1.adapt_count, 3)
self.assertEqual(l2.adapt_count, 3)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
self.assertLessEqual(l1.adapt_time, l2.adapt_time)
# Test with dataset in dict shape
adapt_data = tf.data.Dataset.from_tensor_slices({
'x0': one_array0,
'x2': one_array2,
'x1': one_array1
})
adapt_data = adapt_data.batch(2) # 5 batches of 2 samples
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 4)
self.assertEqual(l1.adapt_count, 4)
self.assertEqual(l2.adapt_count, 4)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
self.assertLessEqual(l1.adapt_time, l2.adapt_time)
# Test error with bad data
with self.assertRaisesRegex(ValueError, 'requires a '):
stage.adapt(None)
maxXval = XtestList[0].toarray()
minXval = XtestList[0].toarray()
sumXval = XtestList[0].toarray()
for i in range(1, len(XtrainList)):
maxXval = np.maximum(maxXval, XtestList[i].toarray())
minXval = np.minimum(minXval, XtestList[i].toarray())
sumXval = sumXval + XtestList[i].toarray()
Y_val = np_utils.to_categorical(Y_test - 1, 1000)
tensorboard = TensorBoard(log_dir='/media/jsl18/Lexar/exp6',
histogram_freq=1,
write_graph=True,
write_images=False)
nb_hidden = 50
input1 = Input(shape=(1000, ))
input2 = Input(shape=(1000, ))
input3 = Input(shape=(1000, ))
concatL = Concatenate()([input1, input2, input3])
#layer1=Dense(nb_hidden,bias_initializer='zeros')(concatL)
#layer1=LeakyReLU(alpha=0.01)(layer1)
out = Dense(nb_classes, bias_initializer='zeros',
activation='softmax')(concatL)
model = Model([input1, input2, input3], out)
#model=load_model('genFit_ens450.h5');
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy', top3_acc, top5_acc])
G = myGeneratorTrain()
K = next(G)
def fill_zipf(length, input_width, output_width, num_true=1):
input_onehot = np.zeros((length, input_width), dtype='float32')
output_labels = np.zeros((length, num_true), dtype='int32')
rand = np.random.zipf(1.2, length * num_true) % input_width
for i in range(length):
for t in range(num_true):
k = rand[t * length + i]
input_onehot[i][k] = 1.0
output_labels[i][t] = min(output_width - 1, int(k * output_width/input_width))
return input_onehot, output_labels
# choose one of the two
sampling_type='sampled_softmax'
sampling_type='nce'
inputs = Input(shape=(input_width,))
target = Input(shape=(num_true,), dtype=tf.int64)
net = Dense(input_width)(inputs)
net = Dense(num_hidden, activation='relu')(net)
net = Sampling(units=output_width, num_sampled=num_sampled, type=sampling_type, num_true=num_true)([net, target])
model = Model(inputs=[inputs, target], outputs=net)
model.compile(optimizer='adam', loss=None, metrics=['binary_crossentropy'])
model.summary()
train_onehot, train_labels = fill_zipf(num_train, input_width, output_width, num_true)
model.fit([train_onehot, train_labels], None,
epochs=num_epochs, verbose=2)
test_input, test_output = fill_zipf(num_test, input_width, output_width, num_true)
predicts = model.predict([test_input, test_output], batch_size=32)
count = 0
def create_model(self,
df,
features,
look_back=24,
future=6,
n1=256,
epochs=30,
train=True,
load_model=None,
lr=0.001,
mode='3-output',
step=1,
name=''):
if mode == '3-output':
self.indices = [10, 11, 12]
elif mode == '1-output':
self.indices = [15]
elif mode == 'reverse':
# want to predict Facilitites-> heating, cooling
# need to create new feature-> sum of facilitites
self.indices = [16]
else:
print('Mode can only be: reverse, 3-output, 1-output')
exit()
self.mode = mode
self.name = name
path = Path(self.path_plots)
xtrain, xtest, xval = self.create_train_test(df, features)
xtrain, ytrain = self.create_dataset(xtrain, look_back, future, step)
xtest, ytest = self.create_dataset(xtest, look_back, future, step=1)
xval, yval = self.create_dataset(xval, look_back, future, step=1)
print(
f'Train on {xtrain.shape}, validate on {xval.shape} and test on {xtest.shape}, predict for {ytrain.shape[1]}'
)
#LSTM needs as input: [samples, timesteps, features]
n_outputs = ytrain.shape[2]
future = ytrain.shape[1]
if train:
batch_size = 32
training_steps = xtrain.shape[0] // batch_size
validation_steps = xval.shape[0] // batch_size
inp = Input(shape=(xtrain.shape[1], xtrain.shape[2]))
gru1 = GRU(n1, return_sequences=True, dropout=0.2)(inp)
gru2 = GRU(n1 // 2,
return_sequences=True,
dropout=0.3,
recurrent_dropout=0.2)(gru1)
gru3 = GRU(n1 // 2,
return_sequences=True,
dropout=0.5,
recurrent_dropout=0.4)(gru2)
gru4 = GRU(n1 // 4,
return_sequences=True,
dropout=0.5,
recurrent_dropout=0.3)(gru3)
# gru5 = GRU(n1//4, return_sequences=True, dropout=0.5, recurrent_dropout=0.3)(gru4)
# gru6 = GRU(n1//4, return_sequences=True, dropout=0.5, recurrent_dropout=0.3)(gru5)
gru8 = GRU(n1 // 4,
return_sequences=True,
dropout=0.5,
recurrent_dropout=0.3)(gru4)
gru9 = GRU(n1 // 8, dropout=0.3, recurrent_dropout=0.3)(gru8)
#output_layers: n layers x n_neurons==future (how much in the future to predict)
out = [Dense(future)(gru9) for i in range(n_outputs)]
model = Model(inputs=inp, outputs=out)
print(model.summary())
plot_model(model,
to_file=f'{path}/Model_{self.mode}.png',
show_shapes=True,
dpi=300)
model.compile(loss='mae', optimizer=Adam(learning_rate=lr))
history = model.fit(
xtrain,
[ytrain[:, :, i] for i in range(n_outputs)],
epochs=epochs,
validation_data=(xval,
[yval[:, :, i] for i in range(n_outputs)]),
batch_size=batch_size,
callbacks=[EarlyStoppingAtNormDifference()],
# verbose=0,
)
self.save_model(model, name)
self.plot_loss(history)
else:
if load_model == None:
raise NoModelFound
else:
model = load_model
self.evaluate(model, xtrain, ytrain, 'Train')
self.evaluate(model, xval, yval, 'Validation', new=0)
self.evaluate(model, xtest, ytest, new=0)
return model
def _clone_functional_model(model, input_tensors=None, layer_fn=_clone_layer):
"""Clone a functional `Model` instance.
Model cloning is similar to calling a model on new inputs,
except that it creates new layers (and thus new weights) instead
of sharing the weights of the existing layers.
Input layers are always cloned.
Args:
model: Instance of `Model`.
input_tensors: optional list of input tensors
to build the model upon. If not provided,
placeholders will be created.
layer_fn: callable to be applied on non-input layers in the model. By
default it clones the layer. Another example is to preserve the layer
to share the weights. This is required when we create a per-replica
copy of the model with distribution strategy; we want the weights to
be shared but still feed inputs separately so we create new input
layers.
Returns:
An instance of `Model` reproducing the behavior
of the original model, on top of new inputs tensors,
using newly instantiated weights.
Raises:
ValueError: in case of invalid `model` argument value or `layer_fn`
argument value.
"""
if not isinstance(model, Model):
raise ValueError(
'Expected `model` argument '
'to be a `Model` instance, got ', model)
if isinstance(model, Sequential):
raise ValueError(
'Expected `model` argument '
'to be a functional `Model` instance, '
'got a `Sequential` instance instead:', model)
if not model._is_graph_network:
raise ValueError('Expected `model` argument '
'to be a functional `Model` instance, '
'but got a subclass model instead.')
new_input_layers = {} # Cache for created layers.
if input_tensors is not None:
# Make sure that all input tensors come from a Keras layer.
input_tensors = tf.nest.flatten(input_tensors)
for i, input_tensor in enumerate(input_tensors):
original_input_layer = model._input_layers[i]
# Cache input layer. Create a new layer if the tensor is originally not
# from a Keras layer.
if not backend.is_keras_tensor(input_tensor):
name = original_input_layer.name
input_tensor = Input(tensor=input_tensor,
name='input_wrapper_for_' + name)
newly_created_input_layer = input_tensor._keras_history.layer
new_input_layers[
original_input_layer] = newly_created_input_layer
else:
new_input_layers[original_input_layer] = original_input_layer
if not callable(layer_fn):
raise ValueError('Expected `layer_fn` argument to be a callable.')
model_configs, created_layers = _clone_layers_and_model_config(
model, new_input_layers, layer_fn)
# Reconstruct model from the config, using the cloned layers.
input_tensors, output_tensors, created_layers = (
functional.reconstruct_from_config(model_configs,
created_layers=created_layers))
metrics_names = model.metrics_names
model = Model(input_tensors, output_tensors, name=model.name)
# Layers not directly tied to outputs of the Model, such as loss layers
# created in `add_loss` and `add_metric`.
ancillary_layers = [
layer for layer in created_layers.values() if layer not in model.layers
]
# TODO(b/162887610): This may need to adjust the inbound node index if the
# created layers had already been used to define other models.
if ancillary_layers:
new_nodes = tf.nest.flatten([
layer.inbound_nodes[1:] if
functional._should_skip_first_node(layer) else layer.inbound_nodes
for layer in created_layers.values()
])
_insert_ancillary_layers(model, ancillary_layers, metrics_names,
new_nodes)
return model
test_x = np.random.rand(10, 64, 83).astype('f')
test_y = np.random.rand(10, 1).astype('f')
model = Sequential([
LSTM(16, return_sequences=True, input_shape=(64, 83)),
LSTM(16, return_sequences=False),
Dense(1, activation='sigmoid')
])
output_testcase(model, test_x, test_y, 'lstm_stacked_64x83', '1e-6')
''' Embedding 64 '''
np.random.seed(10)
test_x = np.random.randint(100, size=(32, 10)).astype('f')
test_y = np.random.rand(32, 20).astype('f')
model = Sequential([
Embedding(100, 64, input_length=10),
Flatten(),
Dense(20, activation='sigmoid')
])
output_testcase(model, test_x, test_y, 'embedding_64', '1e-6')
''' Input '''
test_x = np.random.rand(10, 10).astype('f')
test_y = np.random.rand(10).astype('f')
a = Input(shape=(10, ))
b = Dense(1)(a)
model = Model(inputs=a, outputs=b)
output_testcase(model, test_x, test_y, 'input', '1e-6')
''' RepeatVector '''
test_x = np.random.rand(10, 32).astype('f')
test_y = np.random.rand(10, 3, 32).astype('f')
model = Sequential([Dense(32, input_dim=32), RepeatVector(3)])
output_testcase(model, test_x, test_y, 'repeat_vector', '1e-6')
def get_config(self):
config = {
'units': self.units,
'num_sampled': self.num_sampled,
'kernel_initializer': initializers.serialize(self.kernel_initializer),
'bias_initializer': initializers.serialize(self.bias_initializer),
'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
'activity_regularizer': regularizers.serialize(self.activity_regularizer),
'kernel_constraint': constraints.serialize(self.kernel_constraint),
'bias_constraint': constraints.serialize(self.bias_constraint)
}
base_config = super(NCE, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
if __name__ == "__main__":
inputs = Input(shape=(4,))
target = Input(shape=(1,)) # sparse format, e.g. [1, 3, 2, 6, ...]
net = Dense(8)(inputs)
net = NCE(units=128, num_sampled=32)([net, target])
model = Model(inputs=[inputs, target], outputs=net)
model.compile(optimizer='adam', loss=None)
x = np.random.rand(1000, 4)
y = np.random.randint(128, size=1000)
history = model.fit([x, y], None)
for key in history.history.keys():
print(key)
print(len(y))
print(y)
def createParametricBCNN(optimisingParameters, inputShape, outputLayerNames,
objectTypePossibleLabelSets, gpuQuantity):
dropoutFraction, convolutionLayersPerBlock, extraFirstBlock, initalLayerFilterCount, filterCountBlockMultiplicativeFactor, initalLayerKernalSize, kernalSizeBlockMultiplicitiveFactor, learningRate = optimisingParameters
#extraFirstBlock: Whether an extra block is to be included before the first output.
#filterCountBlockMultiplicativeFactor: The amount of filters the convolution layers in a block have compared to the previous block.
#kernalSizeBlockMultiplicitiveFactor: #The kernal size for convolutional layers in a block compared to the previous block.
currentFilterCount = initalLayerFilterCount
currentKernalSize = initalLayerKernalSize
mainInput = Input(shape=inputShape, name="input")
network = mainInput
outputs = [None for i in range(0, len(outputLayerNames))]
if (extraFirstBlock):
for i in range(0, convolutionLayersPerBlock
): #Creates the convolution layers in the first block.
network = Conv2D(currentFilterCount,
(currentKernalSize, currentKernalSize),
padding="same",
activation="relu")(network)
network = MaxPooling2D(pool_size=(2, 2))(network)
network = Dropout(dropoutFraction)(network)
currentFilterCount *= filterCountBlockMultiplicativeFactor
currentKernalSize *= kernalSizeBlockMultiplicitiveFactor
#The rest of the network is made below.
for currentIndex, currentOutputName in enumerate(
outputLayerNames
): #Loops over all outputs; each output is associated with a block.
for i in range(0, convolutionLayersPerBlock
): #Creates the convolution layers in the block
currentIntegerFilterCount = int(currentFilterCount)
currentIntegerKernalSize = math.floor(currentKernalSize)
currentIntegerKernalSize = currentIntegerKernalSize + 1 if (
currentIntegerKernalSize % 2 == 0
) else currentIntegerKernalSize #Makes sure that the kernal size is an odd number.
currentIntegerKernalSize = max(
3, currentIntegerKernalSize
) #Makes sure that the kernal size is no smaller than 3.
network = Conv2D(
currentIntegerFilterCount,
(currentIntegerKernalSize, currentIntegerKernalSize),
padding="same",
activation="relu")(network)
#The output layers for this block are created.
outputs[currentIndex] = Conv2D(
len(objectTypePossibleLabelSets[currentIndex]), (3, 3),
padding="same",
activation="relu",
name=currentOutputName + "LocationHeatmap")(network)
outputs[currentIndex] = GlobalAveragePooling2D()(outputs[currentIndex])
outputs[currentIndex] = Activation("softmax", name=currentOutputName)(
outputs[currentIndex])
if ((currentIndex + 1) < len(outputLayerNames)
): #If there are still blocks left to create.
network = MaxPooling2D(pool_size=(2, 2))(network)
network = Dropout(dropoutFraction)(network)
currentFilterCount *= filterCountBlockMultiplicativeFactor
currentKernalSize *= kernalSizeBlockMultiplicitiveFactor
if (gpuQuantity >= 2):
#Initally creates the model on a CPU instead of a GPU.
with tf.device("/cpu:0"):
model = Model(inputs=mainInput, outputs=outputs)
gpuModel = multi_gpu_model(model, gpus=gpuQuantity)
return model, gpuModel
else:
model = Model(inputs=mainInput, outputs=outputs)
return model, model
test_data.append(
np.reshape(test_words, (input_dimension, input_dimension)))
new_test_label = np.zeros(num_categories)
new_test_label[i % num_categories] = 1
test_label.append(new_test_label)
i += 1
#Convert data into arrays for testing and establish random seed
np.random.seed()
max_review_length = article_length
test_data = np.asarray(test_data)
test_label = np.asarray(test_label)
#Create model
inputs = Input((input_dimension, input_dimension))
m = Lambda(lambda x: x)(inputs)
m = Reshape((max_review_length, ))(m)
#Embed vectors into vector space
m = Embedding(dictionary_size,
embedding_vector_length,
input_length=max_review_length)(m)
#LSTM
m = (CuDNNLSTM(100))(m)
#Softmax(prediction)
outputs = (Dense(num_categories, activation='softmax'))(m)
#Put together model parts
model = Model(inputs=[inputs], outputs=[outputs])
#Compile model
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-3, momentum=0.9),
# dimensions of our images.
img_width, img_height = 640, 480
CROP_TOP = int(200)
CROP_BOTTOM = int(30)
nb_train_samples = nr_train_samples
nb_validation_samples = nr_val_samples
batch_size = 8
if K.image_data_format() == 'channels_first':
input_shape = (3, img_height, img_width)
else:
input_shape = (img_height, img_width, 3)
inputs = Input(shape=input_shape)
x = Cropping2D(cropping=((CROP_TOP, CROP_BOTTOM), (0, 0)),
input_shape=input_shape)(inputs)
x = GaussianNoise(0.5)(x)
x = darknet_base(x)
x = GlobalAveragePooling2D()(x)
x = Dense(3, activation='softmax')(x)
model = Model(inputs, x)
#model = load_model("models/weights.358-3.57-0.64.hdf5")
# In[86]:
#model=darknet()
sgd = optimizers.SGD(lr=0.00001, decay=1e-6, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=1e-6)
X_test_polar /= 255 X_train_BGR = (X_train_BGR - 0.5) * 2 X_train_polar = (X_train_polar - 0.5) * 2 X_test_BGR = (X_test_BGR - 0.5) * 2 X_test_polar = (X_test_polar - 0.5) * 2 # On a maintenant les vecteurs prets pour l'apprentissage, # mettons donc en place le réseau de neurones model = ResNet50(weights='imagenet', include_top=False) num_classes = 2 print(model.input) input = Input(shape=(190, 254, 3)) x = Flatten()(model(input)) x = Dense(1000, activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.5)(x) x = Dense(2, activation='softmax')(x) new_model = Model(inputs=input, outputs=x) new_model.summary() # Puisque l'on a 89 images de train et de test, il est nécessaire # d'avoir plus de données pour pouvoir effectuer un apprentissage # pertinent. Pour cela, on va utiliser des techniques de data # augmentation.
def unet_inception_tr(pretrained_weights=None, input_size=(240, 240, 3)):
inputs = Input(input_size)
inception1_1 = inception_module(inputs,
filters_1x1=16,
filters_3x3_reduce=8,
filters_3x3=16,
filters_5x5_reduce=8,
filters_5x5=16,
filters_pool_proj=16,
name="inception1")
inception1_2 = inception_module(
inception1_1,
filters_1x1=16,
filters_3x3_reduce=8,
filters_3x3=16,
filters_5x5_reduce=8,
filters_5x5=16,
filters_pool_proj=16,
)
concat_up_1 = Conv2D(16, (3, 3),
padding='same',
kernel_initializer='he_normal')(inception1_2)
concat_up_1 = BatchNormalization()(concat_up_1)
concat_up_1 = Activation('relu')(concat_up_1)
pool1 = MaxPooling2D(pool_size=(2, 2))(inception1_2)
inception2_1 = inception_module(
pool1,
filters_1x1=32,
filters_3x3_reduce=16,
filters_3x3=32,
filters_5x5_reduce=16,
filters_5x5=32,
filters_pool_proj=32,
)
inception2_2 = inception_module(
inception2_1,
filters_1x1=32,
filters_3x3_reduce=16,
filters_3x3=32,
filters_5x5_reduce=16,
filters_5x5=32,
filters_pool_proj=32,
)
concat_up_2 = Conv2D(32, (3, 3),
padding='same',
kernel_initializer='he_normal')(inception2_2)
concat_up_2 = BatchNormalization()(concat_up_2)
concat_up_2 = Activation('relu')(concat_up_2)
pool2 = MaxPooling2D(pool_size=(2, 2))(inception2_2)
inception3_1 = inception_module(
pool2,
filters_1x1=64,
filters_3x3_reduce=32,
filters_3x3=64,
filters_5x5_reduce=32,
filters_5x5=64,
filters_pool_proj=64,
)
inception3_2 = inception_module(
inception3_1,
filters_1x1=64,
filters_3x3_reduce=32,
filters_3x3=64,
filters_5x5_reduce=32,
filters_5x5=64,
filters_pool_proj=64,
)
concat_up_3 = Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal')(inception3_2)
concat_up_3 = BatchNormalization()(concat_up_3)
concat_up_3 = Activation('relu')(concat_up_3)
pool3 = MaxPooling2D(pool_size=(2, 2))(inception3_2)
inception4_1 = inception_module(
pool3,
filters_1x1=128,
filters_3x3_reduce=64,
filters_3x3=128,
filters_5x5_reduce=64,
filters_5x5=128,
filters_pool_proj=128,
)
inception4_2 = inception_module(
inception4_1,
filters_1x1=128,
filters_3x3_reduce=64,
filters_3x3=128,
filters_5x5_reduce=64,
filters_5x5=128,
filters_pool_proj=128,
)
drop4 = Dropout(0.5)(inception4_2)
concat_up_4 = Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal')(drop4)
concat_up_4 = BatchNormalization()(concat_up_4)
concat_up_4 = Activation('relu')(concat_up_4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
inception5_1 = inception_module(
pool4,
filters_1x1=256,
filters_3x3_reduce=128,
filters_3x3=256,
filters_5x5_reduce=128,
filters_5x5=256,
filters_pool_proj=256,
)
inception5_2 = inception_module(
inception5_1,
filters_1x1=256,
filters_3x3_reduce=128,
filters_3x3=256,
filters_5x5_reduce=128,
filters_5x5=256,
filters_pool_proj=256,
)
drop5 = Dropout(0.5)(inception5_2)
up6 = Conv2D(512,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(drop5))
merge6 = concatenate([concat_up_4, up6], axis=3)
inception6_1 = inception_module(
merge6,
filters_1x1=128,
filters_3x3_reduce=64,
filters_3x3=128,
filters_5x5_reduce=64,
filters_5x5=128,
filters_pool_proj=128,
)
inception6_2 = inception_module(
inception6_1,
filters_1x1=128,
filters_3x3_reduce=64,
filters_3x3=128,
filters_5x5_reduce=64,
filters_5x5=128,
filters_pool_proj=128,
)
up7 = Conv2D(256,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling2D(size=(2, 2))(inception6_2))
merge7 = concatenate([concat_up_3, up7], axis=3)
inception7_1 = inception_module(
merge7,
filters_1x1=64,
filters_3x3_reduce=32,
filters_3x3=64,
filters_5x5_reduce=32,
filters_5x5=64,
filters_pool_proj=64,
)
inception7_2 = inception_module(
inception7_1,
filters_1x1=64,
filters_3x3_reduce=32,
filters_3x3=64,
filters_5x5_reduce=32,
filters_5x5=64,
filters_pool_proj=64,
)
up8 = Conv2D(128,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling2D(size=(2, 2))(inception7_2))
merge8 = concatenate([concat_up_2, up8], axis=3)
inception8_1 = inception_module(
merge8,
filters_1x1=32,
filters_3x3_reduce=16,
filters_3x3=32,
filters_5x5_reduce=16,
filters_5x5=32,
filters_pool_proj=32,
)
inception8_2 = inception_module(
inception8_1,
filters_1x1=32,
filters_3x3_reduce=16,
filters_3x3=32,
filters_5x5_reduce=16,
filters_5x5=32,
filters_pool_proj=32,
)
up9 = Conv2D(64,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling2D(size=(2, 2))(inception8_2))
merge9 = concatenate([concat_up_1, up9], axis=3)
inception9_1 = inception_module(
merge9,
filters_1x1=16,
filters_3x3_reduce=8,
filters_3x3=16,
filters_5x5_reduce=8,
filters_5x5=16,
filters_pool_proj=16,
)
inception9_2 = inception_module(
inception9_1,
filters_1x1=16,
filters_3x3_reduce=8,
filters_3x3=16,
filters_5x5_reduce=8,
filters_5x5=16,
filters_pool_proj=16,
)
inception9_3 = inception_module(
inception9_2,
filters_1x1=4,
filters_3x3_reduce=2,
filters_3x3=4,
filters_5x5_reduce=2,
filters_5x5=4,
filters_pool_proj=4,
)
conv10 = Conv2D(4, 1, activation='softmax')(inception9_3)
model = Model(inputs=inputs, outputs=conv10)
model.compile(optimizer=Adam(lr=1e-4),
loss='categorical_crossentropy',
metrics=['accuracy', dice_coef])
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
