def get_vgg16(input_shape,
final_activation='softmax',
weights=None,
classes=10,
weight_decay=0.0005):
if weights is not None:
raise NotImplementedError('Weight load is not implemented.')
model = Sequential()
model.add(
layers.Conv2D(64, (3, 3),
padding='same',
input_shape=input_shape,
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.3))
model.add(
layers.Conv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(
layers.Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(
layers.Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(
layers.Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(
layers.Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.4))
model.add(
layers.Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(layers.Dropout(0.5))
model.add(layers.Flatten())
model.add(
layers.Dense(512, kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.5))
model.add(layers.Dense(classes))
model.add(layers.Activation(final_activation))
return model
def get_model(num_classes, model_name=MODEL_NAME, size_of_image=SIZE_OF_IMAGE):
"""
Returns a CNN model, or None if the parameter model is not recognised.
Parameters:
num_classes (int): Total number of different classes of Pokemon.
model (string): One of the 3 models (custom, MobileNetV2, VGG19). Default: config.MODEL.
size_of_image (int): Size of image after resizing. Default: config.SIZE_OF_IMAGE.
Returns:
model (tf.keras.Model): A CNN model.
"""
if model_name == 'custom':
""" Custom CNN Model """
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
input_shape=(size_of_image, size_of_image, CHANNEL)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(196, activation='relu'))
# model.add(layers.Dropout(0.3))
model.add(layers.Dense(num_classes))
elif model_name == 'MobileNetV2':
""" MobileNetV2 """
base_model = applications.MobileNetV2(input_shape=(size_of_image, size_of_image, CHANNEL),
include_top=False,
weights='imagenet')
base_model.trainable = False
global_average_layer = layers.GlobalAveragePooling2D()
output_layer = layers.Dense(num_classes)
model = tf.keras.Sequential([
base_model,
global_average_layer,
layers.Flatten(),
layers.Dense(196, activation='relu'),
layers.Dropout(0.3),
output_layer
])
elif model_name == 'VGG19':
""" VGG19 """
base_model = applications.VGG19(input_shape=(size_of_image, size_of_image, CHANNEL),
include_top=False,
weights='imagenet')
base_model.trainable = False
global_average_layer = layers.GlobalAveragePooling2D()
output_layer = layers.Dense(num_classes)
model = tf.keras.Sequential([
base_model,
global_average_layer,
layers.Flatten(),
layers.Dense(196, activation='relu'),
layers.Dropout(0.3),
output_layer
])
else:
print("ERROR: Cannot recognise model.")
sys.exit(1)
return model
import tensorflow as tf
from tensorflow.keras import models, datasets, layers
(train_images, train_labels), (test_images,
test_labels) = datasets.mnist.load_data()
train_images, test_images = train_images / 255.0, test_images / 255.0
model = models.Sequential([
layers.Flatten(input_shape=(28, 28)),
layers.Dense(128, activation='relu'),
layers.Dropout(.3),
layers.Dense(128, activation='relu'),
layers.Dropout(.3),
layers.Dense(10, activation='softmax')
])
model.summary()
model.compile(
optimizer='Adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.fit(train_images, train_labels, epochs=10, validation_split=.2)
test_loss, test_acc = model.evaluate(test_images, test_labels)
print('Test Accuracy: ', test_acc)
model.save('./models')
def EfficientNet(width_coefficient,
depth_coefficient,
default_resolution,
dropout_rate=0.2,
drop_connect_rate=0.2,
depth_divisor=8,
blocks_args=DEFAULT_BLOCKS_ARGS,
model_name='EfficientNet',
include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the EfficientNet architecture using given scaling coefficients.
Optionally loads weights pre-trained on ImageNet.
Note that the data format convention used by the model is
the one specified in your Keras config at `~/.keras/keras.json`.
# Arguments
width_coefficient: float, scaling coefficient for network width.
depth_coefficient: float, scaling coefficient for network depth.
default_resolution: int, default input image size.
dropout_rate: float, dropout rate before final classifier layer.
drop_connect_rate: float, dropout rate at skip connections.
depth_divisor: int.
blocks_args: A list of BlockArgs to construct block modules.
model_name: string, model name.
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor
(i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False.
It should have exactly 3 inputs channels.
pooling: optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', 'noisy-student', None}
or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError(
'If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=default_resolution,
min_size=32,
data_format=backend.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if backend.backend() == 'tensorflow':
from tensorflow.python.keras.backend import is_keras_tensor
else:
is_keras_tensor = backend.is_keras_tensor
if not is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
activation = get_swish()
# Build stem
x = img_input
x = layers.Conv2D(round_filters(32, width_coefficient, depth_divisor),
3,
strides=(2, 2),
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name='stem_conv')(x)
x = layers.BatchNormalization(axis=bn_axis, name='stem_bn')(x)
x = layers.Activation(activation, name='stem_activation')(x)
# Build blocks
num_blocks_total = sum(block_args.num_repeat for block_args in blocks_args)
block_num = 0
for idx, block_args in enumerate(blocks_args):
assert block_args.num_repeat > 0
# Update block input and output filters based on depth multiplier.
block_args = block_args._replace(
input_filters=round_filters(block_args.input_filters,
width_coefficient, depth_divisor),
output_filters=round_filters(block_args.output_filters,
width_coefficient, depth_divisor),
num_repeat=round_repeats(block_args.num_repeat, depth_coefficient))
# The first block needs to take care of stride and filter size increase.
drop_rate = drop_connect_rate * float(block_num) / num_blocks_total
x = mb_conv_block(x,
block_args,
activation=activation,
drop_rate=drop_rate,
prefix='block{}a_'.format(idx + 1))
block_num += 1
if block_args.num_repeat > 1:
block_args = block_args._replace(
input_filters=block_args.output_filters, strides=[1, 1])
for bidx in xrange(block_args.num_repeat - 1):
drop_rate = drop_connect_rate * float(
block_num) / num_blocks_total
block_prefix = 'block{}{}_'.format(
idx + 1, string.ascii_lowercase[bidx + 1])
x = mb_conv_block(x,
block_args,
activation=activation,
drop_rate=drop_rate,
prefix=block_prefix)
block_num += 1
# Build top
x = layers.Conv2D(round_filters(1280, width_coefficient, depth_divisor),
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name='top_conv')(x)
x = layers.BatchNormalization(axis=bn_axis, name='top_bn')(x)
x = layers.Activation(activation, name='top_activation')(x)
if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
if dropout_rate and dropout_rate > 0:
x = layers.Dropout(dropout_rate, name='top_dropout')(x)
x = layers.Dense(classes,
activation='softmax',
kernel_initializer=DENSE_KERNEL_INITIALIZER,
name='probs')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D(name='max_pool')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = models.Model(inputs, x, name=model_name)
# Load weights.
if weights == 'imagenet':
if include_top:
file_name = model_name + '_AutoAugment.h5'
else:
file_name = model_name + '_AutoAugment_NoTop.h5'
model.load_weights(os.path.join(os.path.dirname(__file__), file_name))
elif weights == 'noisy-student':
if include_top:
file_name = "{}_{}.h5".format(model_name, 'NoisyStudent')
else:
file_name = "{}_{}_NoTop.h5".format(model_name, 'NoisyStudent')
model.load_weights(os.path.join(os.path.dirname(__file__), file_name))
elif weights is not None:
model.load_weights(weights)
return model
layers.BatchNormalization(),
layers.MaxPooling1D(pool_size=5, strides=2, padding="same"),
layers.Conv1D(256,
kernel_size=5,
strides=1,
padding="same",
activation="relu"),
layers.BatchNormalization(),
layers.MaxPooling1D(pool_size=5, strides=2, padding="same"),
layers.Conv1D(128,
kernel_size=5,
strides=1,
padding="same",
activation="relu"),
layers.MaxPooling1D(pool_size=5, strides=2, padding="same"),
layers.Dropout(0.2),
layers.Conv1D(64,
kernel_size=5,
strides=1,
padding="same",
activation="relu"),
layers.MaxPooling1D(pool_size=5, strides=2, padding="same"),
layers.Flatten(),
layers.Dense(units=32, activation="relu"),
layers.Dropout(0.3),
layers.Dense(units=8, activation="softmax"),
])
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
def __init__(self,
channels,
init_block_channels,
final_block_channels,
final_block_groups,
dilations,
dropout_rate=0.2,
in_channels=3,
in_size=(224, 224),
classes=1000,
data_format="channels_last",
**kwargs):
super(ESPNetv2, self).__init__(**kwargs)
self.in_size = in_size
self.classes = classes
x0_channels = in_channels
self.features = DualPathSequential(return_two=False,
first_ordinals=0,
last_ordinals=2,
name="features")
self.features.add(
ESPInitBlock(in_channels=in_channels,
out_channels=init_block_channels,
data_format=data_format,
name="init_block"))
in_channels = init_block_channels
for i, channels_per_stage in enumerate(channels):
stage = DualPathSequential(name="stage{}_".format(i + 1))
for j, out_channels in enumerate(channels_per_stage):
if j == 0:
unit = DownsampleBlock(in_channels=in_channels,
out_channels=out_channels,
x0_channels=x0_channels,
dilations=dilations[i][j],
data_format=data_format,
name="unit{}".format(j + 1))
else:
unit = ESPBlock(in_channels=in_channels,
out_channels=out_channels,
strides=1,
dilations=dilations[i][j],
data_format=data_format,
name="unit{}".format(j + 1))
stage.add(unit)
in_channels = out_channels
self.features.add(stage)
self.features.add(
ESPFinalBlock(in_channels=in_channels,
out_channels=final_block_channels,
final_groups=final_block_groups,
data_format=data_format,
name="final_block"))
in_channels = final_block_channels
self.features.add(
nn.AveragePooling2D(pool_size=7,
strides=1,
data_format=data_format,
name="final_pool"))
self.output1 = tf.keras.Sequential(name="output1")
self.output1.add(nn.Dropout(rate=dropout_rate, name="dropout"))
self.output1.add(
nn.Dense(units=classes, input_dim=in_channels, name="fc"))
"""## Create the discriminator
It maps a 64x64 image to a binary classification score.
"""
discriminator = keras.Sequential(
[
keras.Input(shape=(64, 64, 3)),
layers.Conv2D(64, kernel_size=4, strides=2, padding="same"),
layers.LeakyReLU(alpha=0.2),
layers.Conv2D(128, kernel_size=4, strides=2, padding="same"),
layers.LeakyReLU(alpha=0.2),
layers.Conv2D(128, kernel_size=4, strides=2, padding="same"),
layers.LeakyReLU(alpha=0.2),
layers.Flatten(),
layers.Dropout(0.2),
layers.Dense(1, activation="sigmoid"),
],
name="discriminator",
)
discriminator.summary()
"""## Create the generator
It mirrors the discriminator, replacing `Conv2D` layers with `Conv2DTranspose` layers.
"""
latent_dim = 128
generator = keras.Sequential(
[
keras.Input(shape=(latent_dim, )),
#load the weights in model
Model.load_weights(name+"_weights.best.h5")
#plot the model
plot(history)
#real testing of model
test_loss, test_acc = Model.evaluate(test_X, test_Y, verbose=2)
print("\n\nAccuracy(",name,") on Test images ==> ",test_acc*100,"% \n")
print("\n\nExact 1-off Accuracy(",name,") on Test images ==> ",get_exact1off_acc(test_X,test_Y,Model),"% \n")
model_age = models.Sequential()
#CONV1
model_age.add(layers.Conv2D(96, (7, 7),strides=(4,4), input_shape=(227, 227, 3)))
model_age.add(layers.Activation('relu'))
model_age.add(layers.BatchNormalization())
model_age.add(layers.MaxPooling2D(pool_size=(2, 2)))
model_age.add(layers.Dropout(0.6))
#CONV2
model_age.add(layers.Conv2D(256, (5, 5)))
model_age.add(layers.Activation('relu'))
model_age.add(layers.BatchNormalization())
model_age.add(layers.MaxPooling2D(pool_size=(2, 2)))
model_age.add(layers.Dropout(0.6))
#CONV3
model_age.add(layers.Conv2D(384, (3, 3)))
model_age.add(layers.Activation('relu'))
model_age.add(layers.BatchNormalization())
model_age.add(layers.MaxPooling2D(pool_size=(2, 2)))
model_age.add(layers.Dropout(0.6))
#CONV4
model_age.add(layers.Conv2D(256, (3, 3)))
model_age.add(layers.Activation('relu'))
def autoencoder(
trainingData,
validationData,
encodingDimension,
trainingLabels=None,
validationLabels=None,
activation='relu',
batchSize=256,
dropoutRate=0.3,
epochs=200,
hiddenDimension=None,
learningRate=0.0002,
lossFunction='mse',
metrics=['mae', 'mse'],
testingData=None,
validationSteps=3,
regularizationRate=0,
):
inputDimension = len(trainingData[0])
inputData = layers.Input(shape=(inputDimension, ))
encoded = layers.Dropout(dropoutRate)(inputData)
if hiddenDimension is not None:
encoded = layers.Dense(
hiddenDimension,
activation=activation,
kernel_regularizer=regularizers.l2(regularizationRate),
)(encoded)
encoded = layers.Dense(
encodingDimension,
activation=activation,
kernel_regularizer=regularizers.l2(regularizationRate),
)(inputData)
decoded = encoded
if hiddenDimension is not None:
decoded = layers.Dense(hiddenDimension, activation=activation)(decoded)
decoded = layers.Dense(inputDimension, activation=activation)(decoded)
autoencoder = Model(inputData, decoded)
encodedInput = layers.Input(shape=(encodingDimension, ))
decoderLayer = (autoencoder.layers[-1](encodedInput)
if hiddenDimension is None else autoencoder.layers[-1](
autoencoder.layers[-2](encodedInput)))
encoder = Model(inputData, encoded)
decoder = Model(encodedInput, decoderLayer)
autoencoder.compile(
optimizer=optimizers.Nadam(learningRate),
loss=lossFunction,
metrics=metrics,
)
if trainingLabels is None:
trainingLabels = trainingData
if validationLabels is None:
validationLabels = validationData
autoencoder.fit(
trainingData,
trainingLabels,
batch_size=batchSize,
epochs=epochs,
shuffle=True,
validation_data=(validationData, validationLabels),
validation_steps=validationSteps,
)
return autoencoder, encoder, decoder
def build_model(local_bm_hyperparameters, local_bm_settings):
model_built = 0
time_steps_days = int(local_bm_hyperparameters['time_steps_days'])
epochs = int(local_bm_hyperparameters['epochs'])
batch_size = int(local_bm_hyperparameters['batch_size'])
workers = int(local_bm_hyperparameters['workers'])
optimizer_function = local_bm_hyperparameters['optimizer']
optimizer_learning_rate = local_bm_hyperparameters['learning_rate']
if optimizer_function == 'adam':
optimizer_function = optimizers.Adam(optimizer_learning_rate)
elif optimizer_function == 'ftrl':
optimizer_function = optimizers.Ftrl(optimizer_learning_rate)
losses_list = []
loss_1 = local_bm_hyperparameters['loss_1']
loss_2 = local_bm_hyperparameters['loss_2']
loss_3 = local_bm_hyperparameters['loss_3']
union_settings_losses = [loss_1, loss_2, loss_3]
if 'mape' in union_settings_losses:
losses_list.append(losses.MeanAbsolutePercentageError())
if 'mse' in union_settings_losses:
losses_list.append(losses.MeanSquaredError())
if 'mae' in union_settings_losses:
losses_list.append(losses.MeanAbsoluteError())
if 'm_mape' in union_settings_losses:
losses_list.append(modified_mape())
if 'customized_loss_function' in union_settings_losses:
losses_list.append(customized_loss())
metrics_list = []
metric1 = local_bm_hyperparameters['metrics1']
metric2 = local_bm_hyperparameters['metrics2']
union_settings_metrics = [metric1, metric2]
if 'rmse' in union_settings_metrics:
metrics_list.append(metrics.RootMeanSquaredError())
if 'mse' in union_settings_metrics:
metrics_list.append(metrics.MeanSquaredError())
if 'mae' in union_settings_metrics:
metrics_list.append(metrics.MeanAbsoluteError())
if 'mape' in union_settings_metrics:
metrics_list.append(metrics.MeanAbsolutePercentageError())
l1 = local_bm_hyperparameters['l1']
l2 = local_bm_hyperparameters['l2']
if local_bm_hyperparameters['regularizers_l1_l2'] == 'True':
activation_regularizer = regularizers.l1_l2(l1=l1, l2=l2)
else:
activation_regularizer = None
nof_features_for_training = local_bm_hyperparameters[
'nof_features_for_training']
# creating model
forecaster_in_block = tf.keras.Sequential()
print('creating the ANN model...')
# first layer (DENSE)
if local_bm_hyperparameters['units_layer_1'] > 0:
forecaster_in_block.add(
layers.Dense(
units=local_bm_hyperparameters['units_layer_1'],
activation=local_bm_hyperparameters['activation_1'],
input_shape=(local_bm_hyperparameters['time_steps_days'],
nof_features_for_training),
activity_regularizer=activation_regularizer))
forecaster_in_block.add(
layers.Dropout(
rate=float(local_bm_hyperparameters['dropout_layer_1'])))
# second LSTM layer
if local_bm_hyperparameters[
'units_layer_2'] > 0 and local_bm_hyperparameters[
'units_layer_1'] > 0:
forecaster_in_block.add(
layers.Bidirectional(
layers.LSTM(
units=local_bm_hyperparameters['units_layer_2'],
activation=local_bm_hyperparameters['activation_2'],
activity_regularizer=activation_regularizer,
dropout=float(local_bm_hyperparameters['dropout_layer_2']),
return_sequences=False)))
forecaster_in_block.add(
RepeatVector(local_bm_hyperparameters['repeat_vector']))
# third LSTM layer
if local_bm_hyperparameters['units_layer_3'] > 0:
forecaster_in_block.add(
layers.Bidirectional(
layers.LSTM(
units=local_bm_hyperparameters['units_layer_3'],
activation=local_bm_hyperparameters['activation_3'],
activity_regularizer=activation_regularizer,
dropout=float(local_bm_hyperparameters['dropout_layer_3']),
return_sequences=True)))
if local_bm_hyperparameters['units_layer_4'] == 0:
forecaster_in_block.add(
RepeatVector(local_bm_hyperparameters['repeat_vector']))
# fourth layer (DENSE)
if local_bm_hyperparameters['units_layer_4'] > 0:
forecaster_in_block.add(
layers.Dense(units=local_bm_hyperparameters['units_layer_4'],
activation=local_bm_hyperparameters['activation_4'],
activity_regularizer=activation_regularizer))
forecaster_in_block.add(
layers.Dropout(
rate=float(local_bm_hyperparameters['dropout_layer_4'])))
# final layer
forecaster_in_block.add(
TimeDistributed(layers.Dense(units=nof_features_for_training)))
forecaster_in_block.save(''.join(
[local_bm_settings['models_path'], 'in_block_NN_model_structure_']),
save_format='tf')
forecast_horizon_days = local_bm_settings['forecast_horizon_days']
forecaster_in_block.build(input_shape=(1, forecast_horizon_days + 1,
nof_features_for_training))
forecaster_in_block.compile(optimizer=optimizer_function,
loss=losses_list,
metrics=metrics_list)
forecaster_in_block_json = forecaster_in_block.to_json()
with open(
''.join([
local_bm_settings['models_path'],
'freq_acc_forecaster_in_block.json'
]), 'w') as json_file:
json_file.write(forecaster_in_block_json)
json_file.close()
print(
'build_model function finish (model structure saved in json and ts formats)'
)
return True, model_built
def _efficientnet(input_shape, blocks_args_list, global_params):
batch_norm_momentum = global_params.batch_norm_momentum
batch_norm_epsilon = global_params.batch_norm_epsilon
# Stem part
model_input = layers.Input(shape=input_shape)
x = layers.Conv2D(
filters=round_filters(32, global_params),
kernel_size=[3, 3],
strides=[2, 2],
kernel_initializer=conv_kernel_initializer,
padding='same',
use_bias=False,
name='stem_conv2d'
)(model_input)
x = layers.BatchNormalization(
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon,
name='stem_batch_norm'
)(x)
x = Swish(name='stem_swish')(x)
# Blocks part
idx = 0
drop_rate = global_params.drop_connect_rate
n_blocks = sum([blocks_args.num_repeat for blocks_args in blocks_args_list])
drop_rate_dx = drop_rate / n_blocks
for blocks_args in blocks_args_list:
assert blocks_args.num_repeat > 0
# Update block input and output filters based on depth multiplier.
blocks_args = blocks_args._replace(
input_filters=round_filters(blocks_args.input_filters, global_params),
output_filters=round_filters(blocks_args.output_filters, global_params),
num_repeat=round_repeats(blocks_args.num_repeat, global_params)
)
# The first block needs to take care of stride and filter size increase.
x = MBConvBlock(blocks_args, global_params, idx, drop_connect_rate=drop_rate_dx * idx)(x)
idx += 1
if blocks_args.num_repeat > 1:
blocks_args = blocks_args._replace(input_filters=blocks_args.output_filters, strides=[1, 1])
for _ in range(blocks_args.num_repeat - 1):
x = MBConvBlock(blocks_args, global_params, idx, drop_connect_rate=drop_rate_dx * idx)(x)
idx += 1
# Head part
x = layers.Conv2D(
filters=round_filters(1280, global_params),
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding='same',
use_bias=False,
name='head_conv2d'
)(x)
x = layers.BatchNormalization(
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon,
name='head_batch_norm'
)(x)
x = Swish(name='head_swish')(x)
x = layers.GlobalAveragePooling2D(name='global_average_pooling2d')(x)
if global_params.dropout_rate > 0:
x = layers.Dropout(global_params.dropout_rate)(x)
x = layers.Dense(
global_params.num_classes,
kernel_initializer=dense_kernel_initializer,
activation='softmax',
name='head_dense'
)(x)
model = models.Model(model_input, x)
return model
def __call__(self, inputs):
if self.dropout is not None and self.sdropout is not None:
raise ValueError("Can only use either dropout or spatial "
"dropout, not both")
dim_layers = _get_dimensional_layers(self.conv_dim)
convolution_nd = dim_layers["convolution"]
s_dropout_nd = dim_layers["s_dropout"]
if self.kernel_l2_reg is not None:
kernel_reg = ks.regularizers.l2(self.kernel_l2_reg)
else:
kernel_reg = None
if self.batchnorm:
use_bias = False
else:
use_bias = True
block_layers = list()
if isinstance(self.padding, str):
padding = self.padding
else:
block_layers.append(dim_layers["zero_padding"](self.padding))
padding = "valid"
block_layers.append(convolution_nd(
filters=self.filters,
kernel_size=self.kernel_size,
strides=self.strides,
padding=padding,
kernel_initializer=self.kernel_initializer,
use_bias=use_bias,
kernel_regularizer=kernel_reg,
dilation_rate=self.dilation_rate)
)
if self.batchnorm:
channel_axis = 1 if ks.backend.image_data_format() == "channels_first" else -1
block_layers.append(layers.BatchNormalization(axis=channel_axis))
if self.activation is not None:
block_layers.append(layers.Activation(self.activation))
if self.pool_type == "global_average_pooling":
pooling_nd = dim_layers[self.pool_type]
block_layers.append(pooling_nd())
elif self.pool_size is not None:
pooling_nd = dim_layers[self.pool_type]
block_layers.append(pooling_nd(
pool_size=self.pool_size, padding=self.pool_padding))
if self.dropout is not None:
block_layers.append(layers.Dropout(self.dropout))
elif self.sdropout is not None:
block_layers.append(s_dropout_nd(self.sdropout))
x = inputs
for block_layer in block_layers:
if self.time_distributed:
x = layers.TimeDistributed(block_layer)(x)
else:
x = block_layer(x)
return x
## Build a model
We will build a convolutional reconstruction autoencoder model. The model will
take input of shape `(batch_size, sequence_length, num_features)` and return
output of the same shape. In this case, `sequence_length` is 288 and
`num_features` is 1.
"""
model = keras.Sequential([
layers.Input(shape=(x_train.shape[1], x_train.shape[2])),
layers.Conv1D(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1D(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1DTranspose(filters=32,
kernel_size=7,
padding="same",
strides=2,
# coding: utf-8
# In[ ]:
import matplotlib.pyplot as plt
import gym
import numpy as np
from tensorflow.keras import models, layers, optimizers
env = gym.make('CartPole-v0')
STATE_DIM, ACTION_DIM = 4, 2
model = models.Sequential([
layers.Dense(100, input_dim=STATE_DIM, activation='relu'),
layers.Dropout(0.1),
layers.Dense(ACTION_DIM, activation="softmax")
])
model.compile(loss='mean_squared_error',
optimizer=optimizers.Adam(0.001))
def choose_action(s):
"""预测动作"""
prob = model.predict(np.array([s]))[0]
return np.random.choice(len(prob), p=prob)
def discount_rewards(rewards, gamma=0.95):
"""计算衰减reward的累加期望，并中心化和标准化处理"""
prior = 0
x = layers.Conv2D(32, 3, activation="relu")(inputs) x = layers.Conv2D(64, 3, activation="relu")(x) block_1_output = layers.MaxPooling2D(3)(x) x = layers.Conv2D(64, 3, activation="relu", padding="same")(block_1_output) x = layers.Conv2D(64, 3, activation="relu", padding="same")(x) block_2_output = layers.add([x, block_1_output]) x = layers.Conv2D(64, 3, activation="relu", padding="same")(block_2_output) x = layers.Conv2D(64, 3, activation="relu", padding="same")(x) block_3_output = layers.add([x, block_2_output]) x = layers.Conv2D(64, 3, activation="relu")(block_3_output) x = layers.GlobalAveragePooling2D()(x) x = layers.Dense(256, activation="relu")(x) x = layers.Dropout(0.5)(x) outputs = layers.Dense(10)(x) model = keras.Model(inputs, outputs, name="toy_resnet") model.summary() """ Plot the model: """ keras.utils.plot_model(model, "mini_resnet.png", show_shapes=True) """ Now train the model: """
def Generator():
# Encoder:
input = tf.keras.Input(shape=[L_node, W_node, Channel])
# input: (None, 256, 256, 1)
e1 = layers.Conv2D(64, 5, 2, 'same')(input)
# e1: (None, 128, 128, 64)
e2 = layers.LeakyReLU(0.2)(e1)
# e2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e2)
e2_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e2)
e2_1 = layers.BatchNormalization()(e2_1)
e2_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e2)
e2_2 = layers.BatchNormalization()(e2_2)
e2_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e2)
e2_3 = layers.BatchNormalization()(e2_3)
e2 = layers.concatenate([e2_1, e2_2, e2_3], 3)
e2 = layers.Conv2D(128, 5, 2, 'same')(e2) # Downsampling
e2 = layers.BatchNormalization()(e2)
# e2: (None, 64, 64, 128)
e3 = layers.LeakyReLU(0.2)(e2)
# e3 = layers.Conv2D(128, 3, 1, 'same', activation='relu')(e3)
e3_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e3)
e3_1 = layers.BatchNormalization()(e3_1)
e3_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e3)
e3_2 = layers.BatchNormalization()(e3_2)
e3_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e3)
e3_3 = layers.BatchNormalization()(e3_3)
e3 = layers.concatenate([e3_1, e3_2, e3_3], 3)
e3 = layers.Conv2D(256, 5, 2, 'same')(e3) # Downsampling
e3 = layers.BatchNormalization()(e3)
# e3: (None, 32, 32, 256)
e4 = layers.LeakyReLU(0.2)(e3)
# e4 = layers.Conv2D(256, 3, 1, 'same', activation='relu')(e4)
e4_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e4)
e4_1 = layers.BatchNormalization()(e4_1)
e4_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e4)
e4_2 = layers.BatchNormalization()(e4_2)
e4_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e4)
e4_3 = layers.BatchNormalization()(e4_3)
e4 = layers.concatenate([e4_1, e4_2, e4_3], 3)
e4 = layers.Conv2D(512, 5, 2, 'same')(e4) # Downsampling
e4 = layers.BatchNormalization()(e4)
# e4: (None, 16, 16, 512)
e5 = layers.LeakyReLU(0.2)(e4)
# e5 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e5)
e5_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e5)
e5_1 = layers.BatchNormalization()(e5_1)
e5_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e5)
e5_2 = layers.BatchNormalization()(e5_2)
e5_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e5)
e5_3 = layers.BatchNormalization()(e5_3)
e5 = layers.concatenate([e5_1, e5_2, e5_3], 3)
e5 = layers.Conv2D(512, 5, 2, 'same')(e5) # Downsampling
e5 = layers.BatchNormalization()(e5)
# e5: (None, 8, 8, 512)
e6 = layers.LeakyReLU(0.2)(e5)
# e6 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e6)
e6_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e6)
e6_1 = layers.BatchNormalization()(e6_1)
e6_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e6)
e6_2 = layers.BatchNormalization()(e6_2)
e6_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e6)
e6_3 = layers.BatchNormalization()(e6_3)
e6 = layers.concatenate([e6_1, e6_2, e6_3], 3)
e6 = layers.Conv2D(512, 5, 2, 'same')(e6) # Dowmsampling
e6 = layers.BatchNormalization()(e6)
# e6: (None, 4, 4, 512)
e7 = layers.LeakyReLU(0.2)(e6)
# e7 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e7)
e7_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e7)
e7_1 = layers.BatchNormalization()(e7_1)
e7_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e7)
e7_2 = layers.BatchNormalization()(e7_2)
e7_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e7)
e7_3 = layers.BatchNormalization()(e7_3)
e7 = layers.concatenate([e7_1, e7_2, e7_3], 3)
e7 = layers.Conv2D(512, 5, 2, 'same')(e7) # Downsampling
e7 = layers.BatchNormalization()(e7)
# e7: (None, 2, 2, 512)
e8 = layers.LeakyReLU(0.2)(e7)
# e8 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e8)
e8_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e8)
e8_1 = layers.BatchNormalization()(e8_1)
e8_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e8)
e8_2 = layers.BatchNormalization()(e8_2)
e8_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e8)
e8_3 = layers.BatchNormalization()(e8_3)
e8 = layers.concatenate([e8_1, e8_2, e8_3], 3)
e8 = layers.Conv2D(512, 5, 2, 'same')(e8) # Downsampling
e8 = layers.BatchNormalization()(e8)
# e8: (None, 1, 1, 512)
# Decoder:
d1 = layers.Activation('relu')(e8)
# d1 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d1)
d1_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d1)
d1_1 = layers.BatchNormalization()(d1_1)
d1_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d1)
d1_2 = layers.BatchNormalization()(d1_2)
d1_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d1)
d1_3 = layers.BatchNormalization()(d1_3)
d1 = layers.concatenate([d1_1, d1_2, d1_3], 3)
d1 = layers.Conv2DTranspose(512, 5, 2, 'same')(d1) # Upsampling
d1 = layers.BatchNormalization()(d1)
d1 = layers.Dropout(0.5)(d1)
d1 = layers.concatenate([d1, e7], 3)
# d1: (None, 2, 2, 512*2)
d2 = layers.Activation('relu')(d1)
# d2 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d2)
d2_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d2)
d2_1 = layers.BatchNormalization()(d2_1)
d2_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d2)
d2_2 = layers.BatchNormalization()(d2_2)
d2_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d2)
d2_3 = layers.BatchNormalization()(d2_3)
d2 = layers.concatenate([d2_1, d2_2, d2_3], 3)
d2 = layers.Conv2DTranspose(512, 5, 2, 'same')(d2) # Upsampling
d2 = layers.BatchNormalization()(d2)
d2 = layers.Dropout(0.5)(d2)
d2 = layers.concatenate([d2, e6], 3)
# d2: (None, 4, 4, 512*2)
d3 = layers.Activation('relu')(d2)
# d3 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d3)
d3_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d3)
d3_1 = layers.BatchNormalization()(d3_1)
d3_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d3)
d3_2 = layers.BatchNormalization()(d3_2)
d3_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d3)
d3_3 = layers.BatchNormalization()(d3_3)
d3 = layers.concatenate([d3_1, d3_2, d3_3], 3)
d3 = layers.Conv2DTranspose(512, 5, 2, 'same')(d3) # Upsampling
d3 = layers.BatchNormalization()(d3)
d3 = layers.Dropout(0.5)(d3)
d3 = layers.concatenate([d3, e5], 3)
# d3: (None, 8, 8, 512*2)
d4 = layers.Activation('relu')(d3)
# d4 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d4)
d4_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d4)
d4_1 = layers.BatchNormalization()(d4_1)
d4_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d4)
d4_2 = layers.BatchNormalization()(d4_2)
d4_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d4)
d4_3 = layers.BatchNormalization()(d4_3)
d4 = layers.concatenate([d4_1, d4_2, d4_3], 3)
d4 = layers.Conv2DTranspose(512, 5, 2, 'same')(d4) # Upsampling
d4 = layers.BatchNormalization()(d4)
d4 = layers.Dropout(0.5)(d4)
d4 = layers.concatenate([d4, e4], 3)
# d4: (None, 16, 16, 512*2)
d5 = layers.Activation('relu')(d4)
# d5 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d5)
d5_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d5)
d5_1 = layers.BatchNormalization()(d5_1)
d5_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d5)
d5_2 = layers.BatchNormalization()(d5_2)
d5_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d5)
d5_3 = layers.BatchNormalization()(d5_3)
d5 = layers.concatenate([d5_1, d5_2, d5_3], 3)
d5 = layers.Conv2DTranspose(256, 5, 2, 'same')(d5) # Upsampling
d5 = layers.BatchNormalization()(d5)
d5 = layers.Dropout(0.5)(d5)
d5 = layers.concatenate([d5, e3], 3)
# d5: (None, 32, 32, 256*2)
d6 = layers.Activation('relu')(d5)
# d6 = layers.Conv2D(256, 3, 1, 'same', activation='relu')(d6)
d6_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d6)
d6_1 = layers.BatchNormalization()(d6_1)
d6_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d6)
d6_2 = layers.BatchNormalization()(d6_2)
d6_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d6)
d6_3 = layers.BatchNormalization()(d6_3)
d6 = layers.concatenate([d6_1, d6_2, d6_3], 3)
d6 = layers.Conv2DTranspose(128, 5, 2, 'same')(d6) # Upsampling
d6 = layers.BatchNormalization()(d6)
d6 = layers.Dropout(0.5)(d6)
d6 = layers.concatenate([d6, e2], 3)
# d6: (None, 64, 64, 128*2)
d7 = layers.Activation('relu')(d6)
# d7 = layers.Conv2D(128, 3, 1, 'same', activation='relu')(d7)
d7_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d7)
d7_1 = layers.BatchNormalization()(d7_1)
d7_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d7)
d7_2 = layers.BatchNormalization()(d7_2)
d7_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d7)
d7_3 = layers.BatchNormalization()(d7_3)
d7 = layers.concatenate([d7_1, d7_2, d7_3], 3)
d7 = layers.Conv2DTranspose(64, 5, 2, 'same')(d7) # Upsampling
d7 = layers.BatchNormalization()(d7)
d7 = layers.Dropout(0.5)(d7)
d7 = layers.concatenate([d7, e1], 3)
# d7: (None, 128, 128, 64*2)
d8 = layers.Activation('relu')(d7)
# d8 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d8)
d8_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d8)
d8_1 = layers.BatchNormalization()(d8_1)
d8_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d8)
d8_2 = layers.BatchNormalization()(d8_2)
d8_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d8)
d8_3 = layers.BatchNormalization()(d8_3)
d8 = layers.concatenate([d8_1, d8_2, d8_3], 3)
d8 = layers.Conv2DTranspose(1, 5, 2, 'same')(d8) # Upsampling
d8 = layers.Activation('tanh')(d8)
# d8: (None, 256, 256, 1)
output = d8
model = tf.keras.Model(input, output)
return model
"""
model = models.Sequential()
#64 filtre, relu aktivasyon fonksiyonu, 3x3'lük filtreler. Padding yok ve stride = 1
model.add(
layers.Conv2D(64, (3, 3), activation='relu', input_shape=(200, 200, 3)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(512, activation='relu'))
#overfittingin önlenmesi için
model.add(layers.Dropout(0.2))
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dropout(0.2))
#Çıkışın alındığı yer. 10 sınıf olduğu için 10 adet nöron
model.add(layers.Dense(10, activation='sigmoid'))
#Bu fonksiyon modeling özetini çıkartır
model.summary()
#loss ve optimizasyon fonksiyonlari model kurulur.
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#model burada eğitilir. Eğitim setinin %10'nunu validation için ayırdıö
loss = model.fit(x_train,
early_stop = EarlyStopping('val_loss', patience=30)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=10, verbose=1)
callbacks = [early_stop, reduce_lr]
##################################################
base_model = Xception(input_shape=(299, 299, 3),
include_top=False,
weights='imagenet')
base_model.trainable = False
inputs = Input(shape=(299, 299, 3))
x = base_model(inputs, training=False)
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dropout(0.2)(x)
outputs = layers.Dense(150, activation='softmax')(x)
model = Model(inputs, outputs)
model.summary()
# Train
train_datagen = ImageDataGenerator(
# featurewise_center=True,
# featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
rescale=1. / 255,
zoom_range=0.2,
def __init__(self,
channels,
init_block_channels,
stem1_blocks_channels,
in_channels=3,
in_size=(331, 331),
classes=1000,
data_format="channels_last",
**kwargs):
super(PNASNet, self).__init__(**kwargs)
self.in_size = in_size
self.classes = classes
self.data_format = data_format
self.features = nasnet_dual_path_sequential(return_two=False,
first_ordinals=2,
last_ordinals=2,
name="features")
self.features.add(
NASNetInitBlock(in_channels=in_channels,
out_channels=init_block_channels,
data_format=data_format,
name="init_block"))
in_channels = init_block_channels
self.features.add(
Stem1Unit(in_channels=in_channels,
out_channels=stem1_blocks_channels,
data_format=data_format,
name="stem1_unit"))
prev_in_channels = in_channels
in_channels = stem1_blocks_channels
for i, channels_per_stage in enumerate(channels):
stage = nasnet_dual_path_sequential(name="stage{}".format(i + 1))
for j, out_channels in enumerate(channels_per_stage):
reduction = (j == 0)
extra_padding = (j == 0) and (i not in [0, 2])
match_prev_layer_dimensions = (j == 1) or ((j == 0) and
(i == 0))
stage.add(
PnasUnit(
in_channels=in_channels,
prev_in_channels=prev_in_channels,
out_channels=out_channels,
reduction=reduction,
extra_padding=extra_padding,
match_prev_layer_dimensions=match_prev_layer_dimensions,
data_format=data_format,
name="unit{}".format(j + 1)))
prev_in_channels = in_channels
in_channels = out_channels
self.features.add(stage)
self.features.add(nn.ReLU(name="activ"))
self.features.add(
nn.AveragePooling2D(pool_size=11,
strides=1,
data_format=data_format,
name="final_pool"))
self.output1 = tf.keras.Sequential(name="output1")
self.output1.add(nn.Dropout(rate=0.5, name="dropout"))
self.output1.add(
nn.Dense(units=classes, input_dim=in_channels, name="fc"))
(['shots_attempted_total'], [SimpleImputer(),
StandardScaler()])],
df_out=False)
Z_train = mapper.fit_transform(X_train)
Z_test = mapper.transform(X_test)
# build keras model for goals
from tensorflow.keras import layers
from tensorflow.keras import Input
from tensorflow.keras import models
from tensorflow.keras.models import Model
model = models.Sequential()
model.add(layers.Dense(30, activation='selu', input_dim=Z_train.shape[1]))
model.add(layers.Dense(10, activation='selu'))
model.add(layers.Dropout(0.25))
model.add(layers.Dense(5, activation='selu'))
model.add(layers.Dense(1))
model.compile(optimizer='Nadam', loss='mse', metrics=['mae'])
model.fit(Z_train, y_train_goals, epochs=100, batch_size=10)
import pydot
from tensorflow.keras.utils import plot_model
plot_model(model)
y_hat_test_goals = model.predict(Z_test)
r2_score(y_test_goals, y_hat_test_goals)
mean_absolute_error(y_test_goals, y_hat_test_goals)
plt.figure(figsize=(8, 8))
plt.scatter(y_test_goals, y_hat_test_goals, alpha=1 / 4)
def dense_block(units, dropout_rate, inputs):
x = layers.Dense(units, activation="relu")(inputs)
x = layers.BatchNormalization()(x)
outputs = layers.Dropout(dropout_rate)(x)
return outputs
strides=(2, 1))(outputs)
if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs)
outputs = layers.Activation(tf.nn.relu)(outputs)
outputs = layers.Conv2D(NUM_CHANNELS,
KERNEL_SIZE,
padding="same",
strides=(2, 2))(outputs)
if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs)
outputs = layers.Activation(tf.nn.relu)(outputs)
outputs = layers.Flatten()(outputs)
outputs_flat = layers.Flatten()(inputs)
outputs_flat = layers.Dense(1512)(outputs_flat)
if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs)
outputs = layers.Activation(tf.nn.relu)(outputs)
outputs_flat = layers.Dropout(DROPOUT_RATE)(outputs_flat)
outputs_flat = layers.Dense(1256)(outputs_flat)
if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs)
outputs = layers.Activation(tf.nn.relu)(outputs)
outputs_flat = layers.Dropout(DROPOUT_RATE)(outputs_flat)
outputs = layers.Concatenate(axis=1)([outputs_flat, outputs])
outputs = layers.Dense(1024)(outputs)
outputs = layers.Dropout(DROPOUT_RATE)(outputs)
if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs)
outputs = layers.Activation(tf.nn.relu)(outputs)
outputs = layers.Dense(512)(outputs)
outputs = layers.Dropout(DROPOUT_RATE)(outputs)
if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs)
outputs = layers.Activation(tf.nn.relu)(outputs)
pi = layers.Dense(pi_output_count, name="policy")(outputs)
dimension = 128
vocabulary_size = len(tk.word_index)
inputs = layers.Input(shape=(input_size, ))
embedding_layer = TokenAndPositionEmbedding(input_size, vocabulary_size + 1,
embed_dim)
X = embedding_layer(inputs)
transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
#transformer_block2 = TransformerBlock(embed_dim, num_heads, ff_dim)
X = transformer_block(X)
#X = transformer_block2(X)
X = layers.GlobalAvgPool1D()(X)
#X = Flatten()(X)
X = layers.Dense(128, activation='relu')(X)
X = layers.Dropout(0.5)(X)
X = layers.Dense(64, activation='relu')(X)
X = layers.Dropout(0.5)(X)
X = layers.Dense(32, activation='relu')(X)
X = layers.Dropout(0.5)(X)
outputs = layers.Dense(1)(X)
model = Model(inputs=inputs, outputs=outputs)
model.compile(optimizer='adam', loss='mse',
metrics=['mse']) # Adam, categorical_crossentropy
model.summary()
# log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
#
# model.fit(np_data, y_data, epochs=10, batch_size= 64, validation_split=0.3, callbacks=[tensorboard_callback])
def build_model(self, phenotype):
model = models.Sequential()
filter_size = 32
nconvs = 0
optimizer = None
model.add(
layers.InputLayer(input_shape=(28, 28,
params['DATASET_NUM_SHAPE'])))
nblocks = int(phenotype[0])
for n in range(nblocks):
for block in phenotype.split(','):
if 'Conv' in block:
if nconvs == 2:
filter_size *= 2
nconvs = 0
model.add(
layers.Conv2D(filter_size, (3, 3),
activation='relu',
padding='same'))
if 'BNorm' in block:
model.add(layers.BatchNormalization())
nconvs += 1
if 'MaxPool' in block:
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
if 'Dropout' in block:
model.add(layers.Dropout(0.25))
for block in phenotype.split(','):
if 'Flatten' in block:
model.add(layers.Flatten())
if 'Fc' in block:
nfc, neurons = re.findall('\d+', block)
for n in range(int(nfc)):
model.add(layers.Dense(int(neurons)))
model.add(layers.Activation('relu'))
if 'Dropout' in block:
model.add(layers.Dropout(0.5))
if 'Lr' in block:
args = re.findall('\d+\.\d+', block)
optimizer = optimizers.Adam(learning_rate=float(args[0]))
model.add(
layers.Dense(params['DATASET_NUM_CLASSES'], activation='softmax'))
model.summary()
# F1 Score metric function
def f1_score(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
recall = true_positives / (possible_positives + K.epsilon())
f1_val = 2 * (precision * recall) / (precision + recall +
K.epsilon())
return f1_val
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy', f1_score])
return model
def _create_keras_temp_model(self):
# Each input sample consists of a bag of x`MAX_CONTEXTS` tuples (source_terminal, path, target_terminal).
# The valid mask indicates for each context whether it actually exists or it is just a padding.
path_source_token_input = keras.Input((self.config.MAX_CONTEXTS,), dtype=tf.int32)
path_input = keras.Input((self.config.MAX_CONTEXTS,), dtype=tf.int32)
path_target_token_input = keras.Input((self.config.MAX_CONTEXTS,), dtype=tf.int32)
context_valid_mask = keras.Input((self.config.MAX_CONTEXTS,))
# Input paths are indexes, we embed these here.
paths_embedded = layers.Embedding(
self.vocabs.path_vocab.size, self.config.PATH_EMBEDDINGS_SIZE, name='path_embedding')(path_input)
# Input terminals are indexes, we embed these here.
token_embedding_shared_layer = layers.Embedding(
self.vocabs.token_vocab.size, self.config.TOKEN_EMBEDDINGS_SIZE, name='token_embedding')
path_source_token_embedded = token_embedding_shared_layer(path_source_token_input)
path_target_token_embedded = token_embedding_shared_layer(path_target_token_input)
# `Context` is a concatenation of the 2 terminals & path embedding.
# Each context is a vector of size 3 * EMBEDDINGS_SIZE.
context_embedded = layers.Concatenate()([path_source_token_embedded, paths_embedded, path_target_token_embedded])
context_embedded = layers.Dropout(1 - self.config.DROPOUT_KEEP_RATE)(context_embedded)
# Lets get dense: Apply a dense layer for each context vector (using same weights for all of the context).
context_after_dense = layers.TimeDistributed(
Dense(self.config.CODE_VECTOR_SIZE, use_bias=False, activation='tanh'))(context_embedded)
# The final code vectors are received by applying attention to the "densed" context vectors.
code_vectors, attention_weights = AttentionLayer(name='attention')(
[context_after_dense, context_valid_mask])
# "Decode": Now we use another dense layer to get the target word embedding from each code vector.
target_index = layers.Dense(
self.vocabs.target_vocab.size, use_bias=False, activation='softmax', name='target_index')(code_vectors)
print(target_index)
print('PPPP'*80)
# Wrap the layers into a Keras model, using our subtoken-metrics and the CE loss.
inputs = [path_source_token_input, path_input, path_target_token_input, context_valid_mask]
self.keras_train_model = keras.Model(inputs=inputs, outputs=target_index)
# Actual target word predictions (as strings). Used as a second output layer.
# Used for predict() and for the evaluation metrics calculations.
topk_predicted_words, topk_predicted_words_scores = TopKWordPredictionsLayer(
self.config.TOP_K_WORDS_CONSIDERED_DURING_PREDICTION,
self.vocabs.target_vocab.get_index_to_word_lookup_table(),
name='target_string')(target_index)
# topk_predicted_words, topk_predicted_words_scores = TopKWordPredictionsLayer(
# self.config.TOP_K_WORDS_CONSIDERED_DURING_PREDICTION,
# self.vocabs.target_vocab.get_index_to_word_lookup_table(),
# name='target_string')(target_index)
# We use another dedicated Keras model for evaluation.
# The evaluation model outputs the `topk_predicted_words` as a 2nd output.
# The separation between train and eval models is for efficiency.
#
self.keras_eval_model = keras.Model(
inputs=inputs, outputs=[target_index, topk_predicted_words], name="code2vec-keras-model")
# model_new = load_model('keke.h5', custom_objects={'AttentionLayer' : AttentionLayer})
#
# # a = model_new.get_layer(-1)
# # a = model_new.get_layer('target_index')#.get_weights()(code_vectors)
# # a = getattr(model_new, 'input_1')
# # a = model_new.__getitem__
# print(tf.keras.backend.get_value(model_new.layers[9]))
# # a = tf.keras.backend.get_value(model_new.output)
# # print(model_new['input_1'])
# print('-'*80)
# self.keras_eval_model = keras.Model(
# inputs=inputs, outputs=[a, topk_predicted_words], name="code2vec-keras-model")
# self.keras_eval_model = model_new
# self.keras_eval_model.summary()
# We use another dedicated Keras function to produce predictions.
# It have additional outputs than the original model.
# It is based on the trained layers of the original model and uses their weights.
predict_outputs = tuple(KerasPredictionModelOutput(
target_index=target_index, code_vectors=code_vectors, attention_weights=attention_weights,
topk_predicted_words=topk_predicted_words, topk_predicted_words_scores=topk_predicted_words_scores))
self.keras_model_predict_function = K.function(inputs=inputs, outputs=predict_outputs)
history = model.fit(X_train, y_train, epochs=20, verbose=1)
# 绘制不同层数的网络决策边界曲线
preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()])
title = "网络层数({})".format(n)
make_plot(X_train, y_train, title, XX, YY, preds)
# Dropout的影响
for n in range(5): # 构建5种不同数量Dropout层的网络
model = Sequential()
model.add(layers.Dense(8, input_dim=2, activation='relu'))
counter = 0
for _ in range(n):
model.add(layers.Dense(64, activation='relu'))
if counter < n: # 添加n个Dropout层
counter += 1
model.add(layers.Dropout(rate=0.5))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(X_train, y_train, epochs=20, verbose=1)
preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()])
title = "Dropout({})".format(n)
make_plot(X_train, y_train, title, XX, YY, preds)
def build_model_with_regularizeation(_lambda):
# 创建带正则化项的神经网络
model = Sequential()
model.add(layers.Dense(8, input_dim=2, activation='relu'))
model.add(
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = keras.Sequential(
[
keras.Input(shape=input_shape),
layers.Conv2D(32, kernel_size=(3, 3), activation="relu"),
layers.MaxPooling2D(pool_size=(2, 2)),
layers.Conv2D(64, kernel_size=(3, 3), activation="relu"),
layers.MaxPooling2D(pool_size=(2, 2)),
layers.Flatten(),
layers.Dropout(0.5),
layers.Dense(num_classes, activation="softmax"),
]
)
model.summary()
batch_size = 128
epochs = 15
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
score = model.evaluate(x_test, y_test, verbose=0)
def create_model(model_type='state_estimator',
model_opt='best_noise_opt'):
'''
inputs:
model_type: str specifying either 'state_estimator' or
'quality_control' type machine learning model.
model_opt: str specifying dataset the model parameters were optimized
on. Valid options for 'state_estimator' model_type:
'noiseless_opt' or 'best_noise_opt'. Valid options for
'quality_control' type: 'uniform_noise_dist_opt'.
'''
valid_model_types = ['state_estimator','quality_control']
if model_type not in valid_model_types:
raise ValueError(
'model_type not recognized: ', model_type,
' Valid values: ', valid_model_types)
valid_model_opts = {
'state_estimator': ['noiseless_opt', 'best_noise_opt'],
'quality_control': ['uniform_noise_dist_opt']}
if model_opt not in valid_model_opts[model_type]:
raise ValueError(
'model_opt not recognized: ', model_opt,
' Valid values: ', valid_model_opts[model_type])
if model_type=='state_estimator' and model_opt=='best_noise_opt':
lr = 1.21e-3
k_size = [[7, 7], [7, 7]]
cnn_maxpool = False
cnn_stack = 2
n_cnn = 2
# these lists should be length n_cnn
n_filters = [[22, 22], [35, 35]]
drop_rates = [[0.655,0.655], [0.194, 0.194]]
layer_norm = False
ave_pool = True
activation='relu'
dense_n = 0
elif model_type=='state_estimator' and model_opt == 'noiseless_opt':
lr = 3.45e-3
k_size = [[5], [5], [5]]
cnn_maxpool=False
cnn_stack = 1
n_cnn = 3
n_filters = [[23], [7], [18]]
drop_rates = [[0.12], [0.28], [0.30]]
layer_norm = True
ave_pool = True
activation = 'relu'
dense_n = 0
elif model_type=='quality_control' and model_opt=='uniform_noise_dist_opt':
lr = 2.65e-4
k_size = [[7, 3]]
cnn_maxpool = True
cnn_stack = 2
n_cnn = 1
n_filters = [[184, 249]]
drop_rates = [[0.05, 0.0]]
layer_norm = True
ave_pool = True
activation='swish'
dense_n = 1
dense_dropout = [0.6]
dense_units = [161]
# set stride to 2 if not using maxpool as size reduction
if cnn_maxpool:
cnn_stride=1
else:
cnn_stride=2
# input layer
inputs = tf_layers.Input(shape=(config.SUB_SIZE,config.SUB_SIZE,1))
x = inputs
for i in range(n_cnn):
for j in range(cnn_stack):
if j==cnn_stack-1:
stride = cnn_stride
else:
stride=1
x = tf_layers.Conv2D(
filters=n_filters[i][j],
kernel_size=k_size[i][j],
padding='same',
strides=stride)(x)
x = tf_layers.Dropout(rate=drop_rates[i][j])(x)
if layer_norm:
x = tf_layers.LayerNormalization()(x)
x = tf_layers.Activation(activation)(x)
if cnn_maxpool:
x = tf_layers.MaxPooling2D(pool_size=(2,2), strides=2)(x)
if ave_pool:
x = tf_layers.GlobalAvgPool2D()(x)
x = tf_layers.Flatten()(x)
for i in range(dense_n):
x = tf_layers.Dense(units=dense_units[i],activation=activation)(x)
x = tf_layers.Dropout(rate=dense_dropout[i])(x)
if model_type=='state_estimator':
outputs = tf_layers.Dense(
units=config.NUM_STATES, activation='softmax')(x)
model = tf_Model(inputs, outputs,
name='device_state_estimator_'+model_opt)
model.compile(
optimizer=tf_Adam(learning_rate=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
elif model_type=='quality_control':
outputs = tf_layers.Dense(
units=config.NUM_QUALITY_CLASSES, activation='softmax')(x)
model = tf_Model(
inputs=inputs, outputs=outputs,
name='data_quality_control_'+model_opt)
model.compile(
optimizer=tf_Adam(learning_rate=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
for spectrogram, _ in spectrogram_ds.take(1):
input_shape = spectrogram.shape
print('Input shape:', input_shape)
num_labels = len(commands)
norm_layer = preprocessing.Normalization()
norm_layer.adapt(spectrogram_ds.map(lambda x, _: x))
model = models.Sequential([
layers.Input(shape=input_shape),
#preprocessing.Resizing(32, 32),
norm_layer,
layers.Conv2D(32, 3, activation='relu'),
layers.Conv2D(64, 3, activation='relu'),
layers.MaxPooling2D(),
layers.Dropout(0.25),
layers.Flatten(),
layers.Dense(128, activation='relu'),
layers.Dropout(0.5),
layers.Dense(num_labels),
])
model.summary()
model.compile(
optimizer=tf.keras.optimizers.Adam(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'],
)
EPOCHS = 50
def build_model(self, phenotype):
# To free memory on google colab.
if K.backend() == 'tensorflow':
K.clear_session()
nconv, npool, nfc, nfcneuron = [int(i) for i in re.findall('\d+', phenotype.split('lr-')[0])]
has_dropout = 'dropout' in phenotype
has_batch_normalization = 'bnorm' in phenotype
has_pool = 'pool' in phenotype
learning_rate = float(phenotype.split('lr-')[1])
# number of filters
filter_size = 32
model = models.Sequential()
try:
# Pooling
for i in range(npool):
# Convolutions
for j in range(nconv):
model.add(layers.Conv2D(filter_size, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3)))
# Duplicate number of filters for each two convolutions
if (((i + j) % 2) == 1): filter_size = filter_size * 2
# Add batch normalization
if has_batch_normalization:
model.add(layers.BatchNormalization())
# Add pooling
if has_pool:
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
# Add dropout
if has_dropout:
model.add(layers.Dropout(0.25))
model.add(layers.Flatten())
# fully connected
for i in range(nfc):
model.add(layers.Dense(nfcneuron))
model.add(layers.Activation('relu'))
if has_dropout:
model.add(layers.Dropout(0.5))
model.add(layers.Dense(10, activation='softmax'))
# model.summary()
except Exception as ex:
# Some NN topologies are invalid
print(ex)
return None
opt = optimizers.Adam(lr=learning_rate)
# F1 Score metric function
def f1_score(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
recall = true_positives / (possible_positives + K.epsilon())
f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon())
return f1_val
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', f1_score])
return model
