def pixelSNAIL(attention=True, out_channels=None, num_pixel_blocks=1, num_grb_per_pixel_block=1, dropout=0.0,
nr_filters=128):
nr_logistic_mix = 10
kernel_size = 3
x_in = Input(shape=(32, 32, 3))
k_d = [kernel_size - 1, kernel_size]
k_dr = [kernel_size - 1, kernel_size - 1]
u = Shift("down")(CausalConv2D(nr_filters, [kernel_size - 1, kernel_size], shift="down")(x_in))
ul = Shift("down")(CausalConv2D(nr_filters, [1, kernel_size], shift="down")(x_in))
ul += Shift("right")(CausalConv2D(nr_filters, [kernel_size - 1, 1], shift="downright")(x_in))
for i in range(num_pixel_blocks):
for j in range(num_grb_per_pixel_block):
u = GatedResidualBlock(x=u, aux=None,
nonlinearity="elu",
dropout=dropout,
conv1=CausalConv2D(filters=nr_filters, kernel_size=k_d, shift="down",
activation="elu", name="causalconv_u_1_{}_{}".format(i, j)),
conv2=CausalConv2D(filters=2 * nr_filters, kernel_size=k_d, shift="down",
activation="elu", name="causalconv_u_2_{}_{}".format(i, j)))
ul = GatedResidualBlock(x=ul, aux=u,
nonlinearity="elu",
dropout=dropout,
conv1=CausalConv2D(filters=nr_filters, kernel_size=k_dr, shift="downright",
activation="elu", name="causalconv_ul_1_{}_{}".format(i, j)),
conv2=CausalConv2D(filters=2 * nr_filters, kernel_size=k_dr, shift="downright",
activation="elu", name="causalconv_ul_2_{}_{}".format(i, j)))
if attention:
content = Concatenate(axis=3)([x_in, ul])
content = tf.debugging.check_numerics(content, "bad conent")
channels = content.shape[-1]
kv = GatedResidualBlock(x=content, aux=None,
nonlinearity="elu",
dropout=dropout,
conv1=NetworkInNetwork(filters=channels, activation=None),
conv2=NetworkInNetwork(filters=2 * channels, activation=None))
kv = NetworkInNetwork(filters=2 * nr_filters, activation=None)(kv)
key, value = tf.split(kv, 2, axis=3)
query = GatedResidualBlock(x=ul, aux=None,
nonlinearity="elu",
dropout=dropout,
conv1=NetworkInNetwork(filters=nr_filters, activation=None),
conv2=NetworkInNetwork(filters=2 * nr_filters, activation=None))
query = NetworkInNetwork(filters=nr_filters, activation=None)(query)
a = CausalAttention()([key, query, value])
a = tf.debugging.check_numerics(a, "bad a!!")
else:
a = None
ul = GatedResidualBlock(x=ul, aux=a,
nonlinearity="elu",
dropout=dropout,
conv1=NetworkInNetwork(filters=nr_filters, activation=None),
conv2=NetworkInNetwork(filters=2 * nr_filters, activation=None))
ul = Activation("elu")(ul)
if out_channels is not None:
filters = out_channels
x_out = NetworkInNetwork(filters=filters, activation=None)(ul)
else:
filters = 10 * nr_logistic_mix
x_out = NetworkInNetwork(filters=filters, activation=None)(ul)
model = tf.keras.Model(inputs=x_in, outputs=x_out)
return model
NAME = "Cats-vs-Dogs-64x2-{}".format(int(time.time()))
tensorboard = TensorBoard(log_dir='logs/{}'.format(NAME))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
x = pickle.load(open("x.pickle", "rb"))
y = pickle.load(open("y.pickle", "rb"))
x = x / 255.0
model = Sequential()
model.add(Conv2D(64, (3, 3), input_shape=x.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), input_shape=x.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation("relu"))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss="binary_crossentropy",
def model(self):
lr = self.lr
dropout = self.dropout
augmentation = self.augmentation
normalization = self.normalization
# Gather inputs. We will reshape TB2.
EMB1 = Input(shape=(128, 4, 1), name='EMB1')
EMB2 = Input(shape=(16, 16, 1), name='EMB2')
EMB3 = Input(shape=(8, 16, 1), name='EMB3')
TB0 = Input(shape=(4, 4, 1), name='TileBar0')
TB1 = Input(shape=(4, 4, 1), name='TileBar1')
TB2 = Input(shape=(2, 4, 1), name='TileBar2')
input1 = EMB1
input2 = ImageScaleBlock(
(16, 16), normalization=True,
name_prefix='emb_stack')([EMB2, EMB3]) # merge EMB2 and EMB3
input3 = ImageScaleBlock(
(4, 4), normalization=True,
name_prefix='tiles_stack')([TB0, TB1, TB2]) # merge TileBar
# From here on out, just follow Max's code.
# EMB1 image (convolutional)
x1 = Conv2D(32, (3, 3), padding='same', name='emb1_conv2d_1')(input1)
x1 = Activation('relu')(x1)
# x1 = Dropout(dropout)(x1)
x1 = Conv2D(32, (3, 3), padding='same', name='emb1_conv2d_2')(x1)
x1 = Activation('relu')(x1)
x1 = MaxPooling2D(pool_size=(2, 1),
padding='same',
name='emb1_maxpool_3')(x1)
x1 = Conv2D(64, (3, 3), padding='same', name='emb1_conv2d_3')(x1)
x1 = Activation('relu')(x1)
#x1 = Dropout(dropout)(x1)
x1 = Conv2D(64, (3, 3), padding='same', name='emb1_conv2d_4')(x1)
x1 = Activation('relu')(x1)
x1 = MaxPooling2D(pool_size=(2, 1),
padding='same',
name='emb1_maxpool_5')(x1)
x1 = Conv2D(128, (2, 2), padding='same', name='emb1_conv2d_6')(x1)
x1 = Activation('relu')(x1)
x1 = Conv2D(128, (2, 2), padding='same', name='emb1_conv2d_7')(x1)
x1 = Activation('relu')(x1)
x1 = MaxPooling2D(pool_size=(2, 1),
padding='same',
name='emb1_maxpool_8')(x1)
x1 = Dropout(dropout, name='emb1_dropout_4')(x1)
x1 = Flatten(name='emb1_flatten_9')(x1)
x1 = Dense(128, activation='relu', name='emb1_dense_9')(x1)
# EMB23 image (convolutional)
x2 = Conv2D(32, (1, 1), padding='same', name='emb23_conv1d_1')(input2)
x2 = Activation('relu')(x2)
# x2 = Dropout(dropout)(x2)
x2 = Conv2D(64, (2, 2), padding='same', name='emb23_conv2d_2')(x2)
# x2 = Dropout(dropout)(x2)
x2 = MaxPooling2D(pool_size=(2, 2),
padding='same',
name='emb23_maxpool_3')(x2)
x2 = Conv2D(128, (2, 2), padding='same', name='emb23_conv2d_4')(x2)
x2 = Activation('relu')(x2)
# x2 = Dropout(dropout)(x2)
x2 = Conv2D(128, (2, 2), padding='same', name='emb23_conv2d_5')(x2)
x2 = Activation('relu')(x2)
x2 = MaxPooling2D(pool_size=(2, 2),
padding='same',
name='emb23_maxpool_6')(x2)
x2 = Dropout(dropout, name='emb23_dropout_4')(x2)
x2 = Flatten(name='emb23_flatten_7')(x2)
x2 = Dense(128, activation='relu', name='emb23_dense_8')(x2)
# tiles image (convolutional)
x3 = Conv2D(32, (1, 1), padding='same', name='tiles_conv1d_1')(input3)
x3 = Activation('relu')(x3)
# x3 = Dropout(dropout)(x3)
x3 = Conv2D(64, (2, 2), padding='same', name='tiles_conv2d_2')(x3)
x3 = Activation('relu')(x3)
# x3 = Dropout(dropout)(x3)
x3 = Conv2D(128, (2, 2), padding='same', name='tiles_conv2d_3')(x3)
x3 = Activation('relu')(x3)
x3 = MaxPooling2D(pool_size=(2, 2),
padding='same',
name='tiles_maxpool_4')(x3)
x3 = Dropout(dropout, name='tiles_dropout_4')(x3)
x3 = Flatten(name='tiles_flatten_5')(x3)
x3 = Dense(128, activation='relu', name='tiles_dense_6')(x3)
# concatenate outputs from the two networks above
x = Concatenate(axis=1, name='concatenate')([x1, x2, x3])
#x = concatenate([x1, x2, x3], name='concatenate')
x = Dropout(dropout, name='concate_dropout_5')(x)
x = Dense(64, name='concated_dense_1')(x)
x = Activation('relu')(x)
x = Dropout(dropout, name='dense_dropout_6')(x)
# final output
output = Dense(2, activation='softmax', name='dense_output')(x)
# output = 5*tf.math.tanh(x) # 0 to +5 range
model = Model(inputs=[EMB1, EMB2, EMB3, TB0, TB1, TB2],
outputs=[output])
# compile model
optimizer = Adam(learning_rate=lr)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['acc'])
return model
def f(x):
if bottleneck:
y = Conv3D(
output_features,
bottleneck_conv_kernel,
strides=stride,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_a",
kernel_initializer=kernel_initializer,
)(x)
else:
y = ZeroPadding3D(
padding=1,
name=f"resunit{resnet_unit_label}_block{block_id}_pad_a",
)(x)
y = Conv3D(
output_features,
conv_kernel,
strides=stride,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_a",
kernel_initializer=kernel_initializer,
)(y)
y = BatchNormalization(
axis=axis,
epsilon=bn_epsilon,
name=f"resunit{resnet_unit_label}_block{block_id}_bn_a",
)(y)
y = Activation(
activation,
name=f"resunit{resnet_unit_label}_block{block_id}_activation_a",
)(y)
y = ZeroPadding3D(
padding=1, name=f"resunit{resnet_unit_label}_block{block_id}_pad_b"
)(y)
y = Conv3D(
output_features,
conv_kernel,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_b",
kernel_initializer=kernel_initializer,
)(y)
y = BatchNormalization(
axis=axis,
epsilon=bn_epsilon,
name=f"resunit{resnet_unit_label}_block{block_id}_bn_b",
)(y)
if bottleneck:
y = Activation(
activation,
name=f"resunit{resnet_unit_label}_block{block_id}_activation_b",
)(y)
y = Conv3D(
output_features * 4,
bottleneck_conv_kernel,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_c",
kernel_initializer=kernel_initializer,
)(y)
y = BatchNormalization(
axis=axis,
epsilon=bn_epsilon,
name=f"resunit{resnet_unit_label}_block{block_id}_bn_c",
)(y)
identity_shortcut = get_shortcut(
x,
resnet_unit_label,
block_id,
output_features * 4,
stride,
axis=axis,
)
else:
identity_shortcut = get_shortcut(
x,
resnet_unit_label,
block_id,
output_features,
stride,
axis=axis,
)
y = Add(name=f"resunit{resnet_unit_label}_block{block_id}_add")(
[y, identity_shortcut]
)
y = Activation(
activation,
name=f"resunit{resnet_unit_label}_block{block_id}_activation_c",
)(y)
return y
input_size = image_size * image_size
# we train our network using float data
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# network parameters
batch_size = 128
hidden_units = 256
data_augmentation = True
epochs = 20
max_batches = len(x_train) / batch_size
# this is 3-layer MLP with ReLU after each layer
model = Sequential()
model.add(Dense(hidden_units, input_dim=input_size))
model.add(Activation('relu'))
model.add(Dense(hidden_units))
model.add(Activation('relu'))
model.add(Dense(num_labels))
# this is the output for one-hot vector
model.add(Activation('softmax'))
model.summary()
# loss function for one-hot vector
# use of sgd optimizer
# accuracy is good metric for classification tasks
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# validate the model on test dataset to determine generalization
# Build a Inception V3 architecture
# Input
input_tensor = Input(shape=(32, 32, 3), dtype='float32', name='input') #32,32,3
# Resale image (up-sampling) for better performance
upsampling = tf.keras.layers.UpSampling2D(size=upsampling_size, name='upsampling')(input_tensor) #96,96,3
# conv1
conv1_conv = Conv2D(32, 3, strides=(2, 2), kernel_initializer='he_normal', name='conv1_conv')(upsampling) #47,47,32
# kernel_initializer: a statistical distribution or function to use for initialising the weights
# - glorot_uniform: (default) also called Xavier uniform;
# Draws samples from a uniform distribution within [-limit, limit], where limit = sqrt(6 / (fan_in + fan_out))
# (fan_in is the number of input units in the weight tensor and fan_out is the number of output units).
# - he_normal:
# Draws samples from a truncated normal distribution centered on 0 with stddev = sqrt(2 / fan_in)
conv1_bn = BatchNormalization(axis=1, name='conv1_bn')(conv1_conv)
conv1_relu = Activation('relu', name='conv1_relu')(conv1_bn)
# conv2_1
conv2_1conv = Conv2D(32, 3, kernel_initializer='he_normal', name='conv2_1conv')(conv1_relu) #45,45,32
conv2_1bn = BatchNormalization(axis=1, name='conv2_1bn')(conv2_1conv)
conv2_1relu = Activation('relu', name='conv2_1relu')(conv2_1bn)
# conv2_2
conv2_2conv = Conv2D(64, 3, padding='SAME', kernel_initializer='he_normal', name='conv2_2conv')(conv2_1relu) #45,45,64
# 지수쌤은 위에 Conv2D에 padding='SAME'을 넣어줬음
# Padded라고 표기 된 convolution과, grid size 유지가 필요한 inception module을 제외하고는 padding을 사용하지 않는다
# ref. https://sike6054.github.io/blog/paper/third-post/
conv2_2bn = BatchNormalization(axis=1, name='conv2_2bn')(conv2_2conv)
conv2_2relu = Activation('relu', name='conv2_2relu')(conv2_2bn)
# maxpool1
from tensorflow.keras.layers import Input, Concatenate, concatenate
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import BatchNormalization, Activation
from tensorflow.keras.layers import ReLU
from tensorflow.keras.layers import GlobalAveragePooling2D, ZeroPadding2D, Add
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Model
# create model
input_img = Input(shape=(256, 256, 1), name='main_input')
#VGG Net
x1 = Conv2D(64, (3, 3))(input_img)
x1 = Activation('relu')(x1)
x1 = Conv2D(64, (3, 3))(x1)
x1 = Activation('relu')(x1)
x1 = MaxPooling2D()(x1)
x1 = Conv2D(64, (3, 3))(x1)
x1 = Activation('relu')(x1)
x1 = Conv2D(64, (3, 3))(x1)
x1 = Activation('relu')(x1)
x1 = MaxPooling2D()(x1)
x1 = Conv2D(64, (3, 3))(x1)
x1 = Activation('relu')(x1)
x1 = MaxPooling2D()(x1)
x1 = Flatten()(x1)
x1 = Dense(256)(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
def MobileNetV3(stack_fn,
last_point_ch,
input_shape=None,
alpha=1.0,
model_type='large',
minimalistic=False,
include_top=True,
weights='imagenet',
input_tensor=None,
classes=1000,
pooling=None,
dropout_rate=0.2,
**kwargs):
"""Instantiates the MobileNetV3 architecture.
# Arguments
stack_fn: a function that returns output tensor for the
stacked residual blocks.
last_point_ch: number channels at the last layer (before top)
input_shape: optional shape tuple, to be specified if you would
like to use a model with an input img resolution that is not
(224, 224, 3).
It should have exactly 3 inputs channels (224, 224, 3).
You can also omit this option if you would like
to infer input_shape from an input_tensor.
If you choose to include both input_tensor and input_shape then
input_shape will be used if they match, if the shapes
do not match then we will throw an error.
E.g. `(160, 160, 3)` would be one valid value.
alpha: controls the width of the network. This is known as the
depth multiplier in the MobileNetV3 paper, but the name is kept for
consistency with MobileNetV1 in Keras.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
model_type: MobileNetV3 is defined as two models: large and small. These
models are targeted at high and low resource use cases respectively.
minimalistic: In addition to large and small models this module also contains
so-called minimalistic models, these models have the same per-layer
dimensions characteristic as MobilenetV3 however, they don't utilize any
of the advanced blocks (squeeze-and-excite units, hard-swish, and 5x5
convolutions). While these models are less efficient on CPU, they are
much more performant on GPU/DSP.
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
pooling: optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
dropout_rate: fraction of the input units to drop on the last layer
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid model type, argument for `weights`,
or invalid input shape when weights='imagenet'
"""
#global backend, layers, models, keras_utils
#backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs)
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError(
'If using `weights` as `"imagenet"` with `include_top` '
'as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
# Determine proper input shape and default size.
# If both input_shape and input_tensor are used, they should match
if input_shape is not None and input_tensor is not None:
try:
is_input_t_tensor = K.is_keras_tensor(input_tensor)
except ValueError:
try:
is_input_t_tensor = K.is_keras_tensor(
get_source_inputs(input_tensor))
except ValueError:
raise ValueError('input_tensor: ', input_tensor,
'is not type input_tensor')
if is_input_t_tensor:
if K.image_data_format == 'channels_first':
if K.int_shape(input_tensor)[1] != input_shape[1]:
raise ValueError(
'input_shape: ', input_shape, 'and input_tensor: ',
input_tensor,
'do not meet the same shape requirements')
else:
if K.int_shape(input_tensor)[2] != input_shape[1]:
raise ValueError(
'input_shape: ', input_shape, 'and input_tensor: ',
input_tensor,
'do not meet the same shape requirements')
else:
raise ValueError('input_tensor specified: ', input_tensor,
'is not a keras tensor')
# If input_shape is None, infer shape from input_tensor
#if input_shape is None and input_tensor is not None:
try:
K.is_keras_tensor(input_tensor)
except ValueError:
raise ValueError('input_tensor: ', input_tensor, 'is type: ',
type(input_tensor), 'which is not a valid type')
if K.is_keras_tensor(input_tensor):
if K.image_data_format() == 'channels_first':
rows = K.int_shape(input_tensor)[2]
cols = K.int_shape(input_tensor)[3]
input_shape = (3, cols, rows)
else:
rows = K.int_shape(input_tensor)[1]
cols = K.int_shape(input_tensor)[2]
input_shape = (cols, rows, 3)
# If input_shape is None and input_tensor is None using standart shape
if input_shape is None and input_tensor is None:
input_shape = (None, None, 3)
if K.image_data_format() == 'channels_last':
row_axis, col_axis = (0, 1)
else:
row_axis, col_axis = (1, 2)
rows = input_shape[row_axis]
cols = input_shape[col_axis]
if rows and cols and (rows < 32 or cols < 32):
raise ValueError(
'Input size must be at least 32x32; got `input_shape=' +
str(input_shape) + '`')
if weights == 'imagenet':
if minimalistic is False and alpha not in [0.75, 1.0] \
or minimalistic is True and alpha != 1.0:
raise ValueError(
'If imagenet weights are being loaded, '
'alpha can be one of `0.75`, `1.0` for non minimalistic'
' or `1.0` for minimalistic only.')
if rows != cols or rows != 224:
warnings.warn('`input_shape` is undefined or non-square, '
'or `rows` is not 224.'
' Weights for input shape (224, 224) will be'
' loaded as the default.')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
img_input = input_tensor
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
if minimalistic:
kernel = 3
activation = relu
se_ratio = None
else:
kernel = 5
activation = hard_swish
se_ratio = 0.25
x = ZeroPadding2D(padding=correct_pad(K, img_input, 3),
name='Conv_pad')(img_input)
x = Conv2D(16,
kernel_size=3,
strides=(2, 2),
padding='valid',
use_bias=False,
name='Conv')(x)
x = BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name='Conv/BatchNorm')(x)
x = Activation(activation)(x)
x = stack_fn(x, kernel, activation, se_ratio)
last_conv_ch = _depth(K.int_shape(x)[channel_axis] * 6)
# if the width multiplier is greater than 1 we
# increase the number of output channels
if alpha > 1.0:
last_point_ch = _depth(last_point_ch * alpha)
x = Conv2D(last_conv_ch,
kernel_size=1,
padding='same',
use_bias=False,
name='Conv_1')(x)
x = BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name='Conv_1/BatchNorm')(x)
x = Activation(activation)(x)
if include_top:
x = GlobalAveragePooling2D()(x)
if channel_axis == 1:
x = Reshape((last_conv_ch, 1, 1))(x)
else:
x = Reshape((1, 1, last_conv_ch))(x)
x = Conv2D(last_point_ch, kernel_size=1, padding='same',
name='Conv_2')(x)
x = Activation(activation)(x)
if dropout_rate > 0:
x = Dropout(dropout_rate)(x)
x = Conv2D(classes, kernel_size=1, padding='same', name='Logits')(x)
x = Flatten()(x)
x = Softmax(name='Predictions/Softmax')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D(name='avg_pool')(x)
elif pooling == 'max':
x = GlobalMaxPooling2D(name='max_pool')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='MobilenetV3' + model_type)
# Load weights.
if weights == 'imagenet':
model_name = "{}{}_224_{}_float".format(
model_type, '_minimalistic' if minimalistic else '', str(alpha))
if include_top:
file_name = 'weights_mobilenet_v3_' + model_name + '.h5'
file_hash = WEIGHTS_HASHES[model_name][0]
else:
file_name = 'weights_mobilenet_v3_' + model_name + '_no_top.h5'
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = get_file(file_name,
BASE_WEIGHT_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def DenseNet(nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.0, dropout_rate=0.0, weight_decay=1e-4, classes=1000, weights_path=None):
'''Instantiate the DenseNet architecture,
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_data_format() == 'channels_last':
concat_axis = 3
img_input = Input(shape=(224, 224, 3), name='data')
else:
concat_axis = 1
img_input = Input(shape=(3, 224, 224), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 32, 32] # For DenseNet-169
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter, (7, 7), strides=2, name='conv1', use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx+2
x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay)
# Add transition_block
x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate, weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x)
x = Scale(axis=concat_axis, name='conv'+str(final_stage)+'_blk_scale')(x)
x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x)
x = GlobalAveragePooling2D(name='pool'+str(final_stage))(x)
x = Dense(classes, name='fc6')(x)
x = Activation('softmax', name='prob')(x)
model = Model(img_input, x, name='densenet')
if weights_path is not None:
model.load_weights(weights_path)
return model
def build_model(img_size): # create model architecture and compile it
model = Sequential()
# Image input shape: 256 x 256 x 3
# 1. Convolution Layer: 10 filters of 5px by 5px
model.add(
Conv2D(10, (5, 5),
input_shape=(img_size, img_size, 3),
kernel_regularizer=regularizers.l2(0.1)))
# Output shape: 10 x 252 x 252
# 2. Batch Normalization: Normalizes previous layer to have mean near 0 and S.D. near 1
model.add(BatchNormalization())
# Output shape: 10 x 252 x 252
# 3. Activation Layer: ReLU uses the formula of f(x)= x if x>0 and 0 if x<=0
# Apparently it's a pretty common one for CNN so we're going with the flow here
model.add(Activation("relu"))
# Output shape: 10 x 252 x 252
# 4. Pooling function: from the paper, it didn't specify function, but looking online, it seems that the default is Max so we are a-okay here
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Output shape: 10 x 126 x 126
#-------------Next Set of Layers--------------
# 5. Convolution Layer: 40 filters of 5px by 5px
model.add(Conv2D(40, (5, 5), kernel_regularizer=regularizers.l2(0.1)))
# Output shape: 40 x 122 x 122
# 6. Batch Normalization Layer
model.add(BatchNormalization())
# Output shape: 40 x 122 x 122
# 7. Activation Layer: Same as above
model.add(Activation("relu"))
# Output shape: 40 x 122 x 122
# 8. Pooling again will decrease "image shape" by half since stride = 2
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Output shape: 40 x 61 x 61
# ----------Hidden layers-----------
# 9. Flattening Layer: Make pooled layers (that look like stacks of grids) into one "column" to feed into ANN
model.add(Flatten())
# 10. Dropout Layer: In Mathematica Dropout[] has a rate of dropping 50% of elements and multiply rest by 2
# !!!!!!! Currently trying to figure out how to do the multiply by 2 but moving on for now !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
model.add(Dropout(0.5, seed=SEED))
model.add(
Dense(500,
activation="linear",
activity_regularizer=regularizers.l2(0.05),
kernel_regularizer=regularizers.l2(
0.1))) # kernel_regularizer=regularizers.l2(0.1)))
model.add(
Dense(500,
activation="relu",
activity_regularizer=regularizers.l2(0.05),
kernel_regularizer=regularizers.l2(0.1)))
#model.add(Activation("relu"))
model.add(Dropout(0.25, seed=SEED))
# The output layer with 2 neurons, for 2 classes
model.add(
Dense(2,
activation="linear",
activity_regularizer=regularizers.l2(0.05),
kernel_regularizer=regularizers.l2(0.1)))
model.add(
Dense(2,
activation="softmax",
activity_regularizer=regularizers.l2(0.05),
kernel_regularizer=regularizers.l2(0.1)))
# model.add(Activation("softmax"))
opt = tf.keras.optimizers.Adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=0.00001,
decay=0.0,
amsgrad=False)
model.compile(loss="sparse_categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
return model
def hard_swish(x):
return Multiply()([Activation(hard_sigmoid)(x), x])
###### BUILD THE MODEL
hidden_dim = 4
obs_dim = 3
action_dim = 2
layer_dim = 2
hiddens = Input(shape=(hidden_dim, ))
obs = Input(shape=(obs_dim, ))
Wh = Dense(layer_dim)(hiddens)
Wo = Dense(layer_dim)(obs)
interm1 = Concatenate(axis=-1)([Wh, Wo])
interm2 = Activation('relu')(interm1)
interm3 = Dense(action_dim)(interm2)
action_weights = Activation('relu', name="output")(interm3)
model = Model(inputs=[hiddens, obs], outputs=action_weights)
############# SAVE THE MODEL
#log_dir = "/home/csidrane/Data/AAHAA/RNNs/run_1_test"
os.mkdir(log_dir)
sess = tf.Session()
with sess.as_default():
sess.run(tf.global_variables_initializer())
[print(n.name) for n in tf.get_default_graph().as_graph_def().node]
trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
print(trainX[0])
print("shape: ", trainX.shape)
model = Sequential()
model.add(GRU(50, input_shape=trainX[0].shape, return_sequences=True))
model.add(Dropout(0.35))
model.add(GRU(100, return_sequences=False))
model.add(Dropout(0.35))
model.add(Dense(1))
model.add(Activation('relu'))
opt = tf.keras.optimizers.Adam(lr=0.001, decay=1e-5)
model.compile(loss='mae', optimizer=opt, metrics=['accuracy', 'mse'])
history = model.fit(trainX,
trainY,
epochs=500,
batch_size=1750,
validation_data=(testX, testY),
verbose=0,
shuffle=False)
score = model.evaluate(testX, testY, verbose=1)
print(model.metrics_names)
print("score: ", score)
# pyplot.plot(history.history['loss'], label='train')
def auto_encoder(width=None,
height=None,
depth=None,
filters=None,
latentDim=None):
def conv_2d(x, f, transpose, chanDim):
if transpose:
x = Conv2DTranspose(f, (3, 3), strides=2, padding="same")(x)
else:
x = Conv2D(f, (3, 3), strides=2, padding="same")(x)
x = LeakyReLU(alpha=0.2)(x)
x = BatchNormalization(axis=chanDim)(x)
return x
# initialize the input shape to be "channels last" along with
# the channels dimension itself
# channels dimension itself
base_model = DenseNet121(input_shape=(224, 224, 3),
weights='imagenet',
include_top=True)
for layer in base_model.layers:
layer.trainable = False
feature = base_model.layers[-2].output
#feature = Dense(512, activation='relu')(feature)
inputShape = (height, width, depth)
chanDim = -1
# define the input to the encoder
inputs = base_model.input
x = inputs
# loop over the number of filters
x = conv_2d(x, 8, 0, chanDim)
x1 = x
x = conv_2d(x, 16, 0, chanDim)
x2 = x
x = conv_2d(x, 32, 0, chanDim)
x3 = x
# flatten the network and then construct our latent vector
volumeSize = K.int_shape(x)
x = Flatten()(x)
latent = Dense(1024, name="encoded")(x)
latent = add([feature, latent])
x = Dense(np.prod(volumeSize[1:]))(latent)
x = Reshape((volumeSize[1], volumeSize[2], volumeSize[3]))(x)
# loop over our number of filters again, but this time in
# reverse order
x = concatenate([x, x3])
x = conv_2d(x, 32, 1, chanDim)
x = concatenate([x, x2])
x = conv_2d(x, 16, 1, chanDim)
x = concatenate([x, x1])
x = conv_2d(x, 3, 1, chanDim)
#x = conv_2d(x,3,1,chanDim)
#x = concatenate([x,inputs])
outputs = Conv2DTranspose(3, (3, 3), padding="same")(x)
outputs = Activation("sigmoid", name="decoded")(x)
# construct our autoencoder model
autoencoder = Model(inputs, outputs, name="autoencoder")
# return the autoencoder model
autoencoder.compile(optimizer='adam', loss='mse')
return autoencoder
timeseries = timeseries.T
inp = Input(shape=(length, 1))
l1a, l1b = DC_CNN_Block(32, 2, 1, 0.001)(inp)
l2a, l2b = DC_CNN_Block(32, 2, 2, 0.001)(l1a)
l3a, l3b = DC_CNN_Block(32, 2, 4, 0.001)(l2a)
l4a, l4b = DC_CNN_Block(32, 2, 8, 0.001)(l3a)
l5a, l5b = DC_CNN_Block(32, 2, 16, 0.001)(l4a)
l6a, l6b = DC_CNN_Block(32, 2, 32, 0.001)(l5a)
l6b = Dropout(0.8)(l6b)
l7a, l7b = DC_CNN_Block(32, 2, 64, 0.001)(l6a)
l7b = Dropout(0.8)(l7b)
l8 = Add()([l1b, l2b, l3b, l4b, l5b, l6b, l7b])
l9 = Activation('relu')(l8)
l21 = Conv1D(1,
1,
activation='linear',
use_bias=False,
kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.05,
seed=42),
kernel_regularizer=l2(0.001))(l9)
model = Model(inputs=inp, outputs=l21)
adam = optimizers.Adam(lr=0.00075,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(loss='mse', optimizer=adam, metrics=['mse'])
def mobilenetv2(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a model based on a pretrained MobileNetV2 for multiclass US classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
if model_config['OPTIMIZER'] == 'sgd':
optimizer = SGD(learning_rate=lr, momentum=0.9)
else:
optimizer = Adam(learning_rate=lr)
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Start with pretrained MobileNetV2
X_input = Input(input_shape, name='input')
base_model = MobileNetV2(include_top=False,
weights='imagenet',
input_shape=input_shape,
input_tensor=X_input)
'''
for layer in base_model.layers[:30]:
layer.trainable = False
for layer in base_model.layers[30:]:
layer.trainable = True
if 'keras.layers.Conv2D' in layer._keras_api_names:
setattr(layer, 'activity_regularizer', l2(l2_lambda))
print("Trainable layer with regularization added", layer.name)
'''
X = base_model.output
# Add custom top layers
X = BatchNormalization()(X)
X = GlobalAveragePooling2D()(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
#X = LeakyReLU()(X)
X = BatchNormalization()(X)
#X = Dropout(dropout)(X)
#X = Dense(nodes_dense1, activity_regularizer=l2(l2_lambda))(X)
#X = LeakyReLU()(X)
X = Dropout(dropout)(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
def resnet_v2(input_shape, depth, num_classes=10):
if (depth - 2) % 9 != 0:
raise ValueError('depth should be 9n+2 (e.g. 20, 101, 164, ...)')
num_filters_in = 16
num_res_blocks = int((depth - 2) / 9)
inputs = Input(shape=input_shape)
# input conv
x = resnet_layer(inputs=inputs,
num_filters=num_filters_in,
conv_first=True)
# downsample feature map 1/2 times for every 2n iterations
# expand output filter size 2 times for every 2n iterations
for stage in range(3):
# residual block
for res_block in range(num_res_blocks):
strides = 1
activation = 'relu'
batch_normalization = True
# downsample the feature map with 1/2 for every 2n iterations
if stage == 0:
num_filters_out = num_filters_in * 4
# if res_block == 0:
# activation = None
# batch_normalization = False
else:
num_filters_out = num_filters_in * 2
if res_block == 0:
strides = 2
y = resnet_layer(inputs=x,
num_filters=num_filters_in,
kernel_size=1,
strides=strides,
activation=activation,
batch_normalization=batch_normalization,
conv_first=False)
y = resnet_layer(inputs=y,
num_filters=num_filters_in,
kernel_size=3,
strides=1,
activation='relu',
conv_first=False)
y = resnet_layer(inputs=y,
num_filters=num_filters_out,
kernel_size=1,
strides=1,
activation='relu',
conv_first=False)
# downsampling the input x is required for skip-connection
# note that activation and BN are not applied! (only resizing the feature map)
if res_block == 0:
x = resnet_layer(inputs=x,
num_filters=num_filters_out,
kernel_size=1,
strides=strides,
activation=None,
batch_normalization=False)
# shortcut-connection
x = add([x, y])
# expand filter size for every 2n iterations
num_filters_in = num_filters_out
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = AveragePooling2D(pool_size=8, padding='same')(x)
y = Flatten()(x)
outputs = Dense(num_classes,
activation='softmax',
kernel_initializer='he_normal')(y)
model = Model(inputs=inputs, outputs=outputs)
return model
def xception(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a model based on a pretrained Xception for multiclass US classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
optimizer = Adam(learning_rate=lr)
frozen_layers = model_config['FROZEN_LAYERS']
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Start with pretrained Xception
X_input = Input(input_shape, name='input')
base_model = Xception(include_top=False,
weights='imagenet',
input_shape=input_shape,
input_tensor=X_input)
# Freeze desired conv layers set in config.yml
for layers in range(len(frozen_layers)):
layer2freeze = frozen_layers[layers]
print('Freezing layer: ' + str(layer2freeze))
base_model.layers[layer2freeze].trainable = False
# Add regularization to Xception conv layers
for layer_idx in model_config['L2_LAYERS']:
if base_models.layers[layer_idx].trainable:
setattr(base_models.layers[layer_idx], 'activity_regularizer',
l2(l2_lambda))
print('Adding regularization to: ' +
str(base_models.layers[layer_idx]))
X = base_model.output
# Add custom top layers
X = GlobalAveragePooling2D()(X)
X = Dropout(dropout)(X)
X = Dense(n_classes, bias_initializer=output_bias, name='logits')(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
return_sequences=True,
return_state=True)(input_train)
encoder_last_h = BatchNormalization(momentum=0.6)(encoder_last_h)
encoder_last_c = BatchNormalization(momentum=0.6)(encoder_last_c)
decoder_stack_h = RepeatVector(output_train.shape[1])(encoder_last_h)
decoder_stack_h = LSTM(n_hidden,
activation='elu',
dropout=0.2,
recurrent_dropout=0.2,
return_state=False,
return_sequences=True)(
decoder_stack_h,
initial_state=[encoder_last_h, encoder_last_c])
alignment_score = dot([decoder_stack_h, encoder_stack_h], axes=[2, 2])
attention = Activation('softmax')(alignment_score)
context = dot([attention, encoder_stack_h], axes=[2, 1])
context = BatchNormalization(momentum=0.6)(context)
decoder_combined_context = concatenate([context, decoder_stack_h])
out = TimeDistributed(Dense(output_train.shape[2]))(decoder_combined_context)
model = Model(inputs=input_train, outputs=out)
opt = Adam(lr=0.01, clipnorm=1)
model.compile(loss='mean_squared_error', optimizer=opt, metrics=['mae'])
print(model.summary())
plot_model(model,
to_file='attention_model_plot.png',
show_shapes=True,
show_layer_names=True)
def custom_resnet(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a deep convolutional neural network model with residual connections for multiclass image classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
optimizer = Adam(learning_rate=lr)
init_filters = model_config['INIT_FILTERS']
filter_exp_base = model_config['FILTER_EXP_BASE']
res_blocks = model_config['RES_BLOCKS']
kernel_size = eval(model_config['KERNEL_SIZE'])
max_pool_size = eval(model_config['MAXPOOL_SIZE'])
strides = eval(model_config['STRIDES'])
print("MODEL CONFIG: ", model_config)
pad = kernel_size[0] // 2
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Input layer
X_input = Input(input_shape)
X = X_input
X = ZeroPadding2D((pad, pad))(X)
# Initialize the model with a convolutional layer
X = Conv2D(init_filters, (7, 7),
strides=strides,
name='conv0',
kernel_initializer='he_uniform')(X)
X = BatchNormalization(axis=3, name='bn_conv0')(X)
X = LeakyReLU()(X)
X = MaxPool2D(max_pool_size, padding='same', name='maxpool0')(X)
# Add residual blocks
for i in range(res_blocks):
f1 = f2 = init_filters * (filter_exp_base**i)
f3 = init_filters * (filter_exp_base**(i + 2))
X = convolutional_block(X,
kernel_size=kernel_size,
filters=[f1, f2, f3],
stage=(i + 1),
block='a',
s=strides)
X = identity_block(X,
kernel_size=kernel_size,
filters=[f1, f2, f3],
stage=(i + 1),
block='b')
X = identity_block(X,
kernel_size=kernel_size,
filters=[f1, f2, f3],
stage=(i + 1),
block='c')
# Add fully connected layers
X = AveragePooling2D(strides, name='avgpool0')(X)
X = Flatten()(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
kernel_initializer='he_uniform',
activity_regularizer=l2(l2_lambda),
name='fc0')(X)
X = LeakyReLU()(X)
X = Dropout(dropout)(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
from tensorflow.keras.layers import (Conv3D, BatchNormalization, AveragePooling3D, concatenate, Lambda, Activation, Input, GlobalAvgPool3D, Dense)
from tensorflow.keras.regularizers import l2 as l2_penalty
from tensorflow.keras.models import Model
from mylibrary.models.metrics import invasion_acc, invasion_precision, invasion_recall, invasion_fmeasure
PARAMS = {
'activation': lambda: Activation('relu'), # the activation functions
'bn_scale': True, # whether to use the scale function in BN
'weight_decay': 0.0001, # l2 weight decay
'kernel_initializer': 'he_uniform', # initialization
'first_scale': lambda x: x / 128. - 1., # the first pre-processing function
'dhw': [32, 32, 32], # the input shape
'k': 16, # the `growth rate` in DenseNet
'bottleneck': 4, # the `bottleneck` in DenseNet
'compression': 2, # the `compression` in DenseNet
'first_layer': 32, # the channel of the first layer
'down_structure': [4, 4, 4], # the down-sample structure
'output_size': 2 # the output number of the classification head
}
def _conv_block(x, filters):
bn_scale = PARAMS['bn_scale']
activation = PARAMS['activation']
kernel_initializer = PARAMS['kernel_initializer']
weight_decay = PARAMS['weight_decay']
bottleneck = PARAMS['bottleneck']
x = BatchNormalization(scale=bn_scale, axis=-1)(x)
x = activation()(x)
def custom_ffcnn(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a feedforward convolutional neural network model with residual connections for multiclass image classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
if model_config['OPTIMIZER'] == 'sgd':
optimizer = SGD(learning_rate=lr, momentum=0.9)
else:
optimizer = Adam(learning_rate=lr)
init_filters = model_config['INIT_FILTERS']
filter_exp_base = model_config['FILTER_EXP_BASE']
n_blocks = model_config['BLOCKS']
kernel_size = eval(model_config['KERNEL_SIZE'])
max_pool_size = eval(model_config['MAXPOOL_SIZE'])
strides = eval(model_config['STRIDES'])
pad = kernel_size[0] // 2
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Input layer
X_input = Input(input_shape)
X = X_input
X = ZeroPadding2D((pad, pad))(X)
# Add blocks of convolutions and max pooling
for i in range(n_blocks):
filters = init_filters * (2**i)
X = Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
name='conv2d_block' + str(i) + '_0',
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
name='conv2d_block' + str(i) + '_1',
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = BatchNormalization(axis=3, name='bn_block' + str(i))(X)
X = MaxPool2D(max_pool_size, padding='same',
name='maxpool' + str(i))(X)
# Model head
X = GlobalAveragePooling2D(name='gloval_avgpool')(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
kernel_initializer='he_uniform',
activity_regularizer=l2(l2_lambda),
activation='relu',
name='fc0')(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense1,
kernel_initializer='he_uniform',
activity_regularizer=l2(l2_lambda),
activation='relu',
name='fc1')(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
def create_model(net_config,
model_name,
loss=None,
use_newloss=False,
use_AdvDataset=False,
adv_param=None,
n_adv_tau=None,
adv_learning_rate=None):
tau_net_setup = NetSetup1D(**net_config.tau_net)
comp_net_setup = NetSetup2D(**net_config.comp_net)
comp_merge_net_setup = NetSetup2D(**net_config.comp_merge_net)
conv_2d_net_setup = NetSetupConv2D(**net_config.conv_2d_net)
dense_net_setup = NetSetup1D(**net_config.dense_net)
input_layers = []
high_level_features = []
if net_config.n_tau_branches > 0:
input_layer_tau = Input(name="input_tau",
shape=(net_config.n_tau_branches, ))
input_layers.append(input_layer_tau)
tau_net_setup.ComputeLayerSizes(net_config.n_tau_branches)
processed_tau = reduce_n_features_1d(input_layer_tau, tau_net_setup,
'tau', net_config.first_layer_reg)
high_level_features.append(processed_tau)
for loc in net_config.cell_locations:
reduced_inputs = []
for comp_id in range(len(net_config.comp_names)):
comp_name = net_config.comp_names[comp_id]
n_comp_features = net_config.n_comp_branches[comp_id]
input_layer_comp = Input(name="input_{}_{}".format(loc, comp_name),
shape=(net_config.n_cells[loc],
net_config.n_cells[loc],
n_comp_features))
input_layers.append(input_layer_comp)
comp_net_setup.ComputeLayerSizes(n_comp_features)
reduced_comp = reduce_n_features_2d(input_layer_comp,
comp_net_setup,
"{}_{}".format(loc, comp_name),
net_config.first_layer_reg)
reduced_inputs.append(reduced_comp)
if len(net_config.comp_names) > 1:
conv_all_start = Concatenate(name="{}_cell_concat".format(loc),
axis=3)(reduced_inputs)
comp_merge_net_setup.ComputeLayerSizes(
conv_all_start.shape.as_list()[3])
prev_layer = reduce_n_features_2d(conv_all_start,
comp_merge_net_setup,
"{}_all".format(loc))
else:
prev_layer = reduced_inputs[0]
current_grid_size = net_config.n_cells[loc]
n_inputs = prev_layer.shape.as_list()[3]
n = 1
while current_grid_size > 1:
if loc == "outer" and n == conv_2d_net_setup.pooling: # Currently works for input ncells 2, 3, 6, 9, 10, 14, 15, 18, 21, 22, 26, 27, 33, ...
poolgridsize = 2 if (
current_grid_size % 2 == 0 and
(current_grid_size / 2) % 2 == 1
) else 3 # Ensure that current_grid_size is odd after pooling
prev_layer = pool_layer(prev_layer, poolgridsize,
conv_2d_net_setup,
"{}_pooling".format(loc), n)
n += 1
current_grid_size = int(current_grid_size / poolgridsize)
win_size = min(current_grid_size, conv_2d_net_setup.window_size)
n_filters = get_n_filters_conv2d(n_inputs, current_grid_size,
win_size,
conv_2d_net_setup.reduction_rate)
prev_layer = conv_block(
prev_layer, n_filters, (win_size, win_size), conv_2d_net_setup,
"{}_all_{}x{}".format(loc, win_size, win_size), n)
n += 1
current_grid_size -= win_size - 1
n_inputs = n_filters
cells_flatten = Flatten(
name="{}_cells_flatten".format(loc))(prev_layer)
high_level_features.append(cells_flatten)
if len(high_level_features) > 1:
features_concat = Concatenate(name="features_concat",
axis=1)(high_level_features)
else:
features_concat = high_level_features[0]
dense_net_setup.ComputeLayerSizes(features_concat.shape.as_list()[1])
final_dense = reduce_n_features_1d(features_concat, dense_net_setup,
'final')
output_layer = Dense(
net_config.n_outputs,
name="final_dense_last",
kernel_initializer=dense_net_setup.kernel_init)(final_dense)
softmax_output = Activation("softmax", name="main_output")(output_layer)
if use_AdvDataset:
final_dense_adv = reduce_n_features_1d(features_concat,
dense_net_setup, 'final_adv')
output_layer_adv = Dense(
1,
name="final_dense_adv",
kernel_initializer=dense_net_setup.kernel_init)(final_dense_adv)
sigmoid_output_adv = Activation("sigmoid",
name="adv_output")(output_layer_adv)
model = DeepTauModel(input_layers,
[softmax_output, sigmoid_output_adv],
loss=loss,
name=model_name,
use_newloss=use_newloss,
use_AdvDataset=True,
adv_parameter=adv_param,
n_adv_tau=n_adv_tau,
adv_learning_rate=adv_learning_rate)
else:
model = DeepTauModel(input_layers,
softmax_output,
loss=loss,
name=model_name,
use_newloss=use_newloss)
return model
def resnet101v2(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a model based on a pretrained ResNet50V2 for multiclass US classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
optimizer = Adam(learning_rate=lr)
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Start with pretrained ResNet101V2
X_input = Input(input_shape, name='input')
base_model = ResNet101V2(include_top=False,
weights='imagenet',
input_shape=input_shape,
input_tensor=X_input)
X = base_model.output
# Add custom top layers
X = GlobalAveragePooling2D()(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense1,
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
def ResNet18(include_top=True,
weights='cifar100_coarse',
input_tensor=None,
input_shape=None,
pooling=None,
classes=20,
**kwargs):
global backend, layers, models, keras_utils
backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs)
if not (weights in {'cifar100_coarse', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `cifar100_coarse` '
'(pre-training on cifar100 coarse (super) classes), '
'or the path to the weights file to be loaded.')
if weights == 'cifar100_coarse' and include_top and classes != 20:
raise ValueError(
'If using `weights` as `"cifar100_coarse"` with `include_top`'
' as true, `classes` should be 20')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=32,
data_format=backend.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not tf.keras.backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input)
x = Convolution2D(64, (7, 7),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = identity_block(x, 3, [64, 64], stage=2, block='a')
x = identity_block(x, 3, [64, 64], stage=2, block='b')
x = conv_block(x, 3, [128, 128], stage=3, block='a')
x = identity_block(x, 3, [128, 128], stage=3, block='b')
x = conv_block(x, 3, [256, 256], stage=4, block='a')
x = identity_block(x, 3, [256, 256], stage=4, block='b')
x = conv_block(x, 3, [512, 512], stage=5, block='a')
x = identity_block(x, 3, [512, 512], stage=5, block='b')
if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='fc20')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D()(x)
'''
else:
warnings.warn('The output shape of `ResNet18(include_top=False)` '
'has been changed since Keras 2.2.0.')
'''
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet18')
# Load weights.
if weights == 'cifar100_coarse':
if include_top:
weights_path = keras_utils.get_file(
'resnet18_cifar100_top.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='e0798dd90ac7e0498cbdea853bd3ed7f')
else:
weights_path = keras_utils.get_file(
'resnet18_cifar100_no_top.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='bfeace78cec55f2b0401c1f41c81e1dd')
model.load_weights(weights_path)
return model
y = combined_df[[
"party", "day_after_1", "day_after_7", "day_after_30", "day_after_60",
"day_after_180", "day_after_365"
]]
#X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
X_train = X.iloc[:-1]
y_train = y.iloc[:-1]
X_test = X.iloc[-1:]
y_test = y.iloc[-1:]
sgd = SGD(lr=0.01, momentum=0.9, nesterov=True)
model = Sequential()
model.add(Dense(10, input_dim=len(X.columns)))
model.add(Activation("relu"))
model.add(Dense(len(y.columns)))
model.add(Flatten())
model.compile(loss="mean_squared_error", optimizer=sgd, metrics=["mae"])
hist = model.fit(X_train,
y_train,
epochs=1000,
verbose=0,
validation_split=0.2)
scores = model.evaluate(X_test, y_test)
print(pd.DataFrame(model.predict(X_test[0:5])).transpose())
print(y_test[0:5].iloc[0])
from tensorflow.keras.utils import plot_model
plot_model(model, to_file="/Users/athaker/Desktop/model.png")
def compile_model(network):
""" Compile a sequential model.
Args:
network (dict): the parameters of the network
Returns:
a compiled network.
Note the input shape here is considered to be (1, 5) and loss function is set to MSE.
Modify if necessary
"""
# Get the network parameters.
n_layers = network[
'n_layers'] # Note n_layers is the number of hidden layers.
layer_info = network['layer_info']
optimizer = network['optimizer']
final_act = network['final_act']
# Set the number of input and output features and time step.
input_features = 5
output_features = 1
time_step = 1
# Add input layer
inputs = Input(shape=(time_step, input_features))
# Add each layer
if n_layers == 0:
# If n_layers == 0, flatten and jump straight to the output layer.
hidden_layer = Reshape((input_features, ))(inputs)
elif n_layers > 0:
# If n_layers > 0, loop through layer_info.
for i in range(n_layers):
if i == 0:
# For the first hidden layer, specify the layer input as 'inputs'
if layer_info[i][0] == 'Dense':
hidden_layer = TimeDistributed(
Dense(layer_info[i][1],
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01),
use_bias=False))(inputs)
hidden_layer = Activation(layer_info[i][2])(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'LSTM':
hidden_layer = LSTM(layer_info[i][1],
return_sequences=True,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01),
use_bias=False)(inputs)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'GRU':
hidden_layer = GRU(layer_info[i][1],
return_sequences=True,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01),
use_bias=False)(inputs)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif i > 0:
# For the next hidden layers, simply add them along with the batch normalization and dropout.
if layer_info[i][0] == 'Dense':
hidden_layer = TimeDistributed(
Dense(layer_info[i][1],
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01)))(hidden_layer)
hidden_layer = Activation(layer_info[i][2])(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'LSTM':
hidden_layer = LSTM(
layer_info[i][1],
return_sequences=True,
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01))(hidden_layer)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
elif layer_info[i][0] == 'GRU':
hidden_layer = GRU(
layer_info[i][1],
return_sequences=True,
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01))(hidden_layer)
hidden_layer = Activation('tanh')(hidden_layer)
hidden_layer = BatchNormalization()(hidden_layer)
hidden_layer = Dropout(0.5)(hidden_layer)
# Add the flattening layer
hidden_layer = Flatten()(hidden_layer)
hidden_layer = Dense(output_features,
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.01))(hidden_layer)
outputs = Activation(final_act)(hidden_layer)
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss='mean_squared_error', optimizer=optimizer)
print(network_arch(network))
return model
X_test = np.load(os.path.join(os.getcwd(), 'data', 'LSTM_x_Test.npy'))
y_test = np.load(os.path.join(os.getcwd(), 'data', 'LSTM_y_Test.npy'))
#%%
# =============================================================================
# Build classification model - GRU
# =============================================================================
from tensorflow.keras.layers import GRU, Activation
model = Sequential()
model.add(
GRU(50,
input_shape=(X_train.shape[1], X_train.shape[2]),
return_sequences=True))
model.add(GRU(1, return_sequences=False))
model.add(Activation('sigmoid'))
model.summary()
adam = Adam(lr=learningRate)
model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
chk = ModelCheckpoint(os.path.join(os.getcwd(), 'data', 'best_model.hdf5'),
monitor='val_accuracy',
save_best_only=True,
mode='max',
verbose=1)
history = model.fit(X_train,
y_train,
epochs=150,
batch_size=128,
callbacks=[chk],
validation_data=(X_test, y_test),
pickle_in = open("y.pickle", "rb")
y = pickle.load(pickle_in)
X = X / 255.0
dense_layers = [0, 1, 2]
layer_sizes = [32, 64, 128]
conv_layers = [1, 2, 3]
print "Constructing the Convet...."
model = Sequential()
model.add(Conv2D(64, (3, 3), input_shape=X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
def model(self, freeze=True):
if self.base_model.lower() == 'vgg16':
base_model = VGG16(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
input_layer = base_model.input
layer = base_model.get_layer('block5_pool').output
elif self.base_model.lower() == 'vgg19':
base_model = VGG19(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
input_layer = base_model.input
layer = base_model.output
elif self.base_model.lower() == 'resnet50':
base_model = ResNet50(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
input_layer = base_model.input
layer = base_model.output
elif 'inception' in self.base_model.lower():
base_model = InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
input_layer = base_model.input
layer = base_model.output
elif self.base_model.lower() == 'mobilenet':
base_model = MobileNet(weights='imagenet',
include_top=False,
input_shape=self.img_shape,
pooling='avg')
input_layer = base_model.input
layer = base_model.output
else:
base_model = DenseNet121(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
input_layer = base_model.input
layer = base_model.output
if self.base_model.lower() == 'vgg16':
layer = Flatten()(layer)
layer = Dense(4096,
activation='relu',
name='fc1',
kernel_initializer='he_normal')(layer)
layer = Dropout(0.5)(layer)
layer = Dense(4096,
activation='relu',
name='fc2',
kernel_initializer='he_normal')(layer)
layer = Dropout(0.5)(layer)
prediction = Dense(self.cls_num,
activation='softmax',
name='predictions')(layer)
elif self.base_model.lower() == 'mobilenet':
# layer=GlobalAveragePooling2D()(layer)
layer = Reshape((1, 1, 1024), name='reshape_1')(layer)
layer = Dropout(1e-3)(layer)
layer = Conv2D(self.cls_num, (1, 1),
padding='same',
name='conv_preds')(layer)
layer = Activation('softmax')(layer)
prediction = Reshape((self.cls_num, ))(layer)
elif self.base_model.lower() == 'resnet50':
layer = GlobalAveragePooling2D()(layer)
prediction = Dense(self.cls_num,
activation='softmax',
name='predictions')(layer)
else:
layer = GlobalAveragePooling2D()(layer)
prediction = Dense(self.cls_num,
activation='softmax',
name='predictions')(layer)
if freeze == True:
for layer in base_model.layers:
layer.trainable = False
else:
for layer in base_model.layers:
layer.trainable = True
model_finetune = Model(input_layer, prediction)
return model_finetune