from keras import Input
from keras.engine import Model
from keras.initializers import RandomNormal
from keras.layers import Conv2D, BatchNormalization, Dense, Flatten, Dropout, PReLU
from util.clipweights import WeightClip
weight_init = RandomNormal(mean=0., stddev=0.02)
def Conv2DBatchNormPReLU(input_layer,
filters, kernel_size, strides=(1, 1), padding='valid'):
res = input_layer
res = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, activation=None, padding=padding,
kernel_initializer=weight_init,
kernel_constraint=WeightClip(-0.01, 0.01), bias_constraint=WeightClip(-0.01, 0.01))(res)
res = BatchNormalization()(res)
res = PReLU(shared_axes=[1, 2])(res)
return res
class Critic(Model):
def __init__(self, input_shape=(224, 224, 3), output_activation=None,
inputs=None, outputs=None, name='Critic'):
if inputs and outputs:
super(Critic, self).__init__(inputs=inputs, outputs=outputs, name=name)
return
''' VGG-like conv filters '''
input_image = Input(shape=input_shape)
x = input_image
def conv_block(self, x, filters, size, stride=(2,2),has_norm_instance=True,
padding='valid',
has_activation_layer=True,
use_leaky_relu=False):
x = Conv2D(filters, size, strides = stride, padding=padding, kernel_initializer=RandomNormal(0, 0.02))(x)
if has_norm_instance:
x = InstanceNormalization(axis=1)(x)
if has_activation_layer:
if use_leaky_relu:
x = LeakyReLU(alpha=0.2)(x)
else:
x = Activation('relu')(x)
return x
pass
def build_model():
model = Sequential()
# -------- build model -------- #
model = Sequential()
# Block 1
model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,64)), name='block1_conv1', input_shape=(30,30,60)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,64)), name='block1_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool'))
# Block 2
model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,128)), name='block2_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,128)), name='block2_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool'))
# Block 3
model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv3'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv4'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool'))
# Block 4
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv3'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv4'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool'))
# Block 5
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv3'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv4'))
model.add(BatchNormalization())
model.add(Activation('relu'))
# model modification for cifar-10
model.add(Flatten(name='flatten'))
model.add(Dense(4096, use_bias = True, kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = 0.01), name='fc_cifa10'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(dropout))
model.add(Dense(4096, kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = 0.01), name='fc2'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(dropout))
model.add(Dense(10, kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = 0.01), name='predictions_cifa10'))
model.add(Flatten())
model.add(Dense(1))
# optimizers should be tested
# sgd + momentum
# others
adam = optimizers.Adam(lr=0.0035, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6)
# sgd = optimizers.SGD(lr=0.005, momentum=0.9, decay=1e-6, nesterov=True)
# rms = optimizers.RMSprop(lr=0.0035, rho=0.9, epsilon=1e-08, decay=1e-6)
model.compile(optimizer=adam, loss='mse')
return model
def rigid_net(vol_size, enc_nf, dec_nf):
"""
architecture for rigid registration.
rigid registration: CNN output ND x ND+1 affine matrix, affine matrix is passed to affine_to_shift function
:param vol_size: volume size
:param enc_nf: list of encoder
:param dec_nf: list of decoder
:return: model
"""
unet_model = unet_core(vol_size, enc_nf, dec_nf, full_size=False)
[src, tgt] = unet_model.inputs
x_out = unet_model.outputs[-1]
# affine transform matrix
# build full connected layer into the model, output the ND x ND+1 affine matrix
flow = Conv3D(3,
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5),
name='flow')(x_out)
#flow = LeakyReLU(0.2)(flow)
flow1 = Lambda(reduce_dim1)(flow)
flow1 = Lambda(my_reshape)(flow1)
print("flow1's shape is: " + str(flow1.shape))
flow2 = Lambda(reduce_dim2)(flow)
flow2 = Lambda(my_reshape)(flow2)
flow3 = Lambda(reduce_dim3)(flow)
flow3 = Lambda(my_reshape)(flow3)
# add convolutinal layer into the model, which outputs affine matrix.
affine_matrix1 = Conv3D(filters=4,
kernel_size=(80, 96, 112),
padding='valid',
kernel_initializer=RandomNormal(mean=0.0,
stddev=1e-5),
name='flow1')(flow1)
affine_matrix1 = Reshape((1, 4))(affine_matrix1)
#affine_matrix1 = LeakyReLU(0.2)(affine_matrix1)
print("affine_matrix1's shape is: " + str(affine_matrix1.shape))
affine_matrix2 = Conv3D(filters=4,
kernel_size=(80, 96, 112),
padding='valid',
kernel_initializer=RandomNormal(mean=0.0,
stddev=1e-5),
name='flow2')(flow2)
affine_matrix2 = Reshape((1, 4))(affine_matrix2)
#affine_matrix2 = LeakyReLU(0.2)(affine_matrix2)
affine_matrix3 = Conv3D(filters=4,
kernel_size=(80, 96, 112),
padding='valid',
kernel_initializer=RandomNormal(mean=0.0,
stddev=1e-5),
name='flow3')(flow3)
affine_matrix3 = Reshape((1, 4))(affine_matrix3)
#affine_matrix3 = LeakyReLU(0.2)(affine_matrix3)
affine_matrix = concatenate([affine_matrix1, affine_matrix2])
affine_matrix = concatenate([affine_matrix, affine_matrix3])
affine_matrix = Lambda(reduce_dim4)(affine_matrix)
print("affine_matrix's shape is :" + str(affine_matrix.shape))
# spatial transform
y = nrn_layers.SpatialTransformer(interp_method='linear',
indexing='xy')([src, affine_matrix])
model = Model(inputs=[src, tgt], outputs=[y, flow])
print("the output's shape is:" + str(y.shape))
return model
def get_model(configuration='baseline'):
AUGMENTATION_CONFIGURATION = configuration
## Model
model = Sequential()
model.add(
Conv2D(32, (3, 3),
kernel_initializer='he_normal',
bias_initializer=RandomNormal(mean=0.1),
input_shape=(16, 16, 1)))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(64, (2, 2),
kernel_initializer='he_normal',
bias_initializer=RandomNormal(mean=0.1)))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(64, (2, 2),
kernel_initializer='he_normal',
bias_initializer=RandomNormal(mean=0.1)))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(
Conv2D(64, (2, 2),
kernel_initializer='he_normal',
bias_initializer=RandomNormal(mean=0.1)))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(
Conv2D(64, (1, 1),
kernel_initializer='he_normal',
bias_initializer=RandomNormal(mean=0.1)))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(Conv2D(1, (1, 1), activation='sigmoid'))
model.add(Flatten())
# For a binary classification problem
sgd = SGD(lr=0.0005, momentum=0.9)
model.compile(optimizer=sgd,
loss='binary_crossentropy',
metrics=['accuracy'])
## Data
h5_file_location = os.path.join('C:\\Users\Jeftha\stack\Rommel\ISMI\data',
'prostatex-train.hdf5')
h5_file = h5py.File(h5_file_location, 'r')
train_data_list, train_labels_list, attr = get_train_data(h5_file, ['ADC'])
windowed = []
for lesion in train_data_list:
windowed.append(apply_window(lesion, (500, 1100)))
train_data_list = np.asarray(windowed)
train_data, val_data, train_labels, val_labels = train_test_split(
train_data_list, train_labels_list, attr, test_size=0.33)
train_data = np.expand_dims(train_data, axis=-1)
val_data = np.expand_dims(val_data, axis=-1)
train_labels = np.expand_dims(train_labels, axis=-1)
val_labels = np.expand_dims(val_labels, axis=-1)
## Stuff for training
generator = get_generator(configuration=AUGMENTATION_CONFIGURATION)
train_generator = generator.flow(
train_data,
train_labels) #, save_to_dir="/nfs/home4/schellev/augmented_images")
batch_size = 128
steps_per_epoch = len(train_labels_list) // batch_size
auc_history = AucHistory(train_data,
train_labels,
val_data,
val_labels,
output_graph_name=AUGMENTATION_CONFIGURATION)
model.fit_generator(train_generator,
steps_per_epoch,
epochs=15000,
verbose=2,
callbacks=[auc_history],
max_q_size=50,
workers=8)
return model
def CNN_LSTM_STATEFUL_model(image_dim, audio_vector_dim):
(img_rows, img_cols, img_channels) = image_dim # (224,224,3)
input_img = Input(shape=(img_rows, img_cols, img_channels))
# Like Hanoi's work with DeepMind Reinforcement Learning, build a model that does not use pooling layers
# to retain sensitivty to locations of objects
# Tried (64,128,256,512)
x = Conv2D(filters=32,
kernel_size=(16, 16),
input_shape=image_dim,
name='Input_Layer',
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.01,
seed=None),
strides=(8, 8))(input_img)
x = Conv2D(filters=64,
kernel_size=(8, 8),
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.01,
seed=None),
strides=(4, 4))(x)
x = Conv2D(filters=128,
kernel_size=(4, 4),
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.01,
seed=None),
strides=(2, 2))(x)
x = Conv2D(filters=256,
kernel_size=(2, 2),
input_shape=image_dim,
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.01,
seed=None),
strides=(1, 1))(x)
x = Flatten()(x)
x = Dense(1024, activation='relu', name='1st_FC')(x)
x = Dropout(0.2)(x)
x = Dense(512, activation='relu', name='2nd_FC')(x)
# x = TimeDistributedDense(1)(x)
# Note that LSTM expects input shape: (nb_samples, timesteps, feature_dim)
x = Reshape((1, 512))(x)
x = LSTM(256, input_shape=(1, 512), dropout=0.2, return_sequences=True)(x)
x = Dropout(0.2)(x)
x = LSTM(128, dropout=0.2, name='LSTM_reg_output')(x)
network_output = Dense(audio_vector_dim)(x)
model = Model(inputs=input_img, outputs=network_output)
# Use the Adam optimizer for gradient descent
adam = Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model.compile(loss='mean_squared_error', optimizer='adam')
print(model.summary())
return model
def writers_adversarial_network(class_num, writers_num):
"""
Our WSAN
:param class_num: The total number of chinese character
:param writers_num:
:return:
"""
# init softmax
random_normal = RandomNormal(stddev=0.001, seed=1995)
reg = 1e-9
top5_acc = functools.partial(top_k_categorical_accuracy, k=5)
top5_acc.__name__ = 'top5_acc'
inputs = Input(shape=(64, 64, 1))
x = Conv2D(80, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(80, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.1)(x)
x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(76, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.1)(x)
x = Conv2D(152, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(152, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
gap1 = GlobalAveragePooling2D()(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.2)(x)
x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(192, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.2)(x)
x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
gap2 = GlobalAveragePooling2D()(x)
comb_gap = layers.Concatenate(axis=-1)([gap1, gap2])
source_classifier = Dropout(0.5)(comb_gap)
source_classifier = Dense(class_num, activation='softmax', name='CR',
kernel_initializer=random_normal, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(
source_classifier)
grl = GradientReversal(hp_lambda=0.01)(comb_gap) # grl
domain_classifier = Dropout(0.5)(grl)
domain_classifier = Dense(writers_num, activation='softmax', name='SR',
kernel_initializer=random_normal, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(
domain_classifier)
opt = optimizers.Adadelta(lr=0.3, rho=0.95, epsilon=None, decay=1e-5)
comb_model = Model(inputs=inputs, outputs=[source_classifier, domain_classifier])
# comb_model.summary()
# 设置多GPU
comb_model = multi_gpu_model(comb_model, gpus=2)
comb_model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy', top5_acc])
source_model = Model(inputs=inputs, outputs=[source_classifier])
source_model.compile(optimizer=opt, loss='categorical_crossentropy',
metrics=['accuracy', top5_acc])
domain_model = Model(inputs=inputs, outputs=[domain_classifier])
domain_model.compile(optimizer=opt, loss='categorical_crossentropy',
metrics=['accuracy'])
return comb_model, source_model, domain_model
def build_generator(img_shape, gf):
"""U-Net Generator"""
def conv3d(layer_input, filters, f_size=(4, 4, 4), bn=True):
d = create_convolution_block(input_layer=layer_input,
n_filters=filters,
batch_normalization=bn,
strides=(2, 2, 2),
kernel_size=f_size)
return d
def deconv3d(layer_input,
skip_input,
filters,
f_size=(4, 4, 4),
drop=True):
"""Layers used during upsampling"""
u = create_convolution_block_up(input_layer=layer_input,
skip_conn=skip_input,
n_filters=filters,
batch_normalization=True,
strides=(2, 2, 2),
kernel_size=f_size,
dropout=drop)
return u
# Image input
d0 = Input(batch_shape=img_shape)
# Downsampling
e1 = conv3d(d0, gf, bn=False) # 64
e2 = conv3d(e1, gf * 2) # 128
e3 = conv3d(e2, gf * 4) # 256
e4 = conv3d(e3, gf * 8) # 512
e5 = conv3d(e4, gf * 8) # 512
# bottleneck
e6 = bottleneck(e5,
gf * 8,
batch_normalization=False,
kernel_size=(4, 4, 4),
activation='relu',
padding='same',
strides=(2, 2, 2),
instance_normalization=False) # 512
# Upsampling
u1 = deconv3d(e6, e5, gf * 8, drop=True)
u2 = deconv3d(u1, e4, gf * 8, drop=True)
u3 = deconv3d(u2, e3, gf * 4, drop=True)
u4 = deconv3d(u3, e2, gf * 2, drop=False)
u5 = deconv3d(u4, e1, gf, drop=False)
#
#
init = RandomNormal(mean=0.0, stddev=0.02) # new
u6 = Conv3DTranspose(filters=gf,
kernel_size=(4, 4, 4),
padding='same',
kernel_initializer=init,
strides=(2, 2, 2))(u5)
#
final_convolution = Conv3D(1, (1, 1, 1))(u6)
act = Activation('relu')(final_convolution)
return Model(d0, act)
from generate_example import generate_random_example from create_model import create_model from loss_func import asymmetric_loss_generator from keras.initializers import RandomNormal from keras.optimizers import adam from keras.regularizers import l2 from keras.models import save_model import numpy as np from typing import List, Union from segment_img import read_img, predict_whole_img, visualize_prediction ROWS = 40 COLS = 40 CHANNELS = 3 K_I = RandomNormal(mean=0.0, stddev=0.2) B_I = RandomNormal(mean=0.0, stddev=0.2) K_R = l2(0.01) B_R = l2(0.01) KERNEL_SIZE = 3 STRIDE = 2 LAYERS = 1 CONV_ACTIVATION = 'tanh' CONNECT_ACTIVATION = 'tanh' BATCH_SIZE = 20 EPOCHS = 70 NEG_RATIO_POS = 100 LEARN_RATE = 0.001
def build_model():
model = Sequential()
model.add(
Conv2D(192, (5, 5),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.01),
input_shape=x_train.shape[1:],
activation='relu'))
model.add(
Conv2D(160, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(
Conv2D(96, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
model.add(Dropout(dropout))
model.add(
Conv2D(192, (5, 5),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(
Conv2D(192, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(
Conv2D(192, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
model.add(Dropout(dropout))
model.add(
Conv2D(192, (3, 3),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(
Conv2D(192, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(
Conv2D(10, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05),
activation='relu'))
model.add(GlobalAveragePooling2D())
model.add(Activation('softmax'))
sgd = optimizers.SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
return model
def createModel(input_sequence_dim, audio_vector_dim):
frame_h, frame_w, channels = input_sequence_dim # (80,80,1) we are going to pass in single greyscale images
# ConvLSTM expects an input with shape
# (n_frames, width, height, channels) In the example they used n_frames = 15
input_sequence = Input(shape=(None, frame_h, frame_w, channels))
# input to a ConvLSTM2D is of form (samples, time, filters, output_row, output_col)`
x = ConvLSTM2D(filters=32,
kernel_size=(8, 8),
padding='same',
strides=(4, 4),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
return_sequences=True)(input_sequence)
x = BatchNormalization()(x)
x = ConvLSTM2D(filters=64,
kernel_size=(4, 4),
padding='same',
strides=(2, 2),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
return_sequences=True)(x)
x = BatchNormalization()(x)
x = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
padding='same',
strides=(1, 1),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
return_sequences=True)(x)
x = BatchNormalization()(x)
'''
x = ConvLSTM2D(filters=64,
kernel_size=(2, 2),
padding='same',
strides=(1, 1),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
return_sequences=True)(x)
x = BatchNormalization()(x)
'''
# The Conv3D layer causes the the hidden "volumes" to become a flat image
# The 3D convolution changes a (None, None, 40, 40, 40) into a (None,None,40,40,40)
# Output of a Conv3D is (samples, filters, new_conv_dim1, new_conv_dim2, new_conv_dim3)
x = Conv3D(filters=1,
kernel_size=(3, 3, 3),
activation='sigmoid',
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
data_format='channels_last')(x)
'''
# Run a CNN over the "flat" image
x = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
padding='same',
activation='relu')(x)
'''
x = Flatten()(x)
# x = Dense(1024, activation='relu', name='1st_FC')(x)
# x = Dropout(0.5)(x)
x = Dense(512, activation='relu')(x)
network_output = Dense(audio_vector_dim, name='regression_out')(x)
model = Model(inputs=input_sequence, outputs=network_output)
# Use the Adam optimizer for gradient descent
adam = Adam(lr=0.5e-6)
model.compile(loss='mse', optimizer=adam)
def define_discriminator(image_shape):
# weight initialization
init = RandomNormal(stddev=0.02)
# source image input
in_src_image = Input(shape=image_shape)
# target image input
in_target_image = Input(shape=image_shape)
# concatenate images channel-wise
merged = Concatenate()([in_src_image, in_target_image])
# C64
d = Conv2D(filters=64,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=init)(merged)
d = LeakyReLU(alpha=0.2)(d)
# C128
d = Conv2D(filters=128,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
d = BatchNormalization()(d)
d = LeakyReLU(alpha=0.2)(d)
# C256
d = Conv2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
d = BatchNormalization()(d)
d = LeakyReLU(alpha=0.2)(d)
# C512
d = Conv2D(filters=512,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
d = BatchNormalization()(d)
d = LeakyReLU(alpha=0.2)(d)
# second last output layer
d = Conv2D(filters=512,
kernel_size=(4, 4),
padding='same',
kernel_initializer=init)(d)
d = BatchNormalization()(d)
d = LeakyReLU(alpha=0.2)(d)
# patch output
d = Conv2D(filters=1,
kernel_size=(4, 4),
padding='same',
kernel_initializer=init)(d)
patch_out = Activation('sigmoid')(d)
# define model
model = Model([in_src_image, in_target_image], patch_out)
# compile model
opt = Adam(lr=0.0002, beta_1=0.5)
model.compile(loss='binary_crossentropy',
optimizer=opt,
loss_weights=[0.5])
return model
def train(tr_features, tr_labels, ts_features, ts_labels):
# Hyperparamters
epochs = 2000
batch_size = 16
n_dim = tr_features.shape[1]
print(n_dim)
n_classes = len(cfg.classes)
n_hidden_units_one = 30 * n_dim
n_hidden_units_two = 20 * n_dim
n_hidden_units_three = 20 * n_dim
n_hidden_units_four = n_dim
n_hidden_units_five = n_dim // 2
n_hidden_units_six = n_dim // 4
sd = 1 / np.sqrt(n_dim)
lr = 0.0001
patience = 20
# Build and compile model
model = Sequential()
# model.add(Dropout(0.2))
model.add(
Dense(n_hidden_units_one,
input_dim=n_dim,
activation='sigmoid',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
model.add(Dropout(0.2))
model.add(
Dense(n_hidden_units_two,
activation='sigmoid',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
model.add(Dropout(0.2))
model.add(
Dense(n_hidden_units_three,
activation='sigmoid',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
model.add(Dropout(0.2))
model.add(
Dense(n_hidden_units_four,
activation='sigmoid',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
model.add(Dropout(0.2))
model.add(
Dense(n_hidden_units_five,
activation='sigmoid',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
model.add(Dropout(0.2))
model.add(
Dense(n_hidden_units_six,
activation='sigmoid',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
model.add(Dropout(0.2))
model.add(
Dense(n_classes,
activation='softmax',
kernel_initializer=RandomNormal(stddev=sd),
bias_initializer=RandomNormal(stddev=sd)))
adam = Adam(lr=lr)
# sgd =
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
# Callbacks
earlyStopping = EarlyStopping(monitor='val_acc',
patience=patience,
verbose=1)
weights_file = os.path.join(cfg.save_dir,
'weights.{epoch:02d}-{val_acc:.2f}.hdf5')
modelCheckpt = ModelCheckpoint(weights_file,
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=True)
tsboard = TensorBoard(log_dir=os.path.join(cfg.save_dir, 'logs'),
batch_size=batch_size)
# model.load_weights("/data/juliej/results/weights.34-0.81.hdf5")
# Train
model.fit(
tr_features,
tr_labels,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(ts_features, ts_labels),
shuffle=True,
# initial_epoch=35,
callbacks=[earlyStopping, modelCheckpt, tsboard])
def __init__(self,
n_clusters,
input_dim,
encoded=None,
decoded=None,
alpha=1.0,
pretrained_weights=None,
cluster_centres=None,
batch_size=256,
**kwargs):
super(DeepEmbeddingClustering, self).__init__()
self.n_clusters = n_clusters
self.input_dim = input_dim
self.encoded = encoded
self.decoded = decoded
self.alpha = alpha
self.pretrained_weights = pretrained_weights
self.cluster_centres = cluster_centres
self.batch_size = batch_size
self.learning_rate = 0.1
self.iters_lr_update = 20000
self.lr_change_rate = 0.1
# greedy layer-wise training before end-to-end training:
self.encoders_dims = [self.input_dim, 500, 500, 2000, 10]
self.input_layer = Input(shape=(self.input_dim, ), name='input')
dropout_fraction = 0.2
init_stddev = 0.01
self.layer_wise_autoencoders = []
self.encoders = []
self.decoders = []
for i in range(1, len(self.encoders_dims)):
encoder_activation = 'linear' if i == (len(self.encoders_dims) -
1) else 'relu'
encoder = Dense(self.encoders_dims[i],
activation=encoder_activation,
input_shape=(self.encoders_dims[i - 1], ),
kernel_initializer=RandomNormal(mean=0.0,
stddev=init_stddev,
seed=None),
bias_initializer='zeros',
name='encoder_dense_%d' % i)
self.encoders.append(encoder)
decoder_index = len(self.encoders_dims) - i
decoder_activation = 'linear' if i == 1 else 'relu'
decoder = Dense(self.encoders_dims[i - 1],
activation=decoder_activation,
kernel_initializer=RandomNormal(mean=0.0,
stddev=init_stddev,
seed=None),
bias_initializer='zeros',
name='decoder_dense_%d' % decoder_index)
self.decoders.append(decoder)
autoencoder = Sequential([
Dropout(dropout_fraction,
input_shape=(self.encoders_dims[i - 1], ),
name='encoder_dropout_%d' % i), encoder,
Dropout(dropout_fraction,
name='decoder_dropout_%d' % decoder_index), decoder
])
autoencoder.compile(loss='mse',
optimizer=SGD(lr=self.learning_rate,
decay=0,
momentum=0.9))
self.layer_wise_autoencoders.append(autoencoder)
# build the end-to-end autoencoder for finetuning
# Note that at this point dropout is discarded
self.encoder = Sequential(self.encoders)
self.encoder.compile(loss='mse',
optimizer=SGD(lr=self.learning_rate,
decay=0,
momentum=0.9))
self.decoders.reverse()
self.autoencoder = Sequential(self.encoders + self.decoders)
self.autoencoder.compile(loss='mse',
optimizer=SGD(lr=self.learning_rate,
decay=0,
momentum=0.9))
if cluster_centres is not None:
assert cluster_centres.shape[0] == self.n_clusters
assert cluster_centres.shape[1] == self.encoder.layers[
-1].output_dim
if self.pretrained_weights is not None:
self.autoencoder.load_weights(self.pretrained_weights)
def _build(self):
self.int_dim = self.config['int_features']
self.float_dim = self.config['float_features']
self.feature_sizes = self.config['int_feature_sizes']
self.embed_size = self.config['embed_size']
self.total_feature_size = sum(self.feature_sizes) + self.float_dim
self.all_dim = self.int_dim + self.float_dim
##################################################################
# Begin Define shared layers
##################################################################
self.input_embedding_layer = Lambda(self.build_embedding_input)
self.new_input_layer = Lambda(self.build_new_input)
# Add Embedding For all features
# [None, all_dim, emb_size]
from keras.initializers import RandomNormal
self.V_embedding_layer = Embedding(
self.total_feature_size,
self.embed_size,
embeddings_initializer=RandomNormal(stddev=0.001))
# [None, all_dim, 1]
self.x_embedding_layer = Embedding(
self.total_feature_size,
1,
embeddings_initializer=RandomNormal(stddev=0.001))
# FM Part
self.FM_layer = Lambda(function=self.FM_part)
self.dropout_layer1 = Dropout(self.config['input_dropout'])
#y_FM = self.FM_part(x_embedding, V_embedding, new_input)
self.GLU_layers = []
for i in range(self.config['hidden_layers']):
dense = Dense(self.config['hidden_units'])
gate = Dense(self.config['hidden_units'], activation='sigmoid')
self.GLU_layers.append((dense, gate))
self.dense_DNN_layer = Dense(1, activation="elu")
self.concate_layer = Lambda(lambda x: K.concatenate(
[K.reshape(x[0], (-1, 1)),
K.reshape(x[1], (-1, 1))], axis=1))
self.dense_concat_layer = Dense(1, activation="sigmoid")
##################################################################
# End Define shared layers
##################################################################
##################################################################
# Build Predict model
##################################################################
input = Input(shape=(self.all_dim, ))
input_embedding = self.input_embedding_layer(input)
new_input = self.new_input_layer(input)
# Add Embedding For all features
# [None, all_dim, emb_size]
V_embedding = self.V_embedding_layer(input_embedding)
# [None, all_dim, 1]
x_embedding = self.x_embedding_layer(input_embedding)
# FM Part
y_FM = self.FM_layer([x_embedding, V_embedding, new_input])
# DNN Part
flatten = Flatten()(V_embedding)
XF = self.dropout_layer1(flatten)
for i in range(self.config['hidden_layers']):
#GLU
dense, gate = self.GLU_layers[i]
XF = multiply([dense(XF), gate(XF)]) #以上三步构成了所谓的GLU激活函数
#XF = dense(XF)
#Normal dense
#input = Dense(self.config['hidden_units'],activation=self.config['hidden_activation'])(input)
#input = Dropout(self.config['hidden_dropout'])(input)
y_DNN = self.dense_DNN_layer(XF)
concated = self.concate_layer([y_DNN, y_FM])
score = self.dense_concat_layer(concated)
self.predict_model = Model(inputs=input, outputs=score)
self.predict_model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy', auc])
self.predict_model.summary()
##################################################################
# Build Rank model
##################################################################
lr = self.config['learning_rate']
lr = 1e-4
print("learning_rate=%f" % (lr))
optimizer = Adam(lr)
optimizer = SGD(lr, momentum=0.95, decay=0.99)
pos_input = Input(shape=(self.all_dim, ), name="Pos_Inp")
neg_input = Input(shape=(self.all_dim, ), name="Neg_Inp")
pos_score = self.predict_model(pos_input)
neg_score = self.predict_model(neg_input)
margin = 0.6
loss = Lambda(lambda x: K.relu(margin - x[0] + x[1]),
name="loss")([pos_score, neg_score])
#loss = Lambda(lambda x : K.sigmoid(-x[0] + x[1]),name="loss")([pos_score,neg_score])
"""
train_model = Model(inputs=[pos_input,neg_input],outputs=[pos_score,neg_score,loss])
train_model.compile(optimizer=optimizer,
loss=['binary_crossentropy','binary_crossentropy',lambda y_true, y_pred: y_pred],
loss_weights=[5.0,1.0,10.0],
metrics={'loss':'binary_crossentropy'})
"""
train_model = Model(inputs=[pos_input, neg_input], outputs=loss)
train_model.compile(
optimizer=optimizer,
loss=lambda y_true, y_pred: y_pred,
)
self.model = train_model
self.model.summary()
plot_model(self.model, "deepfmrank_train_model.png")
plot_model(self.predict_model, "deepfmrank_predict_model.png")
max_radius = max_radius,
noise_treatment = noise_treatment,
weights = weights,
scaling_weights = scaling_weights)
##------------------------------ Model ---------------------------------------##
# Create
model = Sequential()
# Add layers
model.add(
Conv3D(filters=32,
kernel_size=9,
strides=2,
padding='valid',
kernel_initializer=RandomNormal(mean=0.0,
stddev=stddev_conv3d * 9**(-3 / 2)),
bias_initializer='zeros',
kernel_regularizer=l2(0.001),
bias_regularizer=None,
input_shape=(v_size, v_size, v_size, n_channels)))
model.add(LeakyReLU(alpha=0.1))
model.add(Dropout(rate=0.2))
model.add(
Conv3D(filters=64,
kernel_size=5,
strides=1,
padding='valid',
kernel_initializer=RandomNormal(mean=0.0,
def RRDB1(inp: Union[Layer, tf.Tensor],
filters: int = 64,
kernel_size: int = 3,
use_bias: bool = True,
use_sn: bool = False,
kernel_initializer: Initializer = RandomNormal(stddev=0.02)):
ConvLayer = ConvSN2D if use_sn else Conv2D
def dense_block(inp):
x1 = ConvLayer(filters,
kernel_size=kernel_size,
strides=1,
padding='same',
use_bias=use_bias,
kernel_initializer=kernel_initializer)(inp)
x1 = LeakyReLU(0.2)(x1)
x1 = Concatenate()([inp, x1])
x2 = ConvLayer(filters,
kernel_size=kernel_size,
strides=1,
padding='same',
use_bias=use_bias,
kernel_initializer=kernel_initializer)(x1)
x2 = LeakyReLU(0.2)(x2)
x2 = Concatenate()([inp, x1, x2])
x3 = ConvLayer(filters,
kernel_size=kernel_size,
strides=1,
padding='same',
use_bias=use_bias,
kernel_initializer=kernel_initializer)(x2)
x3 = LeakyReLU(0.2)(x3)
x3 = Concatenate()([inp, x1, x2, x3])
x4 = ConvLayer(filters,
kernel_size=kernel_size,
strides=1,
padding='same',
use_bias=use_bias,
kernel_initializer=kernel_initializer)(x3)
x4 = LeakyReLU(0.2)(x4)
x4 = Concatenate()([inp, x1, x2, x3, x4])
x5 = ConvLayer(filters,
kernel_size=kernel_size,
strides=1,
padding='same',
use_bias=use_bias,
kernel_initializer=kernel_initializer)(x4)
x5 = Lambda(lambda x: x * 0.2)(x5)
x = Add()([x5, inp])
return x
x = dense_block(inp)
x = dense_block(x)
x = dense_block(x)
x = Lambda(lambda x: x * 0.2)(x)
out = Add()([x, inp])
return out
from data import _seed
import numpy as np
np.random.seed(_seed)
import sys
from keras.layers import Dense, merge, Dropout, RepeatVector, Embedding
from keras.layers import recurrent, Input, Activation
from keras.models import Model
from keras.initializers import RandomNormal, Orthogonal
from experiment import evaluate_model, handlesysargs
from model import model as modelwrapper
print("MTL model sharing everything but output, based on gru")
INITIALIZER = RandomNormal(mean=0.0, stddev=0.05, seed=_seed)
RINITIALIZER = Orthogonal(gain=1.0, seed=_seed)
RNN = recurrent.GRU
EMBED_HIDDEN_SIZE = 30 #50
SENT_HIDDEN_SIZE = 100
QUERY_HIDDEN_SIZE = 100
def compile_model(inputs, repeat):
(vocab_size, story_maxlen, query_maxlen) = inputs[0]
#(vocab_size2, story_maxlen2, query_maxlen2) = inputs[1]
#mvocab_size = vocab_size1
#if (vocab_size2 > mvocab_size):
# mvocab_size = vocab_size2
ensinput = Input((story_maxlen, ))
sentemb = Embedding(vocab_size,
EMBED_HIDDEN_SIZE,
def CNN_LSTM_model(image_dim, audio_vector_dim, learning_rate,
weight_init): #(224,224,3)timespace-imageセット3枚
(
img_rows, img_cols, img_channels
) = image_dim # (224,224,3)*hoge=extract_cont_TopAngles.pyで最後に繋げた長さがtraining_data数
input_img = Input(shape=(img_rows, img_cols, img_channels))
# Like Hanoi's work with DeepMind Reinforcement Learning, build a model that does not use pooling layers
# to retain sensitivty to locations of objects
#物体の位置に関する繊細さを得るためにこのモデルはプーリング層を使わないで行った。
#CNN layers that increase in filter number, decrease in filter size
# and decrease in filter stride. The authors reasoned
# that such a design pattern enables the CNN to be
# sensitive to the location of small details
# Tried (64,128,256,512)
x = Conv2D(
filters=32,
kernel_size=(16, 16),
input_shape=image_dim,
name='Input_Layer',
activation='relu',
padding='same',
kernel_initializer=RandomNormal(
mean=0.0, stddev=weight_init,
seed=None), #正規分布に従って重みを初期化,stddev=weight_init = 0.01 分布の標準偏差
strides=(8, 8))(input_img)
x = Conv2D(filters=64,
kernel_size=(8, 8),
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=weight_init,
seed=None),
strides=(4, 4))(x)
x = Conv2D(filters=128,
kernel_size=(4, 4),
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=weight_init,
seed=None),
strides=(2, 2))(x)
x = Conv2D(filters=256,
kernel_size=(2, 2),
input_shape=image_dim,
activation='relu',
padding='same',
kernel_initializer=RandomNormal(mean=0.0,
stddev=weight_init,
seed=None),
strides=(1, 1))(x)
x = Flatten()(x)
x = Dense(1024,
activation='relu',
name='1st_FC',
kernel_initializer=RandomNormal(mean=0.0,
stddev=weight_init,
seed=None))(x)
x = Dropout(0.2)(x)
x = Dense(512,
activation='relu',
name='2nd_FC',
kernel_initializer=RandomNormal(mean=0.0,
stddev=weight_init,
seed=None))(x)
# x = TimeDistributedDense(1)(x)
# Note that LSTM expects input shape: (nb_samples, timesteps, feature_dim)
x = Reshape((1, 512))(x)
x = LSTM(256, input_shape=(1, 512), dropout=0.2, return_sequences=True)(x)
x = Dropout(0.2)(x)
x = LSTM(128, dropout=0.2, name='LSTM_reg_output')(x)
#network_output = Dense(audio_vector_dim)(x)#最後にオーディオデータの次元数にあわせる
network_output = Dense(10)(x) #最後にオーディオデータの次元数にあわせる
model = Model(inputs=input_img, outputs=network_output)
# Use the Adam optimizer for gradient descent
adam = Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
#sgd = SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False)
print("learning rate:", learning_rate)
model.compile(loss='mean_squared_error', optimizer='adam')
print(model.summary())
return model
print(x_test.shape[0], 'test samples')
# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
m = 0
sd = 0.05
#use_bias = True, bias_initializer = RandomNormal(mean=0.05, stddev=0.05, seed=None),
model.add(
Conv2D(192, (5, 5),
padding='same',
use_bias=True,
bias_initializer=RandomNormal(mean=m, stddev=sd, seed=None),
input_shape=x_train.shape[1:]))
model.add(Activation('relu'))
model.add(
Conv2D(160, (1, 1),
use_bias=True,
bias_initializer=RandomNormal(mean=m, stddev=sd, seed=None)))
model.add(Activation('relu'))
model.add(
Conv2D(96, (1, 1),
use_bias=True,
bias_initializer=RandomNormal(mean=m, stddev=sd, seed=None)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=2))
model.add(Dropout(0.5))
import keras.backend as K import tensorflow as tf import numpy as np from keras.layers import Input, Reshape, Flatten, Dense, BatchNormalization, PReLU from keras.layers import Activation, Add, Lambda, AveragePooling2D, LeakyReLU, GlobalAvgPool2D from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Model from keras.initializers import RandomNormal, glorot_uniform conv_init = RandomNormal(0, 0.02) # def SubpixelConv2D(input_shape, scale=2): # # # Copyright (c) 2017 Jerry Liu # # Released under the MIT license # # https://github.com/twairball/keras-subpixel-conv/blob/master/LICENSE # # def subpixel_shape(input_shape): # dims = [input_shape[0], # input_shape[1] * scale, # input_shape[2] * scale, # int(input_shape[3] / (scale ** 2))] # output_shape = tuple(dims) # return output_shape # # def subpixel(x): # return tf.depth_to_space(x, scale) # # return Lambda(subpixel, output_shape=subpixel_shape) def Resblock_generator(layer_input, channels):
def __init__(self,
BiGRU_rand=True,
STATIC=False,
ExternalEmbeddingModel=None,
n_symbols=None,
wordmap=None):
self.embedding_dim = 300
self.hidden_dims = 100
self.dropout_prob = (0.5, 0.8)
self.loss = 'categorical_crossentropy'
self.optimizer = 'rmsprop'
self.l1_reg = 0
self.l2_reg = 3 ##according to kim14
self.std = 0.05 ## standard deviation
# Training Parameters
self.set_training_paramters(batch_size=64, num_epochs=10)
self.set_processing_parameters(
sequence_length=30, vocab_size=50000) ## changed to fit short text
# Defining Model Layers if clstm_rand:
##Embedding Layer Randomly initialized
if BiGRU_rand:
##Embedding Layer Randomly initialized
embedding_layer = Embedding(output_dim=self.embedding_dim,
input_dim=self.vocab_size)
Classes = dh.read_labels()
n_classes = len(Classes)
else:
## Use pretrained model
#n_symbols, wordmap = dh.get_word_map_num_symbols()
self.set_etxrernal_embedding(ExternalEmbeddingModel)
vecDic = dh.GetVecDicFromGensim(self.ExternalEmbeddingModel)
Classes = dh.read_labels()
n_classes = len(Classes)
## Define Embedding Layer
embedding_weights = dh.GetEmbeddingWeights(
embedding_dim=self.embedding_dim,
n_symbols=n_symbols,
wordmap=wordmap,
vecDic=vecDic)
embedding_layer = Embedding(output_dim=self.embedding_dim,
input_dim=n_symbols,
trainable=STATIC)
embedding_layer.build(
(None, )) # if you don't do this, the next step won't work
embedding_layer.set_weights([embedding_weights])
Sequence_in = Input(shape=(self.sequence_length, ), dtype='int32')
embedding_seq = embedding_layer(Sequence_in)
x = Dropout(self.dropout_prob[0])(embedding_seq)
x = Bidirectional(
GRU(self.hidden_dims,
kernel_initializer=RandomNormal(stddev=self.std),
kernel_regularizer=L1L2(l1=self.l1_reg, l2=self.l2_reg),
return_sequences=False))(x)
preds = Dense(n_classes, activation='softmax')(x)
## return graph model
model = Model(Sequence_in, preds)
model.compile(loss=self.loss,
optimizer=self.optimizer,
metrics=['accuracy'])
self.model = model
def miccai2018_net(vol_size,
enc_nf,
dec_nf,
use_miccai_int=True,
int_steps=7,
indexing='xy'):
"""
architecture for probabilistic diffeomoprhic VoxelMorph presented in the MICCAI 2018 paper.
You may need to modify this code (e.g., number of layers) to suit your project needs.
The stationary velocity field operates in a space (0.5)^3 of vol_size for computational reasons.
:param vol_size: volume size. e.g. (256, 256, 256)
:param enc_nf: list of encoder filters. right now it needs to be 1x4.
e.g. [16,32,32,32]
:param dec_nf: list of decoder filters. right now it must be 1x6, see unet function.
:param use_miccai_int: whether to use the manual miccai implementation of scaling and squaring integration
note that the 'velocity' field outputted in that case was
since then we've updated the code to be part of a flexible layer. see neuron.layers.VecInt
:param int_steps: the number of integration steps
:param indexing: xy or ij indexing. we recommend ij indexing if training from scratch.
miccai 2018 runs were done with xy indexing.
:return: the keras model
"""
# get unet
unet_model = unet_core(vol_size, enc_nf, dec_nf, full_size=False)
[src, tgt] = unet_model.inputs
x_out = unet_model.outputs[-1]
# velocity mean and logsigma layers
flow_mean = Conv3D(3,
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5),
name='flow')(x_out)
flow_log_sigma = Conv3D(
3,
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-10),
bias_initializer=keras.initializers.Constant(value=-10),
name='log_sigma')(x_out)
flow_params = concatenate([flow_mean, flow_log_sigma])
# velocity sample
flow = Lambda(sample, name="z_sample")([flow_mean, flow_log_sigma])
# integrate if diffeomorphic (i.e. treating 'flow' above as stationary velocity field)
if use_miccai_int:
# for the miccai2018 submission, the scaling and squaring layer
# was manually composed of a Transform and and Add Layer.
flow = Lambda(lambda x: x,
name='flow-fix')(flow) # remanant of old code
v = flow
for _ in range(int_steps):
v1 = nrn_layers.SpatialTransformer(interp_method='linear',
indexing=indexing)([v, v])
v = keras.layers.add([v, v1])
flow = v
else:
# new implementation in neuron is cleaner.
# the 2**int_steps is a correcting factor left over from the miccai implementation.
# * (2**int_steps)
flow = Lambda(lambda x: x, name='flow-fix')(flow)
flow = nrn_layers.VecInt(method='ss', name='flow-int',
int_steps=7)(flow)
# get up to final resolution
flow = Lambda(interp_upsampling,
output_shape=vol_size + (3, ),
name='pre_diffflow')(flow)
flow = Lambda(lambda arg: arg * 2, name='diffflow')(flow)
# transform
y = nrn_layers.SpatialTransformer(interp_method='linear',
indexing=indexing)([src, flow])
# prepare outputs and losses
outputs = [y, flow_params]
# build the model
return Model(inputs=[src, tgt], outputs=outputs)
def __init__(self,
att_rand=True,
ExternalEmbeddingModel=None,
n_symbols=None,
wordmap=None,
STATIC=True):
self.dropout_prob = (0.36, 0.36)
self.hidden_dims = 100
self.std = 0.05
self.l1_reg = 3
self.l2_reg = 3
self.loss = 'categorical_crossentropy'
self.optimizer = 'rmsprop'
self.sequence_length = 30
self.embedding_dim = 300
self.vocab_size = 50000
self.num_epochs = 5
self.batch_size = 64
## Define the BiGRU model
if att_rand:
##Embedding Layer Randomly initialized
embedding_layer = Embedding(output_dim=self.embedding_dim,
input_dim=self.vocab_size)
else:
## Use pretrained model
#n_symbols, wordmap = dh.get_word_map_num_symbols()
self.set_etxrernal_embedding(ExternalEmbeddingModel)
vecDic = dh.GetVecDicFromGensim(self.ExternalEmbeddingModel)
## Define Embedding Layer
embedding_weights = dh.GetEmbeddingWeights(
embedding_dim=self.embedding_dim,
n_symbols=n_symbols,
wordmap=wordmap,
vecDic=vecDic)
embedding_layer = Embedding(output_dim=self.embedding_dim,
input_dim=n_symbols,
trainable=STATIC)
embedding_layer.build(
(None, )) # if you don't do this, the next step won't work
embedding_layer.set_weights([embedding_weights])
###################################################
SeqIn = Input(shape=(self.sequence_length, ), dtype='int32')
embedding_seq = embedding_layer(SeqIn)
M1 = Dropout(self.dropout_prob[0])(embedding_seq)
#M1 = Activation('tanh')(embedding_seq)
activations = Bidirectional(
GRU(self.hidden_dims,
kernel_initializer=RandomNormal(stddev=self.std),
kernel_regularizer=L1L2(l1=self.l1_reg, l2=self.l2_reg),
return_sequences=True))(M1)
## Timedistributed dense for each activation
attention = TimeDistributed(Dense(1, activation='tanh'))(activations)
attention = Flatten()(attention)
attention = Activation('softmax')(attention)
attention = RepeatVector(2 * (self.hidden_dims))(attention)
attention = Permute([2, 1])(attention)
# apply the attention
sent_representation = merge([activations, attention], mode='mul')
sent_representation = Lambda(lambda xin: K.sum(xin, axis=1))(
sent_representation)
Classes = dh.read_labels()
n_classes = len(Classes)
preds = Dense(n_classes, activation='softmax')(sent_representation)
model = Model(input=SeqIn, output=preds)
model.compile(optimizer=self.optimizer,
loss=self.loss,
metrics=['accuracy'])
self.model = model
other_t: features
}) # NUM_CLUSTER * num images
# select reliable images
reliable_image_idx = np.unique(np.argwhere(similarities > LAMBDA)[:, 1])
print('ckpt %d: # reliable images %d' % (ckpt, len(reliable_image_idx)))
sys.stdout.flush()
images = np.array(labeled_images +
[unlabeled_images[i][0] for i in reliable_image_idx])
labels = to_categorical(
known_labels + [kmeans.labels_[i] + KNOWN for i in reliable_image_idx])
# retrain: fine tune
init_model = load_model('checkpoint/0.ckpt')
x = init_model.get_layer('avg_pool').output
x = Flatten(name='flatten')(x)
x = Dropout(0.5)(x)
x = Dense(NUM_IDS,
activation='softmax',
name='fc8',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(x)
net = Model(input=init_model.input, output=x)
for layer in net.layers:
layer.trainable = True
net.compile(optimizer=SGD(lr=0.001, momentum=0.9),
loss='categorical_crossentropy')
net.fit_generator(datagen.flow(images, labels, batch_size=BATCH_SIZE),
steps_per_epoch=len(images) / BATCH_SIZE + 1,
epochs=NUM_EPOCH)
net.save('checkpoint/%d.ckpt' % ckpt)
def __init__(self,
cnn_rand=True,
STATIC=False,
ExternalEmbeddingModel=None,
n_symbols=None,
wordmap=None):
# Model hyperparameters
self.embedding_dim = 300 ##
self.filter_sizes = (3, 8)
self.num_filters = 10
self.hidden_dims = 100
self.dropout_prob = (0.5, 0.8)
self.loss = 'categorical_crossentropy'
self.optimizer = 'rmsprop'
self.l1_reg = 0
self.l2_reg = 3 ##according to kim14
self.std = 0.05 ## standard deviation
# Training Parameters
self.set_training_paramters(batch_size=64, num_epochs=10)
self.set_processing_parameters(
sequence_length=100,
vocab_size=vocabsize) ## changed to fit short text
# Defining Model Layers
if cnn_rand:
##Embedding Layer Randomly initialized
embedding_layer = Embedding(output_dim=self.embedding_dim,
input_dim=self.vocab_size)
Classes = dh.read_labels()
n_classes = len(Classes)
else:
## Use pretrained model
#n_symbols, wordmap = dh.get_word_map_num_symbols()
self.set_etxrernal_embedding(ExternalEmbeddingModel)
vecDic = dh.GetVecDicFromGensim(self.ExternalEmbeddingModel)
Classes = dh.read_labels()
n_classes = len(Classes)
## Define Embedding Layer
embedding_weights = dh.GetEmbeddingWeights(embedding_dim=300,
n_symbols=n_symbols,
wordmap=wordmap,
vecDic=vecDic)
embedding_layer = Embedding(output_dim=self.embedding_dim,
input_dim=n_symbols,
trainable=STATIC)
embedding_layer.build(
(None, )) # if you don't do this, the next step won't work
embedding_layer.set_weights([embedding_weights])
Sequence_in = Input(shape=(self.sequence_length, ), dtype='int32')
embedding_seq = embedding_layer(Sequence_in)
x = Dropout(self.dropout_prob[0])(embedding_seq)
## define Core Convultional Layers
conv_blocks = []
for sz in self.filter_sizes:
conv = Convolution1D(filters=self.num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(x)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
x = Concatenate()(
conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
x = Dropout(self.dropout_prob[1])(x)
x = Dense(self.hidden_dims,
activation="relu",
kernel_initializer=RandomNormal(stddev=self.std),
kernel_regularizer=L1L2(l1=self.l1_reg, l2=self.l2_reg))(x)
preds = Dense(n_classes, activation='softmax')(x)
## return graph model
model = Model(Sequence_in, preds)
model.compile(loss=self.loss,
optimizer=self.optimizer,
metrics=['accuracy'])
self.model = model
def deconv_block(self, x, filters, size ):
x = Conv2DTranspose(filters, kernel_size=size, strides=2, padding='same', use_bias=False, kernel_initializer=RandomNormal(0, 0.02))(x)
x = InstanceNormalization(axis=1)(x)
x = Activation('relu')(x)
return x
pass
def create_base_network_vgg(input_shape):
'''Base network to be shared (eq. to feature extraction).
'''
model = Input(shape=input_shape)
x = Conv2D(16, (3, 3),
padding="same",
activation='relu',
data_format='channels_first',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(model)
x = Conv2D(16, (3, 3),
padding="same",
activation='relu',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
#x = Dropout(0.2)(x)
x = Conv2D(32, (3, 3),
padding="same",
activation='relu',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(x)
x = Conv2D(32, (3, 3),
padding="same",
activation='relu',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2))(x) #if stride==None default to pool_size
#x = Dropout(0.5)(x)
x = Conv2D(64, (3, 3),
padding="same",
activation='relu',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(x)
x = Conv2D(64, (3, 3),
padding="same",
activation='relu',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(x)
x = Conv2D(64, (3, 3),
padding="same",
activation='relu',
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed),
bias_initializer=RandomNormal(mean=0.0,
stddev=0.05,
seed=random_seed))(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2))(x) #if stride==None default to pool_size
x = Flatten()(x)
#x = Dense(500, activation='relu')(x)
x = Dense(
2048,
kernel_initializer=keras.initializers.lecun_normal(seed=random_seed),
activation='relu')(x)
#x = Dropout(0.1)(x)
x = Dense(
2048,
kernel_initializer=keras.initializers.lecun_normal(seed=random_seed),
activation='relu')(x)
#x = Dropout(0.1)(x)
x = Dense(
2048,
kernel_initializer=keras.initializers.lecun_normal(seed=random_seed),
activation='relu')(x)
#x = Dropout(0.1)(x)
#model = Dense(num_classes)(model)
return Model(model, x)
from keras.activations import relu
from keras.initializers import RandomNormal
from keras.optimizers import RMSprop, SGD, Adam
K.set_image_data_format('channels_last')
channel_axis = -1
channel_first = False
def __conv_init(a):
print("conv_init", a)
k = RandomNormal(0, 0.02)(a) # for convolution kernel
k.conv_weight = True
return k
conv_init = RandomNormal(0, 0.02)
gamma_init = RandomNormal(1., 0.02) # for batch normalization
# Basic discriminator
def conv2d(f, *a, **k):
return Conv2D(f, kernel_initializer=conv_init, *a, **k)
def batchnorm():
return BatchNormalization(momentum=0.9,
axis=channel_axis,
epsilon=1.01e-5,
gamma_initializer=gamma_init)
def __conv_init(a):
print("conv_init", a)
k = RandomNormal(0, 0.02)(a) # for convolution kernel
k.conv_weight = True
return k
