from keras.optimizers import SGD,Adam
from keras.losses import binary_crossentropy
from keras.callbacks import EarlyStopping, TensorBoard, LearningRateScheduler, ModelCheckpoint
import numpy.random as rng
import para
import random
import copy
import time
import numpy as np
import tensorflow as tf
import keras
import generator
from keras.initializers import RandomNormal
convnet = Sequential()
convnet.add(Conv2D(32,(5,5),activation='relu',input_shape=para.input_shape3, kernel_initializer=RandomNormal(0,1e-2)
,kernel_regularizer=l2(2e-4),bias_initializer=RandomNormal(0.5,1e-2)))
convnet.add(MaxPooling2D())
convnet.add(SpatialDropout2D(0.12))
convnet.add(Conv2D(64,(3,3),activation='relu', kernel_initializer=RandomNormal(0,1e-2)
,kernel_regularizer=l2(2e-4),bias_initializer=RandomNormal(0.5,1e-2)))
convnet.add(MaxPooling2D())
convnet.add(SpatialDropout2D(0.12))
convnet.add(Conv2D(128,(2,2),activation='relu', kernel_initializer=RandomNormal(0,1e-2)
,kernel_regularizer=l2(2e-4),bias_initializer=RandomNormal(0.5,1e-2)))
convnet.add(Flatten())
import numpy as np
import keras
from keras.initializers import RandomNormal
from keras.layers import Dense, LeakyReLU
from keras.regularizers import l2
import chess_environment.chessboard as cb
from dqn_tools.memory import SimpleMemory
from dqn_tools.trainers import DQNTrainer, load_trainer
from training_tools import DQNChessRecord
seed = 12345
np.random.seed(seed)
# temporary simple model for testing base concept
weight_decay = l2(1e-2)
weight_initializer = RandomNormal(mean=0., stddev=0.02, seed=seed)
model = keras.Sequential([
Dense(150,
input_shape=(384, ),
kernel_initializer=weight_initializer,
bias_initializer=weight_initializer,
kernel_regularizer=weight_decay,
bias_regularizer=weight_decay),
LeakyReLU(alpha=0.3),
Dense(300,
kernel_initializer=weight_initializer,
bias_initializer=weight_initializer,
kernel_regularizer=weight_decay,
bias_regularizer=weight_decay),
LeakyReLU(alpha=0.3),
Dense(1,
def __conv_init(a):
print("conv_init", a)
k = RandomNormal(0, 0.02)(a) # for convolution kernel
k.conv_weight = True
return k
def mod_testing2(inp: Union[Tensor, Layer],
kernel_initializer: Initializer = RandomNormal(stddev=0.02)):
m = conv_layer(inp,
64,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
64,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
128,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
128,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
128,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
128,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
256,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
256,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
512,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
512,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = GlobalAveragePooling2D()(m)
m = Dropout(0.4)(m)
m = Dense(1024, activation=None)(m)
m = LeakyReLU(0.2)(m)
return m
def build_patch_discriminator_(self,
model_shape,
filters=32,
k_size=4,
drop=False,
rate=0.5,
summary=False,
model_file=None,
name='gan_d_'):
"""
Create a Discriminator Model using hyperparameters values defined as follows
"""
if (model_file):
"""
Load pretreined model
"""
model = self.utils.build_pretrained_model(model_file)
if (summary):
model.summary()
return model
else:
"""
Create a Discriminator Model using hyperparameters values defined as follows
"""
init = RandomNormal(stddev=0.02)
n_rows = model_shape[0]
n_cols = model_shape[1]
c_dims = model_shape[2]
input_shape = (n_rows, n_cols, c_dims)
input_layer = Input(shape=input_shape, name=name + 'input')
d = self.Conv2D_Block(input_layer,
filters,
k_size=k_size,
name=name + '1',
bn=False)
d = self.Conv2D_Block(d,
2 * filters,
k_size=k_size,
insnorm=True,
name=name + '2')
d = self.Conv2D_Block(d,
4 * filters,
k_size=k_size,
insnorm=True,
name=name + '3')
d = self.Conv2D_Block(d,
8 * filters,
k_size=k_size,
insnorm=True,
strides=1,
name=name + '4')
if drop:
d = Dropout(rate=0.5, name=name + '_dropout')(d, training=True)
logits = Conv2D(1,
k_size,
strides=1,
padding='same',
kernel_initializer=init,
name=name + 'logits')(d)
out = Activation('sigmoid', name=name + 'sigmoid')(logits)
model = Model(inputs=[input_layer],
outputs=[out, logits],
name='Discriminator_' + name[-3:])
if (summary):
model.summary()
return model
def build_resnet_generator_insnorm(self,
model_shape,
filters=32,
k_size=3,
last_act='tanh',
n_residuals=9,
summary=False,
model_file=None,
name='gan_g_'):
"""
Create a Generator Model with hyperparameters values defined as follows
"""
if (model_file):
"""
Load pretreined model
"""
model = self.utils.build_pretrained_model(model_file)
if (summary):
model.summary()
return model
else:
init = RandomNormal(stddev=0.02)
n_rows = model_shape[0]
n_cols = model_shape[1]
in_c_dims = model_shape[2]
out_c_dims = model_shape[3]
n_rows_e1, n_rows_e2, n_rows_e4, n_rows_e8 = n_rows // 1, n_rows // 2, n_rows // 4, n_rows // 8
rows_matching = np.equal(
[2 * n_rows_e2, 2 * n_rows_e4, 2 * n_rows_e8],
[n_rows_e1, n_rows_e2, n_rows_e4])
index_rows = np.where(np.logical_not(rows_matching))[0]
n_cols_e1, n_cols_e2, n_cols_e4, n_cols_e8 = n_cols // 1, n_cols // 2, n_cols // 4, n_cols // 8
cols_matching = np.equal(
[2 * n_cols_e2, 2 * n_cols_e4, 2 * n_cols_e8],
[n_cols_e1, n_cols_e2, n_cols_e4])
index_cols = np.where(np.logical_not(cols_matching))[0]
input_shape = (n_rows, n_cols, in_c_dims)
input_layer = Input(shape=input_shape, name=name + '_input')
x = self.Conv2D_Block(input_layer,
filters,
k_size=7,
strides=1,
activation='relu',
bn=False,
name=name + 'e1') # rows, cols
x = self.Conv2D_Block(x,
2 * filters,
k_size=k_size,
insnorm=True,
activation='relu',
name=name + 'e2') # rows/2, cols/2
x = self.Conv2D_Block(x,
4 * filters,
k_size=k_size,
insnorm=True,
activation='relu',
name=name + 'e3') # rows/4, cols/4
for i in range(n_residuals):
x = self.residual_block(x,
n_kernels=4 * filters,
k_size=k_size,
activation='relu',
insnorm=True,
add=False,
name=name + str(i + 1) + '_')
x = self.Conv2DTranspose_Block(x,
2 * filters,
k_size=k_size,
insnorm=True,
name=name + 'd1') # rows/2, cols/2
x = self.Conv2DTranspose_Block(x,
1 * filters,
k_size=k_size,
insnorm=True,
name=name + 'd2') # rows, cols
x = Conv2DTranspose(out_c_dims,
7,
strides=1,
padding='same',
kernel_initializer=init,
name=name + 'd_out')(x) # rows, cols
x = InstanceNormalization(axis=-1, name=name + 'ins_norm')(x)
output = Activation(last_act, name=name + last_act)(x)
model = Model(inputs=[input_layer],
outputs=[output],
name='Generator' + name[-3:])
if (summary):
model.summary()
return model
def vdcnn_model(embedding_size=97, seq_maxlen=512, n_quantized_characters=97):
"""Very Deep CNN model, uses encoded character level data
https://arxiv.org/pdf/1606.01781.pdf"""
input_text = Input(shape=(seq_maxlen, ))
x = Embedding(input_dim=n_quantized_characters,
output_dim=embedding_size,
input_length=seq_maxlen)(input_text)
#x = _convolutional_block(64)(x)
x = Conv1D(filters=64, kernel_size=3, strides=2, padding="same")(x)
#filters = [64, 128, 256, 512]
filters = [64, 128, 256, 512, 1024]
for n_filt in filters:
### convolutional block
x = Conv1D(n_filt,
kernel_size=3,
strides=1,
padding='same',
activation='linear',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(x)
x = BatchNormalization()(x)
x = PReLU()(x) #PReLU
x = Conv1D(n_filt,
kernel_size=3,
strides=1,
padding='same',
activation='linear',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
###
#if n_filt != filters[-1]:
# print("adding", n_filt, filters[-1])
x = MaxPooling1D(pool_size=3, strides=2, padding='same')(x)
# k-max pooling (Finds values and indices of the k largest entries for the last dimension)
top_k = 8
def _top_k(x):
x = tf.transpose(x, [0, 2, 1])
k_max = tf.nn.top_k(x, k=top_k)
return tf.reshape(k_max[0], (-1, filters[-1] * top_k))
k_max = Lambda(_top_k, output_shape=(filters[-1] * top_k, ))(x)
#x = GlobalMaxPool1D()(x)
x = Dropout(0.2)(Dense(128)(k_max))
x = PReLU()(x)
x = Dropout(0.2)(Dense(128)(x))
x = PReLU()(x)
y_pred = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_text, outputs=y_pred)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy', auc_roc])
return model
def build_generator2D_(self,
model_shape,
filters=32,
k_size=4,
z_size=500,
summary=False,
model_file=None,
name='gan_g_'):
"""
Create a Generator Model with hyperparameters values defined as follows
"""
if (model_file):
"""
Load pretreined model
"""
model = self.utils.build_pretrained_model(model_file)
if (summary):
model.summary()
return model
else:
n_rows = model_shape[0]
n_cols = model_shape[1]
c_dims = model_shape[2]
input_shape = (z_size, )
if n_rows % 8 != 0:
height = n_rows // 8 + 1
else:
height = n_rows // 8
if n_cols % 8 != 0:
width = n_cols // 8 + 1
else:
width = n_cols // 8
num_init_neurons = 8 * filters
reshape_size = (height, width, num_init_neurons)
# 8*height, 4*height, 2*height, height = n_rows, n_rows//2, n_rows//4, n_rows//8
rows_matching = np.equal([2 * height, 4 * height, 8 * height],
[n_rows // 4, n_rows // 2, n_rows])
index_rows = np.where(np.logical_not(rows_matching))[0]
if len(index_rows) > 0:
index_rows = index_rows[0]
# print(index_rows)
# 8*width, 4*width, 2*width, width = n_cols//1, n_cols//2, n_cols//4, n_cols//8
cols_matching = np.equal([2 * width, 4 * width, 8 * width],
[n_cols // 4, n_cols // 2, n_cols])
index_cols = np.where(np.logical_not(cols_matching))[0]
if len(index_cols) > 0:
index_cols = index_cols[0]
# print(index_cols)
input_layer = Input(shape=input_shape, name=name + 'input')
g = Dense(width * height * num_init_neurons,
kernel_initializer=RandomNormal(stddev=0.02),
name=name + 'dense')(input_layer)
g = Reshape(reshape_size, name=name + 'reshape')(g)
g = BatchNormalization(momentum=0.8,
name=name + 'bn_dense')(g, training=True)
g = Activation(activation='relu', name=name + 'relu')(g)
g = self.Conv2DTranspose_Block(g, 4 * filters, name=name + '1')
if index_rows == 0 or index_cols == 0:
g = BilinearUpsampling(output_size=(n_rows // 4, n_cols // 4),
name=name + 'bilinear')(g)
g = self.Conv2DTranspose_Block(g,
2 * filters,
k_size=k_size,
name=name + '2')
if index_rows == 1 or index_cols == 1:
g = BilinearUpsampling(output_size=(n_rows // 2, n_cols // 2),
name=name + 'bilinear')(g)
g = self.Conv2DTranspose_Block(g,
1 * filters,
k_size=k_size,
name=name + '3')
if index_rows == 2 or index_cols == 2:
g = BilinearUpsampling(output_size=(n_rows, n_cols),
name=name + 'bilinear')(g)
g = self.Conv2DTranspose_Block(g,
c_dims,
strides=1,
activation='tanh',
k_size=k_size,
name=name + '4',
bn=False)
model = Model(inputs=[input_layer], outputs=[g], name='Generator')
if (summary):
model.summary()
return model
def convolutional(config, container):
if not config.module_name:
raise ValueError('Missing module name in section')
config.layer_name = (
config.layer_name if config.layer_name else str(len(container.all_layers) - 1)
)
config.size = int(config.size)
if container.frames > 1:
logging.info(f"frames: {container.frames}")
prev_layer_shape = K.int_shape(container.all_layers[-1])
input_channels = prev_layer_shape[-1]
image_size = prev_layer_shape[-3], prev_layer_shape[-2]
num_convs = int(container.frames / config.tstride)
if num_convs > 1:
logging.info(f"num_convs: {num_convs}")
inputs_needed = (config.tstride * (num_convs - 1)) + config.tsize
# inputs_needed = frames + tsize - 1
if inputs_needed > 1:
logging.info(f"inputs_needed: {inputs_needed}")
old_frames_to_read = inputs_needed - container.frames
if old_frames_to_read < 0:
logging.info("negative number of old frames!!!!!!!!!")
if old_frames_to_read:
logging.info(f"history frames needed: {old_frames_to_read}")
new_frames_to_save = min(container.frames, old_frames_to_read)
if new_frames_to_save:
logging.info(f"new frames to save: {new_frames_to_save}")
# attach output ports to inputs we will need next pass
if config.no_output is False:
for f in range(new_frames_to_save):
container.out_index.append(len(container.all_layers) - container.frames + f)
container.out_names.append(config.module_name + "_save_" + str(f))
# attach output ports to unsaved inputs if we need to share inputs to a slave network
if config.share is True:
for f in range(new_frames_to_save, container.frames):
container.out_index.append(len(container.all_layers) - container.frames + f)
container.out_names.append(config.module_name + "_share_" + str(f))
# create input ports for required old frames
for f in range(old_frames_to_read):
xx = config.module_name + "_history_" + str(f)
container.in_names.append(xx)
if config.image_input:
container.image_inputs.append(xx)
container.all_layers.append(
Input(shape=(image_size[0], image_size[1], input_channels), name=xx)
)
container.in_index.append(len(container.all_layers) - 1)
# create input ports for merged-in frames
if config.merge_in > 0:
input_channels = input_channels + config.merge_in
for f in range(inputs_needed):
xx = config.module_name + "_merge_in_" + str(f)
container.in_names.append(xx)
container.all_layers.append(
Input(shape=(image_size[0], image_size[1], config.merge_in), name=xx)
)
logging.info(f"merge_in input at: {len(container.all_layers) - 1}")
container.in_index.append(len(container.all_layers) - 1)
padding = "same" if config.pad == 1 and config.stride == 1 else "valid"
# extract parameter for this module from Pytorch checkpoint file
conv_weights_pt = np.random.rand(
input_channels, config.filters, config.tsize, config.size, config.size
)
conv_bias = [0]
if config.module_name + ".weight" in container.weights:
conv_weights_pt = container.weights[config.module_name + ".weight"]
shape = container.weights[config.module_name + ".weight"].shape
logging.info(f"weight: {config.module_name}.weight {shape}")
# convert to tsize list of 2d conv weight matrices, transposed for Keras
w_list = []
if len(conv_weights_pt.shape) == 5: # check if this is a 3D conv being unfolded
for t in range(config.tsize):
w_list.append(
np.transpose(
conv_weights_pt[:, :, config.tsize - 1 - t, :, :], [2, 3, 1, 0]
)
)
else: # this is simply a single 2D conv
w_list.append(np.transpose(conv_weights_pt[:, :, :, :], [2, 3, 1, 0]))
# concatenate along the in_dim axis the tsize matrices
conv_weights = np.concatenate(w_list, axis=2)
if not config.batch_normalize:
conv_bias = container.weights[config.module_name + ".bias"]
else:
logging.info(f"cannot find weight: {config.module_name}.weight")
container.fake_weights = True
conv_weights = np.random.rand(
config.size, config.size, config.tsize * input_channels, config.filters
)
conv_bias = np.zeros(config.filters)
if config.batch_normalize:
bn_bias = container.weights[config.module_name + ".batchnorm.bias"]
bn_weight = container.weights[config.module_name + ".batchnorm.weight"]
bn_running_var = container.weights[
config.module_name + ".batchnorm.running_var"
]
bn_running_mean = container.weights[
config.module_name + ".batchnorm.running_mean"
]
bn_weight_list = [
bn_weight, # scale gamma
bn_bias, # shift beta
bn_running_mean, # running mean
bn_running_var, # running var
]
expected_weights_shape = (
config.size,
config.size,
config.tsize * input_channels,
config.filters,
)
logging.info(
f"weight shape, expected : {expected_weights_shape} "
f"checkpoint: {conv_weights_pt.shape} "
f"created: {conv_weights.shape} "
)
if conv_weights.shape != expected_weights_shape:
logging.info("weight matrix shape is wrong, making a fake one")
container.fake_weights = True
conv_weights = np.random.rand(
config.size, config.size, config.tsize * input_channels, config.filters
)
conv_bias = np.zeros(config.filters)
conv_weights = (
[conv_weights] if config.batch_normalize else [conv_weights, conv_bias]
)
inputs = []
outputs = []
for f in range(inputs_needed):
inputs.append(
container.all_layers[len(container.all_layers) - inputs_needed + f]
)
if config.merge_in > 0:
inputs.append(
container.all_layers[
len(container.all_layers) - (2 * inputs_needed) + f
]
)
# Create Conv3d from Conv2D layers
if config.stride > 1:
for f in range(len(inputs)):
inputs[f] = ZeroPadding2D(((1, 0), (1, 0)))(inputs[f])
for f in range(int(container.frames / config.tstride)):
layers = []
if config.tsize > 1:
for t in range(config.tsize):
# offset is constant with f, except if tstride,
# then steps by extra step every time through
if config.merge_in == 0:
layers.append(inputs[t + (f * (config.tstride))])
else:
layers.append(inputs[2 * (t + (f * (config.tstride)))])
layers.append(inputs[2 * (t + (f * (config.tstride))) + 1])
cat_layer = Concatenate()(layers)
else:
if config.merge_in == 0:
cat_layer = inputs[f * (config.tstride)]
else:
layers.append(inputs[2 * f * (config.tstride)])
layers.append(inputs[2 * f * (config.tstride) + 1])
cat_layer = Concatenate()(layers)
outputs.append(
(
Conv2D(
config.filters,
(config.size, config.size),
strides=(config.stride, config.stride),
kernel_regularizer=l2(5e-4),
use_bias=not config.batch_normalize,
weights=conv_weights,
activation=None,
padding=padding,
)
)(cat_layer)
)
if config.batch_normalize:
for f in range(int(container.frames / config.tstride)):
outputs[f] = BatchNormalization(weights=bn_weight_list)(outputs[f])
if config.activation == "relu6":
for f in range(int(container.frames / config.tstride)):
outputs[f] = ReLU(max_value=6)(outputs[f])
elif config.activation == "leaky":
for f in range(int(container.frames / config.tstride)):
if not container.conversion_parameters["use_prelu"]:
outputs[f] = LeakyReLU(alpha=0.1)(outputs[f])
else:
outputs[f] = PReLU(
alpha_initializer=RandomNormal(mean=0.1, stddev=0.0, seed=None),
shared_axes=[1, 2],
)(outputs[f])
for f in range(int(container.frames / config.tstride)):
container.all_layers.append(outputs[f])
frames = int(container.frames / config.tstride)
if frames == 0:
raise ValueError("tried to time stride single frame")
container.layer_names[config.layer_name] = len(container.all_layers) - 1
def build_model():
model = Sequential()
model.add(
Conv2D(192, (5, 5),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.01),
input_shape=x_train.shape[1:]))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Conv2D(160, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Conv2D(96, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
model.add(Dropout(dropout))
model.add(
Conv2D(192, (5, 5),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Conv2D(192, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Conv2D(192, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
model.add(Dropout(dropout))
model.add(
Conv2D(192, (3, 3),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Conv2D(192, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Conv2D(10, (1, 1),
padding='same',
kernel_regularizer=keras.regularizers.l2(0.0001),
kernel_initializer=RandomNormal(stddev=0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(GlobalAveragePooling2D())
model.add(Activation('softmax'))
sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
return model
def invResidual(config, container):
if not config.module_name:
raise ValueError('Missing module name in section')
config.layer_name = (
config.layer_name if config.layer_name else str(len(container.all_layers) - 1)
)
logging.info(f"frames: {container.frames}")
s = 0
if config.shift:
logging.info("3D conv block")
tsize = 3
else:
logging.info("2D conv block")
tsize = 1
config.size = int(config.size)
prev_layer = container.all_layers[-1]
prev_layer_shape = K.int_shape(prev_layer)
input_channels = prev_layer_shape[-1]
x_channels = input_channels * config.xratio
image_size = prev_layer_shape[-3], prev_layer_shape[-2]
logging.info(f"input image size: {image_size}")
num_convs = int(container.frames / config.tstride)
inputs_needed = (config.tstride * (num_convs - 1)) + tsize
# inputs_needed = frames + tsize - 1
if inputs_needed > 1:
logging.info(f"inputs_needed: {inputs_needed}")
old_frames_to_read = inputs_needed - container.frames
new_frames_to_save = min(container.frames, old_frames_to_read)
logging.info(
f"num_convs: {num_convs}, inputs_needed: {inputs_needed}, history frames needed: {old_frames_to_read},"
f"frames to save: {new_frames_to_save}, tstride: {config.tstride}"
)
# create (optional) expansion pointwise convolution layer
input_indexes = []
for i in range(num_convs):
input_indexes.append(
len(container.all_layers) - container.frames + (i * config.tstride)
)
if config.xratio != 1:
logging.info("---------- Insert channel multiplier pointwise conv -------------")
# attach output ports to inputs we will need next pass if tsize>1
for f in range(new_frames_to_save):
container.out_index.append(len(container.all_layers) - container.frames + f)
container.out_names.append(config.module_name + "_save_" + str(f))
# create input ports for required old frames if tsize>1
for f in range(old_frames_to_read):
h_name = config.module_name + "_history_" + str(f)
container.all_layers.append(
Input(shape=(image_size[0], image_size[1], input_channels), name=h_name)
)
container.in_names.append(h_name)
container.in_index.append(len(container.all_layers) - 1)
# get weights
n = config.module_name + ".conv." + str(s) + ".0."
if n + "weight" in container.weights:
weights_pt = container.weights[n + "weight"]
logging.info(f"checkpoint: {weights_pt.shape}")
weights_k = np.transpose(weights_pt, [2, 3, 1, 0])
bias = container.weights[n + "bias"]
else:
logging.info(f"missing weight {n}weight")
weights_k = np.random.rand(1, 1, tsize * input_channels, x_channels)
bias = np.zeros(x_channels)
container.fake_weights = True
expected_weights_shape = (1, 1, tsize * input_channels, x_channels)
logging.info(
f"weight shape, expected : {expected_weights_shape} transposed: {weights_k.shape}"
)
if weights_k.shape != expected_weights_shape:
logging.info("weight matrix shape is wrong, making a fake one")
weights_k = np.random.rand(1, 1, tsize * input_channels, x_channels)
bias = np.zeros(x_channels)
container.fake_weights = True
weights = [weights_k, bias]
inputs = []
outputs = []
for f in range(inputs_needed):
inputs.append(
container.all_layers[len(container.all_layers) - inputs_needed + f]
)
if config.merge_in > 0:
inputs.append(
container.all_layers[
len(container.all_layers) - (2 * inputs_needed) + f
]
)
for f in range(int(container.frames / config.tstride)):
layers = []
if tsize > 1:
for t in range(tsize):
# offset is constant with f, except if tstride,
# then steps by extra step every time through
layers.append(inputs[(tsize - t - 1) + (f * (config.tstride))])
cat_layer = Concatenate()(layers)
else:
cat_layer = inputs[f * (config.tstride)]
outputs.append(
(
Conv2D(
x_channels,
(1, 1),
use_bias=not config.batch_normalize,
weights=weights,
activation=None,
padding="same",
)
)(cat_layer)
)
logging.info(
f"parallel convs: {int(container.frames / config.tstride)} : {K.int_shape(cat_layer)}"
)
if config.activation == "leaky":
for f in range(int(container.frames / config.tstride)):
if not container.conversion_parameters["use_prelu"]:
outputs[f] = LeakyReLU(alpha=0.1)(outputs[f])
else:
outputs[f] = PReLU(
alpha_initializer=RandomNormal(mean=0.1, stddev=0.0, seed=None),
shared_axes=[1, 2],
)(outputs[f])
elif config.activation == "relu6":
for f in range(int(container.frames / config.tstride)):
outputs[f] = ReLU(max_value=6)(outputs[f])
for f in range(int(container.frames / config.tstride)):
container.all_layers.append(outputs[f])
s += 1
container.frames = int(container.frames / config.tstride)
else:
logging.info("Skipping channel multiplier pointwise conv, no expansion")
# create groupwise convolution
# get weights
logging.info("---------- Depthwise conv -------------")
n = config.module_name + ".conv." + str(s) + ".0."
logging.info(f"module name base: {n}")
if n + "weight" in container.weights:
weights_pt = container.weights[n + "weight"]
logging.info(f"checkpoint: {weights_pt.shape}")
weights_k = np.transpose(weights_pt, [2, 3, 0, 1])
bias = container.weights[n + "bias"]
else:
logging.info(f"missing weight {n}weight")
weights_k = np.random.rand(config.size, config.size, x_channels, 1)
bias = np.zeros(x_channels)
container.fake_weights = True
expected_weights_shape = (config.size, config.size, x_channels, 1)
logging.info(f"weight shape, expected : {expected_weights_shape} transposed: {weights_k.shape}")
if weights_k.shape != expected_weights_shape:
logging.error("weight matrix shape is wrong, making a fake one")
container.fake_weights = True
weights_k = np.random.rand(config.size, config.size, x_channels, 1)
bias = np.zeros(x_channels)
weights = [weights_k, bias]
inputs = []
outputs = []
padding = "same" if config.pad == 1 and config.stride == 1 else "valid"
for f in range(container.frames):
inputs.append(
container.all_layers[len(container.all_layers) - container.frames + f]
)
if config.stride > 1:
for f in range(len(inputs)):
if config.size == 3: # originally for all sizes
inputs[f] = ZeroPadding2D(
((config.size - config.stride, 0), (config.size - config.stride, 0))
)(inputs[f])
elif config.size == 5: # I found this works...
inputs[f] = ZeroPadding2D(((2, 2), (2, 2)))(inputs[f])
else:
logging.info(f"I have no idea what to do for size {config.size}")
exit()
logging.info(f"parallel convs: {f} : {K.int_shape(inputs[0])}, padding: {padding}")
for f in range(container.frames):
outputs.append(
(
DepthwiseConv2D(
(config.size, config.size),
strides=(config.stride, config.stride),
use_bias=not config.batch_normalize,
weights=weights,
activation=None,
padding=padding,
)
)(inputs[f])
)
if config.activation == "leaky":
for f in range(int(container.frames)):
if not container.conversion_parameters["use_prelu"]:
outputs[f] = LeakyReLU(alpha=0.1)(outputs[f])
else:
outputs[f] = PReLU(
alpha_initializer=RandomNormal(mean=0.1, stddev=0.0, seed=None),
shared_axes=[1, 2],
)(outputs[f])
elif config.activation == "relu6":
for f in range(int(container.frames)):
outputs[f] = ReLU(max_value=6)(outputs[f])
for f in range(int(container.frames)):
container.all_layers.append(outputs[f])
s += 1
# create pointwise convolution
# get weights
logging.info("---------- Pointwise conv -------------")
n = config.module_name + ".conv." + str(s) + "."
logging.info(f"module name base: {n}")
if n + "weight" in container.weights:
weights_pt = container.weights[n + "weight"]
logging.info(f"checkpoint: {weights_pt.shape}")
weights_k = np.transpose(weights_pt, [2, 3, 1, 0])
bias = container.weights[n + "bias"]
else:
logging.error(f"missing weight {n}weight")
container.fake_weights = True
weights_k = np.random.rand(1, 1, x_channels, config.out_channels)
bias = np.zeros(config.out_channels)
expected_weights_shape = (1, 1, x_channels, config.out_channels)
logging.info(
f"weight shape, expected : {expected_weights_shape}"
f"transposed: {weights_k.shape}"
)
if weights_k.shape != expected_weights_shape:
logging.error("weight matrix shape is wrong, making a fake one")
container.fake_weights = True
weights_k = np.random.rand(1, 1, x_channels, config.out_channels)
bias = np.zeros(config.out_channels)
weights = [weights_k, bias]
logging.info(f"combined shape: {weights[0].shape} {weights[1].shape}")
inputs = []
outputs = []
for f in range(container.frames):
inputs.append(
container.all_layers[len(container.all_layers) - container.frames + f]
)
shape = K.int_shape(container.all_layers[len(container.all_layers) - container.frames])
logging.info(f"parallel convs: {f} : {shape}")
for f in range(container.frames):
conv_input = container.all_layers[
len(container.all_layers) - container.frames + f
]
outputs.append(
(
Conv2D(
config.out_channels,
(1, 1),
use_bias=not config.batch_normalize,
weights=weights,
activation=None,
padding="same",
)
)(conv_input)
)
if config.stride == 1 and input_channels == config.out_channels:
for f in range(int(container.frames)):
container.all_layers.append(
Add()([container.all_layers[input_indexes[f]], outputs[f]])
)
else:
for f in range(int(container.frames)):
container.all_layers.append(outputs[f])
s += 1
def generator_network(img_shape, filters, name):
def resblock(feature_in, filters, num):
init = RandomNormal(stddev=0.02)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name=('resblock_%d_CONV_1' % num),
kernel_initializer=init)(feature_in)
temp = BatchNormalization(axis=-1)(temp)
temp = Activation('relu')(temp)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name=('resblock_%d_CONV_2' % num),
kernel_initializer=init)(temp)
temp = BatchNormalization(axis=-1)(temp)
temp = Activation('relu')(temp)
return Add()([temp, feature_in])
init = RandomNormal(stddev=0.02)
image = Input(img_shape)
y = Lambda(lambda x: 2.0 * x - 1.0, output_shape=lambda x: x)(image)
b1_in = Conv2D(filters, (9, 9),
strides=1,
padding='SAME',
name='CONV_1',
activation='relu',
kernel_initializer=init)(y)
b1_in = Activation('relu')(b1_in)
# residual blocks
b1_out = resblock(b1_in, filters, 1)
b2_out = resblock(b1_out, filters, 2)
b3_out = resblock(b2_out, filters, 3)
b4_out = resblock(b3_out, filters, 4)
# conv. layers after residual blocks
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name='CONV_2',
kernel_initializer=init)(b4_out)
#temp = BatchNormalization(axis=-1)(temp)
temp = Activation('relu')(temp)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name='CONV_3',
kernel_initializer=init)(temp)
#temp = BatchNormalization(axis=-1)(temp)
temp = Activation('relu')(temp)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name='CONV_4',
kernel_initializer=init)(temp)
#temp = BatchNormalization(axis=-1)(temp)
temp = Activation('relu')(temp)
temp = Conv2D(3, (9, 9),
strides=1,
padding='SAME',
name='CONV_5',
kernel_initializer=init)(temp)
temp = Activation('tanh')(temp)
temp = Lambda(lambda x: 0.5 * x + 0.5, output_shape=lambda x: x)(temp)
return Model(inputs=image, outputs=temp, name=name)
def miccai2018_net(vol_size,
enc_nf,
dec_nf,
int_steps=7,
use_miccai_int=False,
indexing='ij',
bidir=False,
vel_resize=1 / 2):
"""
architecture for probabilistic diffeomoprhic VoxelMorph presented in the MICCAI 2018 paper.
You may need to modify this code (e.g., number of layers) to suit your project needs.
The stationary velocity field operates in a space (0.5)^3 of vol_size for computational reasons.
:param vol_size: volume size. e.g. (256, 256, 256)
:param enc_nf: list of encoder filters. right now it needs to be 1x4.
e.g. [16,32,32,32]
:param dec_nf: list of decoder filters. right now it must be 1x6, see unet function.
:param use_miccai_int: whether to use the manual miccai implementation of scaling and squaring integration
note that the 'velocity' field outputted in that case was
since then we've updated the code to be part of a flexible layer. see neuron.layers.VecInt
**This param will be phased out (set to False behavior)**
:param int_steps: the number of integration steps
:param indexing: xy or ij indexing. we recommend ij indexing if training from scratch.
miccai 2018 runs were done with xy indexing.
**This param will be phased out (set to 'ij' behavior)**
:return: the keras model
"""
ndims = len(vol_size)
assert ndims in [1, 2,
3], "ndims should be one of 1, 2, or 3. found: %d" % ndims
# get unet
unet_model = unet_core(vol_size, enc_nf, dec_nf, full_size=False)
[src, tgt] = unet_model.inputs
x_out = unet_model.outputs[-1]
# velocity mean and logsigma layers
Conv = getattr(KL, 'Conv%dD' % ndims)
flow_mean = Conv(ndims,
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5),
name='flow')(x_out)
# we're going to initialize the velocity variance very low, to start stable.
flow_log_sigma = Conv(
ndims,
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-10),
bias_initializer=keras.initializers.Constant(value=-10),
name='log_sigma')(x_out)
flow_params = concatenate([flow_mean, flow_log_sigma])
# velocity sample
flow = Sample(name="z_sample")([flow_mean, flow_log_sigma])
# integrate if diffeomorphic (i.e. treating 'flow' above as stationary velocity field)
if use_miccai_int:
# for the miccai2018 submission, the squaring layer
# scaling was essentially built in by the network
# was manually composed of a Transform and and Add Layer.
v = flow
for _ in range(int_steps):
v1 = nrn_layers.SpatialTransformer(interp_method='linear',
indexing=indexing)([v, v])
v = keras.layers.add([v, v1])
flow = v
else:
# new implementation in neuron is cleaner.
z_sample = flow
flow = nrn_layers.VecInt(method='ss',
name='flow-int',
int_steps=int_steps)(z_sample)
if bidir:
rev_z_sample = Negate()(z_sample)
neg_flow = nrn_layers.VecInt(method='ss',
name='neg_flow-int',
int_steps=int_steps)(rev_z_sample)
# get up to final resolution
flow = trf_resize(flow, vel_resize, name='diffflow')
if bidir:
neg_flow = trf_resize(neg_flow, vel_resize, name='neg_diffflow')
# transform
y = nrn_layers.SpatialTransformer(interp_method='linear',
indexing=indexing)([src, flow])
if bidir:
y_tgt = nrn_layers.SpatialTransformer(
interp_method='linear', indexing=indexing)([tgt, neg_flow])
# prepare outputs and losses
outputs = [y, flow_params]
if bidir:
outputs = [y, y_tgt, flow_params]
# build the model
return Model(inputs=[src, tgt], outputs=outputs)
if isinstance(__version__, (list, tuple)):
version_str = ".".join([str(n) for n in __version__[1:]])
else:
version_str = __version__
mswindows = sys.platform == "win32"
class EncoderType(enum.Enum):
ORIGINAL = "original"
SHAOANLU = "shaoanlu"
ENCODER = EncoderType.ORIGINAL
conv_init = RandomNormal(0, 0.02)
def inst_norm():
return InstanceNormalization()
ENCODER = EncoderType.ORIGINAL
hdf = {
'encoderH5': 'encoder_{version_str}{ENCODER.value}.h5'.format(**vars()),
'decoder_AH5':
'decoder_A_{version_str}{ENCODER.value}.h5'.format(**vars()),
'decoder_BH5': 'decoder_B_{version_str}{ENCODER.value}.h5'.format(**vars())
}
def _create_bias_layer(self, inputLayer):
userCount = self.params["userCount"]
itemCount = self.params["itemCount"]
latentFeatures = self.params["latentFeatures"]
regularizationScale = self.params["regularizationScale"]
userInputLayer = inputLayer["userLayer"]
itemInputLayer = inputLayer["itemLayer"]
# Define user Bias
userBiasLayer = Embedding(input_dim=userCount, output_dim=1, input_length=1, embeddings_initializer=RandomNormal(
), embeddings_regularizer=l2(regularizationScale))(userInputLayer)
userBiasLayer = Flatten()(userBiasLayer)
# Define item Bias
itemBiasLayer = Embedding(input_dim=itemCount, output_dim=1, input_length=1, embeddings_initializer=RandomNormal(
), embeddings_regularizer=l2(regularizationScale))(itemInputLayer)
itemBiasLayer = Flatten()(itemBiasLayer)
return {
"userBiasLayer": userBiasLayer,
"itemBiasLayer": itemBiasLayer
}
def build_discriminator2D(self,
model_shape,
filters=32,
k_size=4,
drop=True,
rate=0.5,
summary=False,
model_file=None,
name='gan_d_'):
"""
Create a Discriminator Model using hyperparameters values defined as follows
"""
if (model_file):
"""
Load pretreined model
"""
model = self.utils.build_pretrained_model(model_file)
if (summary):
model.summary()
return model
else:
"""
Create a Discriminator Model using hyperparameters values defined as follows
"""
n_rows = model_shape[0]
n_cols = model_shape[1]
c_dims = model_shape[2]
input_shape = (n_rows, n_cols, c_dims)
input_layer = Input(shape=input_shape, name=name + 'input')
d = self.Conv2D_Block(input_layer,
filters,
k_size=k_size,
name=name + '1',
bn=False) # 30x30x32
d = self.Conv2D_Block(d,
2 * filters,
k_size=k_size,
name=name + '2') # 15x15x64
d = self.Conv2D_Block(d,
4 * filters,
k_size=k_size,
name=name + '3') # 8x8x128
d = self.Conv2D_Block(d,
8 * filters,
strides=1,
k_size=k_size,
name=name + '4') # 8x8x256
d = Flatten(name=name + 'flatten')(d)
if drop:
d = Dropout(rate=rate, name=name + 'dropout')(d, training=True)
logits = Dense(1,
activation='linear',
kernel_initializer=RandomNormal(stddev=0.02),
name=name + 'dense')(d)
out = Activation('sigmoid', name=name + 'sigmoid')(logits)
model = Model(inputs=[input_layer],
outputs=[out, logits],
name='Discriminator')
if (summary):
model.summary()
return model
def _create_embedding_layer(self, userInputLayer, itemInputLayer):
userCount = self.params["userCount"]
itemCount = self.params["itemCount"]
latentFeatures = self.params["latentFeatures"]
regularizationScale = self.params["regularizationScale"]
# tf.keras.regularizers.l2 #loss = l2 * reduce_sum(square(x))
userEmbeddingLayer = Embedding(input_dim=userCount, output_dim=latentFeatures, input_length=1,
embeddings_regularizer=l2(regularizationScale), embeddings_initializer=RandomNormal())(userInputLayer)
userEmbeddingLayer = Flatten()(userEmbeddingLayer)
itemEmbeddingLayer = Embedding(input_dim=itemCount, output_dim=latentFeatures, input_length=1,
embeddings_regularizer=l2(regularizationScale), embeddings_initializer=RandomNormal())(itemInputLayer)
itemEmbeddingLayer = Flatten()(itemEmbeddingLayer)
return {
"userLayer": userEmbeddingLayer,
"itemLayer": itemEmbeddingLayer
}
def build_encoder2D(self,
model_shape,
filters=32,
k_size=4,
z_size=500,
drop=True,
rate=0.5,
bn=True,
summary=False,
model_file=None,
name='gan_e_'):
"""
Create a Discriminator Model using hyperparameters values defined as follows
"""
if (model_file):
"""
Load pretreined model
"""
model = self.utils.build_pretrained_model(model_file)
if (summary):
model.summary()
return model
else:
"""
Create a Discriminator Model using hyperparameters values defined as follows
"""
n_rows = model_shape[0]
n_cols = model_shape[1]
c_dims = model_shape[2]
input_shape = (n_rows, n_cols, c_dims)
input_layer = Input(shape=input_shape, name=name + 'input')
x = self.Conv2D_Block(input_layer,
filters,
k_size=k_size,
name=name + '1',
bn=False)
x = self.Conv2D_Block(x,
2 * filters,
k_size=k_size,
name=name + '2')
x = self.Conv2D_Block(x,
4 * filters,
k_size=k_size,
name=name + '3')
x = self.Conv2D_Block(x,
8 * filters,
strides=1,
k_size=k_size,
name=name + '4')
x = Flatten(name=name + 'flatten')(x)
if drop:
x = Dropout(rate=rate, name=name + 'dropout')(x)
x = Dense(z_size,
activation='linear',
kernel_initializer=RandomNormal(stddev=0.02),
name=name + 'dense')(x)
if bn:
x = BatchNormalization(center=False,
scale=False,
name=name + "bn_out")(x)
model = Model(inputs=[input_layer], outputs=[x], name='Encoder')
if (summary):
model.summary()
return model
def createModel(args, opt):
numNodes = [8, 16, 32, 64, 128, 256, 512]
k = (args.kernel_size * 2) + 1
w8s = {
"Zeros": Zeros(),
"Ones": Ones(),
"Constant": Constant(value=0.2),
"RandNormal": RandomNormal(mean=0.0, stddev=0.05, seed=0)
}
model = Sequential()
model.add(GaussianNoise(args.gauss_noise, input_shape=(32, 32, 3)))
model.add(
Conv2D(filters=numNodes[0],
kernel_size=(k, k),
padding='same',
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Convolution 1
model.add(Activation('relu')) # ReLU 1
model.add(
Conv2D(
filters=numNodes[1],
kernel_size=(k, k),
padding='same', #
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Convlution 2
model.add(Activation('relu')) # ReLU 2
model.add(Dropout(args.dropout_conv)) # Dropout 1
model.add(MaxPooling2D(pool_size=2)) # Max Pooling 1
#
model.add(
Conv2D(
filters=numNodes[2],
kernel_size=(k, k),
padding='same', #
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Convolution 3
model.add(Activation('relu')) # ReLU 3
model.add(
Conv2D(
filters=numNodes[3],
kernel_size=(k, k),
padding='same', #
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Convolution 4
model.add(Activation('relu')) # ReLU 4
model.add(Dropout(args.dropout_conv)) # Dropout 2
model.add(MaxPooling2D(pool_size=2)) # Max Pooling 2
#
model.add(
Conv2D(
filters=numNodes[4],
kernel_size=(k, k),
padding='same', #
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Convolution 5
model.add(Activation('relu')) # ReLU 5
model.add(
Conv2D(
filters=numNodes[5],
kernel_size=(k, k),
padding='same', #
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Convolution 6
model.add(Activation('relu')) # ReLU 6
model.add(Dropout(args.dropout_conv)) # Dropout 3
model.add(MaxPooling2D(pool_size=2)) # Max Pooling 3
#
model.add(Flatten()) # Flatten
model.add(
Dense(numNodes[6],
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Dense 1 (FC)
model.add(Activation('relu')) # ReLU 8
model.add(Dropout(args.dropout_dense)) # Dropout 4
model.add(
Dense(
100,
activation='softmax', #
kernel_initializer=w8s[args.w8s],
bias_initializer=w8s[args.w8s])) # Dense 2 (FC)
#
#print(model.summary())
#print("Model has {} paramters".format(model.count_params()))
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return model
def build_resnet_generator(self,
model_shape,
filters=32,
k_size=3,
last_act='tanh',
summary=False,
model_file=None,
name='gan_g_'):
"""
Create a Generator Model with hyperparameters values defined as follows
"""
if (model_file):
"""
Load pretreined model
"""
model = self.utils.build_pretrained_model(model_file)
if (summary):
model.summary()
return model
else:
init = RandomNormal(stddev=0.02)
n_rows = model_shape[0]
n_cols = model_shape[1]
in_c_dims = model_shape[2]
out_c_dims = model_shape[3]
n_rows_e1, n_rows_e2, n_rows_e4, n_rows_e8 = n_rows // 1, n_rows // 2, n_rows // 4, n_rows // 8
rows_matching = np.equal(
[2 * n_rows_e2, 2 * n_rows_e4, 2 * n_rows_e8],
[n_rows_e1, n_rows_e2, n_rows_e4])
index_rows = np.where(np.logical_not(rows_matching))[0]
n_cols_e1, n_cols_e2, n_cols_e4, n_cols_e8 = n_cols // 1, n_cols // 2, n_cols // 4, n_cols // 8
cols_matching = np.equal(
[2 * n_cols_e2, 2 * n_cols_e4, 2 * n_cols_e8],
[n_cols_e1, n_cols_e2, n_cols_e4])
index_cols = np.where(np.logical_not(cols_matching))[0]
input_shape = (n_rows, n_cols, in_c_dims)
input_layer = Input(shape=input_shape, name=name + '_input')
e1 = self.Conv2D_Block(input_layer,
n_kernels=filters,
k_size=7,
strides=1,
bn=False,
name=name + 'e1') # rows, cols
e2 = self.Conv2D_Block(e1,
2 * filters,
k_size=k_size,
bn_training=True,
name=name + 'e2') # rows/2, cols/2
e3 = self.Conv2D_Block(e2,
4 * filters,
k_size=k_size,
bn_training=True,
name=name + 'e3') # rows/4, cols/4
e4 = self.Conv2D_Block(e3,
8 * filters,
k_size=k_size,
bn=False,
name=name + 'e4') # rows/8, cols/8
rb1 = self.residual_block(e4,
n_kernels=8 * filters,
k_size=k_size,
bn_training=True,
name=name + '1_')
rb2 = self.residual_block(rb1,
n_kernels=8 * filters,
k_size=k_size,
bn_training=True,
name=name + '2_')
rb3 = self.residual_block(rb2,
n_kernels=8 * filters,
k_size=k_size,
bn_training=True,
name=name + '3_')
rb3 = Dropout(rate=0.5, name=name + 'drop_1')(rb3, training=True)
rb4 = self.residual_block(rb3,
n_kernels=8 * filters,
k_size=k_size,
bn_training=True,
name=name + '4_')
rb4 = Dropout(rate=0.5, name=name + 'drop_2')(rb4, training=True)
rb5 = self.residual_block(rb4,
n_kernels=8 * filters,
k_size=k_size,
bn_training=True,
name=name + '5_')
rb5 = Dropout(rate=0.5, name=name + 'drop_3')(rb5, training=True)
d1 = self.Conv2DTranspose_Block(rb5,
4 * filters,
k_size=k_size,
activation='linear',
name=name + 'd1') # rows/4, cols/4
if index_rows == 2 or index_cols == 2:
d1 = BilinearUpsampling(output_size=(n_rows // 4, n_cols // 4),
name=name + '_bilinear')(d1)
d1 = Concatenate(name=name + 'conc_1')([d1, e3])
d1 = Activation('relu', name=name + '_act_1')(d1)
d2 = self.Conv2DTranspose_Block(d1,
2 * filters,
k_size=k_size,
activation='linear',
name=name + 'd2') # rows/2, cols/2
if index_rows == 1 or index_cols == 1:
d2 = BilinearUpsampling(output_size=(n_rows // 2, n_cols // 2),
name=name + '_bilinear')(d2)
d2 = Concatenate(name=name + 'conc_2')([d2, e2])
d2 = Activation('relu', name=name + '_act_2')(d2)
d3 = self.Conv2DTranspose_Block(d2,
1 * filters,
k_size=k_size,
activation='linear',
name=name + 'd3') # rows, cols
if index_rows == 0 or index_cols == 0:
d3 = BilinearUpsampling(output_size=(n_rows, n_cols),
name=name + '_bilinear')(d2)
d3 = Concatenate(name=name + 'conc_3')([d3, e1])
d3 = Activation('relu', name=name + 'act_3')(d3)
output = Conv2DTranspose(out_c_dims,
7,
strides=1,
padding='same',
kernel_initializer=init,
name=name + 'd_out')(d3) # rows, cols
# output = InstanceNormalization(axis=-1, name=name+'ins_norm')(output)
output = Activation(last_act, name=name + last_act)(output)
model = Model(inputs=[input_layer],
outputs=[output],
name='Generator' + name[-3:])
if (summary):
model.summary()
return model
ave_acc = 0
ave_auc = 0
for train, test in GroupKFold(n_splits=n_splits).split(X, y, groups=groups):
# referring to inputs and outputs: X(train), y(train), X(test), y(test)
mcnn = Sequential()
mcnn.add(
Embedding(vocab_size,
EMBEDDING_DIM,
input_length=seq_len,
embeddings_initializer=Constant(embedding_matrix)))
mcnn.add(
Conv1D(filters=20,
kernel_size=(8),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.05)))
mcnn.add(MaxPooling1D(pool_size=2))
mcnn.add(
Conv1D(filters=20,
kernel_size=(5),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.05)))
mcnn.add(MaxPooling1D(pool_size=2))
mcnn.add(
Conv1D(filters=20,
kernel_size=(2),
kernel_initializer=RandomNormal(mean=0.0, stddev=0.05)))
mcnn.add(MaxPooling1D(pool_size=2))
mcnn.add(Flatten())
mcnn.add(Dense(300, activation='relu'))
mcnn.add(Dropout(0.5))
mcnn.add(Dense(1, activation='sigmoid'))
netGBH5 = 'netGB_GAN.h5'
netDAH5 = 'netDA_GAN.h5'
netDBH5 = 'netDB_GAN.h5'
netDA2H5 = 'netDA_GAN.h5'
netDB2H5 = 'netDB_GAN.h5'
netD_codeH5 = 'netD_code_GAN.h5'
def __conv_init(a):
print("conv_init", a)
k = RandomNormal(0, 0.02)(a) # for convolution kernel
k.conv_weight = True
return k
conv_init = RandomNormal(0, 0.02)
gamma_init = RandomNormal(1., 0.02) # for batch normalization
class GANModel():
img_size = 64
channels = 3
img_shape = (img_size, img_size, channels)
encoded_dim = 1024
nc_in = 3 # number of input channels of generators
nc_D_inp = 6 # number of input channels of discriminators
def __init__(self, model_dir):
self.model_dir = model_dir
optimizer = Adam(1e-4, 0.5)
def generator_network(name):
def resnet_block(n_filters, input_layer):
# weight initialization
init = RandomNormal(stddev=0.02)
# first layer convolutional layer
g = Conv2D(n_filters, (3, 3), padding='same',
kernel_initializer=init)(input_layer)
g = InstanceNormalization(axis=-1)(g)
g = Activation('relu')(g)
# second convolutional layer
g = Conv2D(n_filters, (3, 3), padding='same',
kernel_initializer=init)(g)
g = InstanceNormalization(axis=-1)(g)
# concatenate merge channel-wise with input layer
g = Add()([g, input_layer])
return g
init = RandomNormal(stddev=0.02)
image = Input((64, 64, 3))
y = Lambda(lambda x: 2.0 * x - 1.0, output_shape=lambda x: x)(image)
g = Conv2D(64, (7, 7), padding='same', kernel_initializer=init)(y)
g = InstanceNormalization(axis=-1)(g)
g = Activation('relu')(g)
# d128
g = Conv2D(128, (3, 3),
strides=(2, 2),
padding='same',
kernel_initializer=init)(g)
g = InstanceNormalization(axis=-1)(g)
g = Activation('relu')(g)
# d256
g = Conv2D(256, (3, 3),
strides=(2, 2),
padding='same',
kernel_initializer=init)(g)
g = InstanceNormalization(axis=-1)(g)
g = Activation('relu')(g)
for _ in range(3):
g = resnet_block(256, g)
# u128
g = Conv2DTranspose(128, (3, 3),
strides=(2, 2),
padding='same',
kernel_initializer=init)(g)
g = InstanceNormalization(axis=-1)(g)
g = Activation('relu')(g)
# u64
g = Conv2DTranspose(64, (3, 3),
strides=(2, 2),
padding='same',
kernel_initializer=init)(g)
g = InstanceNormalization(axis=-1)(g)
g = Activation('relu')(g)
# c7s1-3
g = Conv2D(3, (7, 7), padding='same', kernel_initializer=init)(g)
g = InstanceNormalization(axis=-1)(g)
out_image = Activation('tanh')(g)
out_image = Lambda(lambda x: 0.5 * x + 0.5,
output_shape=lambda x: x)(out_image)
# define model
model = Model(inputs=image, outputs=out_image, name=name)
return model
def mod_testing8(inp: Union[Tensor, Layer],
kernel_initializer: Initializer = RandomNormal(stddev=0.02)):
m = GaussianNoise(0.2)(inp)
m = conv_layer(m,
16,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
16,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
16,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
32,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
32,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
64,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
64,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
128,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
128,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
256,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
256,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
512,
kernel_size=3,
strides=1,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = conv_layer(m,
512,
kernel_size=3,
strides=2,
dropout=None,
batch_norm=None,
act="leaky",
use_bias=True,
kernel_initializer=kernel_initializer)
m = Flatten()(m)
m = Dense(128, activation=None)(m)
m = LeakyReLU(0.2)(m)
return m
def generator_deconv(noise_dim, img_dim, batch_size, model_name="generator_deconv", dset="mnist"):
"""DCGAN generator based on Deconv2D
Args:
noise_dim: Dimension of the noise input
img_dim: dimension of the image output
batch_size: needed to reshape after the deconv2D
model_name: model name (default: {"generator_deconv"})
dset: dataset (default: {"mnist"})
Returns:
keras model
"""
assert K.backend() == "tensorflow", "Deconv not implemented with theano"
s = img_dim[1]
f = 512
if dset == "mnist":
start_dim = int(s / 4)
nb_upconv = 2
elif dset == "celebA":
start_dim = int(s / 16)
nb_upconv = 4
else:
o = s
nb_upconv = 0
while o > 7:
o = o/2
nb_upconv += 1
start_dim = int(o)
reshape_shape = (start_dim, start_dim, f)
bn_axis = -1
output_channels = img_dim[-1]
gen_input = Input(shape=noise_dim, name="generator_input")
# Noise input and reshaping
x = Dense(f * start_dim * start_dim, input_dim=noise_dim, use_bias=False)(gen_input)
x = Reshape(reshape_shape)(x)
x = BatchNormalization(axis=bn_axis)(x)
x = Activation("relu")(x)
# Transposed conv blocks: Deconv2D->BN->ReLU
for i in range(nb_upconv - 1):
nb_filters = int(f / (2 ** (i + 1)))
s = start_dim * (2 ** (i + 1))
o_shape = (batch_size, s, s, nb_filters)
x = Deconv2D(nb_filters, (3, 3),
output_shape=o_shape, strides=(2, 2),
padding="same", use_bias=False,
kernel_initializer=RandomNormal(stddev=0.02))(x)
x = BatchNormalization(axis=-1)(x)
x = Activation("relu")(x)
# Last block
s = start_dim * (2 ** (nb_upconv))
o_shape = (batch_size, s, s, output_channels)
x = Deconv2D(output_channels, (3, 3),
output_shape=o_shape, strides=(2, 2),
padding="same", use_bias=False,
kernel_initializer=RandomNormal(stddev=0.02))(x)
x = Activation("tanh")(x)
generator_model = Model(inputs=[gen_input], outputs=[x], name=model_name)
visualize_model(generator_model)
return generator_model
def model_initialisation(N_input, N_classes, initialiser,
seed_value, path_init):
"""
Model initialization according to a scheme given as input
Input:
~ N_input, N_classes Integers
~ initialiser string, among 'normal', 'orth', 'glorot'
~ seed_value seed for reproducibility, it specifies the
directory in which to store the results
~ path_init path to store the initialised model
Returns:
~ model keras.Models.Sequential instance. A linear stack
of layers. Parameters are initialised according
to the scheme specified
Note that the task to be performed by the network are straight-forward. An equally
overall simple model and algorithmic setup suffices to capture the problem complexity
"""
print("\nModel initialisation. Scheme:")
model = Sequential()
if (initialiser == 'orth'):
print("Orthogonal weights initialisation")
weights_initializer = Orthogonal(gain = 1.0, seed = seed_value)
elif (initialiser == 'normal'):
print("Normal weights initialisation")
weights_initializer = RandomNormal(mean = 0.0,
stddev = 0.1,
seed = seed_value)
elif (initialiser == 'glorot'):
print("Glorot weights initialisation")
weights_initializer = glorot_normal(seed = seed_value)
elif (initialiser == 'zeros'):
weights_initializer = Zeros()
else:
print('NO initialiser match')
#end
model.add(Dense(input_dim = N_input, units = 20,
kernel_initializer = weights_initializer,
bias_initializer = RandomNormal(mean = 0.0,
stddev = 0.1,
seed = seed_value),
activation = 'relu'))
model.add(Dense(input_dim = 20, units = 10,
kernel_initializer = weights_initializer,
bias_initializer = RandomNormal(mean = 0.0,
stddev = 0.1,
seed = seed_value),
activation = 'relu'))
model.add(Dense(units = N_classes,
kernel_initializer = weights_initializer,
bias_initializer = RandomNormal(mean = 0.0,
stddev = 0.1,
seed = seed_value),
activation = 'softmax'))
"""
The optimization algorithm details are defined here
once for all, the model is returned with these details
embedded yet. Hereafter, in the actual training stage
it is ready to use with the
model.fit(args)
method
"""
sgd = keras.optimizers.SGD(lr = 0.01, decay = 1e-6,
momentum = 0.6, nesterov = True)
model.compile(loss = 'categorical_crossentropy',
optimizer = sgd, metrics = ['accuracy'])
streams.check_create_directory(path_init + r'\init')
model.save(path_init + r'\init' + r'\model_init.h5')
return model
def generator_upsampling(noise_dim, img_dim, model_name="generator_upsampling", dset="mnist"):
"""DCGAN generator based on Upsampling and Conv2D
Args:
noise_dim: Dimension of the noise input
img_dim: dimension of the image output
model_name: model name (default: {"generator_upsampling"})
dset: dataset (default: {"mnist"})
Returns:
keras model
"""
s = img_dim[1]
f = 512
if dset == "mnist":
start_dim = int(s / 4)
nb_upconv = 2
elif dset == "celebA":
start_dim = int(s / 16)
nb_upconv = 4
else:
o = s
nb_upconv = 0
while o > 7:
o = o/2
nb_upconv += 1
start_dim = int(o)
if K.image_dim_ordering() == "th":
bn_axis = 1
reshape_shape = (f, start_dim, start_dim)
output_channels = img_dim[0]
else:
reshape_shape = (start_dim, start_dim, f)
bn_axis = -1
output_channels = img_dim[-1]
gen_input = Input(shape=noise_dim, name="generator_input")
# Noise input and reshaping
x = Dense(f * start_dim * start_dim, input_dim=noise_dim)(gen_input)
x = Reshape(reshape_shape)(x)
x = BatchNormalization(axis=bn_axis)(x)
x = Activation("relu")(x)
# Upscaling blocks: Upsampling2D->Conv2D->ReLU->BN->Conv2D->ReLU
for i in range(nb_upconv):
x = UpSampling2D(size=(2, 2))(x)
nb_filters = int(f / (2 ** (i + 1)))
x = Conv2D(nb_filters, (3, 3), padding="same", kernel_initializer=RandomNormal(stddev=0.02))(x)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Conv2D(nb_filters, (3, 3), padding="same", kernel_initializer=RandomNormal(stddev=0.02))(x)
x = Activation("relu")(x)
# Last Conv to get the output image
x = Conv2D(output_channels, (3, 3), name="gen_conv2d_final",
padding="same", activation='tanh', kernel_initializer=RandomNormal(stddev=0.02))(x)
generator_model = Model(inputs=[gen_input], outputs=[x], name=model_name)
visualize_model(generator_model)
return generator_model
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False # randomly flip images
)
datagen.fit(x_train)
# ### Model
# In[4]:
base_model = VGG19(weights='imagenet')
# In[5]:
conv_init = RandomNormal(stddev=0.03)
fc_init = RandomNormal(stddev=0.01)
# In[6]:
img_input = Input(shape=(32, 32, 3))
# get layers from VGG19
x = base_model.layers[1](img_input)
for layer in base_model.layers[2:21]:
x = layer(x)
# add layers by ourselves
x = Flatten()(x)
x = Dense(4096, activation='relu', name='fc6', kernel_initializer=fc_init)(x)
x = Dropout(0.5)(x)
x = base_model.layers[24](x)
x = Dropout(0.5)(x)
n_dim_style = 8 n_resblocks = 4 # number of residual blocks in decoder and content encoder n_adain = 2*n_resblocks n_dim_adain = 256 nc_base = 64 # Number of channels of the first conv2d of encoder n_downscale_content = 2 # Number of content encoder dowscaling n_dowscale_style = 4 # Number of style encoder dowscaling use_groupnorm = False # else use_layer norm in upscaling blocks w_l2 = 1e-4 # L2 weight regularization # Optimization configs use_lsgan = True use_mixup = True mixup_alpha = 0.2 batchSize = 1 # was 4 conv_init_dis = RandomNormal(0, 0.02) # initializer of dicriminators' conv layers conv_init = 'he_normal' # initializer of generators' conv layers lrD = 0.00001 # Discriminator learning rate lrG = 0.00001 # Generator learning rate opt_decay = 0 # Learning rate decay over each update. TOTAL_ITERS = 300000 # Max training iterations # Loss weights for generators w_D = 1 # Adversarial loss (Gan) lambda_s=1; # Latent codes- style lambda_c=1; # Latent codes- content luma_w_ir=7; chrome_w_ir=3; luma_w_vis=7; chrome_w_vis=4;
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.input_shape = (128, 128, 3)
self.encoder_dim = 512 if self.low_mem else 1024
self.kernel_initializer = RandomNormal(0, 0.02)
def __conv_init(a):
print("conv_init", a)
k = RandomNormal(0, 0.02)(a) # for convolution kernel
k.conv_weight = True
return k
