def __init__(self,
hidden_size=1024,
intermediate_hidden_size=4096,
dropout=0.0,
init_stddev=0.02,
shared_intermediate=False,
shared_output=False,
**kwargs):
super(FeedForward, self).__init__(**kwargs)
self.hidden_size = hidden_size
self.intermediate_hidden_size = intermediate_hidden_size
self.dropout = dropout
self.init_stddev = init_stddev
self.shared_intermediate = shared_intermediate
self.intermediate_dense = self.get_layer(
shared_intermediate,
'ff_intermediate_dense',
layers.Dense,
intermediate_hidden_size,
kernel_initializer=initializers.TruncatedNormal(
stddev=init_stddev),
activation=gelu,
)
self.output_dense = self.get_layer(
shared_output,
'ff_output_dense',
layers.Dense,
hidden_size,
kernel_initializer=initializers.TruncatedNormal(
stddev=init_stddev),
)
def __init__(self,
filters,
kernel_size,
name,
use_bn=True,
use_res=True,
padding='SAME',
use_bias=True):
super().__init__(name=name)
self.use_bn = use_bn
self.use_res = use_res
stddev = np.sqrt(2 / (kernel_size**2 * filters))
self.conv1 = layers.Conv3D(
filters,
kernel_size,
padding=padding,
use_bias=use_bias,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='conv1')
self.conv2 = layers.Conv3D(
filters,
kernel_size,
padding=padding,
use_bias=use_bias,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='conv2')
if use_bn:
self.bn1 = layers.BatchNormalization(momentum=0.99, name='bn1')
self.bn2 = layers.BatchNormalization(momentum=0.99, name='bn2')
if use_res:
self.res = _Residual('res')
def __init__(self, filters, kernel_size, pool_size, name, concat_or_add='concat', use_bn=True, use_res=True,
padding='SAME', use_bias=True):
super().__init__(name=name)
self.use_bn = use_bn
self.use_res = use_res
self.concat_or_add = concat_or_add
stddev = np.sqrt(2 / (kernel_size ** 2 * filters))
self.deconv = layers.Conv2DTranspose(filters // 2, kernel_size, strides=pool_size, padding=padding,
use_bias=use_bias,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='deconv')
self.conv1 = layers.Conv2D(filters // 2, kernel_size, padding=padding, use_bias=use_bias,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='conv1')
self.conv2 = layers.Conv2D(filters // 2, kernel_size, padding=padding, use_bias=use_bias,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='conv2')
self.conv3 = layers.Conv2D(filters //2, 1, padding=padding, use_bias=use_bias,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='conv3')
if use_bn:
self.bn_deconv = layers.BatchNormalization(momentum=0.99, name='bn_deconv')
self.bn1 = layers.BatchNormalization(momentum=0.99, name='bn1')
self.bn2 = layers.BatchNormalization(momentum=0.99, name='bn2')
self.bn3 = layers.BatchNormalization(momentum=0.99, name='bn3')
if use_res:
self.res = _Residual('res')
def __init__(self, num_classes=6, L1=128, L2=256,
cell_units=128, num_linear=768, p=10, time_step=100,
F1=64, dropout_keep_prob=0.8):
super().__init__()
# strides = [1,1,1,1]
self.conv1 = Conv2D(filters=L1, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.dropout_keep_prob = dropout_keep_prob
self.max_pool = MaxPool2D(pool_size=2, strides=2, padding='valid')
self.conv2 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv3 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv4 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv5 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv6 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.dropout = Dropout(dropout_keep_prob)
self.reshape1 = Reshape((-1, time_step, L2 * p))
self.reshape2 = Reshape((-1, L2 * p))
# self.reshape3 = Reshape((-1, time_step, num_linear))
self.flatten = Flatten()
self.d1 = Dense(num_linear, use_bias=True, bias_initializer=initializers.constant(0.1),
kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1))
self.bn = BatchNormalization()
self.d2 = Dense(F1, use_bias=True, bias_initializer=initializers.constant(0.1),
kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1))
self.d3 = Dense(num_classes, use_bias=True, bias_initializer=initializers.constant(0.1),
kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1), activation='softmax')
self.bilstm = Bidirectional(LSTM(cell_units, return_sequences=True), merge_mode='concat')
self.attention = attention()
def __init__(self, conf, rate, nc):
"""
Parameters
----------
conf: Dictionary
Configuration dictionary.
"""
# Initialize.
self.conf = conf
self.raw_data_path = self.conf['raw_data_path']
self.hps = self.conf['hps']
self.nn_arch = self.conf['nn_arch']
super(Module2, self).__init__()
# Design layers.
# Split separable conv3d.
self.sep_conv3d_1 = SeparableConv3D(
nc,
kernel_size=3,
depth_multiplier=1,
dilation_rate=(rate[0] * self.nn_arch['conv_rate_multiplier'],
rate[1] * self.nn_arch['conv_rate_multiplier'],
rate[2] * self.nn_arch['conv_rate_multiplier']),
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal())
self.bn_1 = BatchNormalization(momentum=self.hps['bn_momentum'],
scale=self.hps['bn_scale'])
self.act_1 = Activation('relu')
self.conv3d_2 = Conv3D(
np.maximum(1, int(nc / 2)),
kernel_size=3,
strides=2,
padding='valid',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal(),
kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
self.bn_2 = BatchNormalization(momentum=self.hps['bn_momentum'],
scale=self.hps['bn_scale'])
self.act_2 = Activation('relu')
self.sep_conv3d_3 = SeparableConv3D(
nc,
kernel_size=3,
depth_multiplier=1,
dilation_rate=(rate[0] * self.nn_arch['conv_rate_multiplier'],
rate[1] * self.nn_arch['conv_rate_multiplier'],
rate[2] * self.nn_arch['conv_rate_multiplier']),
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal())
self.bn_3 = BatchNormalization(momentum=self.hps['bn_momentum'],
scale=self.hps['bn_scale'])
self.act_3 = Activation('relu')
def create_keras_three_layer_dense_model(*,
input_size,
output_size,
verbose=False,
**kwargs
):
# ...................................................
# Create model
"use the same parameters for 1st and seocnd layer"
model = Sequential()
#.. First hidden layer
model.add(Dense(
input_dim=input_size,
units=kwargs['h1_unit_size'],
activation=kwargs["h1_activation"],
kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.01, seed=0)
))
#.. Second hidden layer
model.add(Dense(
units=kwargs['h1_unit_size'],
activation=kwargs["h1_activation"],
kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.01, seed=0)
))
#.. Output layer
model.add(Dense(
units=output_size,
activation=kwargs["out_activation"],
kernel_regularizer=tf.keras.regularizers.l2(0.001),
kernel_initializer=initializers.VarianceScaling(scale=1.0, seed=0)
))
# Print network summary
if verbose==True:
print(model.summary())
else:
pass
# ...................................................
# Define Loss Function and Trianing Operation
""" # [option]: Use only default values,
model.compile( optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['acc'])
"""
model.compile(
optimizer= kwargs["optimizer"],
loss= losses.sparse_categorical_crossentropy,
metrics= kwargs["metrics"] # even one arg must be in the list
)
return model
def build_model(learning_rate=0.001,
decay=10e-6,
momentum=0.9,
l2_parameter=0.1,
hidden_layer_size1=256,
hidden_layer_size2=64,
input_dim=1024,
third_layer=True):
model = models.Sequential()
model.add(
layers.Dense(units=hidden_layer_size1,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0, stddev=np.sqrt(2. / input_dim), seed=None),
activation='relu',
input_shape=(input_dim, )))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(units=hidden_layer_size2,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0,
stddev=np.sqrt(2. / hidden_layer_size1),
seed=None),
activation='relu'))
model.add(layers.BatchNormalization())
if third_layer == True:
model.add(
layers.Dense(units=16,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0,
stddev=np.sqrt(2. / hidden_layer_size2),
seed=None),
activation='relu'))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(1,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0, stddev=np.sqrt(2. / 16), seed=None)))
model.compile(optimizer=optimizers.SGD(lr=learning_rate,
decay=decay,
momentum=momentum,
nesterov=True),
loss='mse',
metrics=['mse'])
return model
class MLP(layers.Layer):
"""
MLP as used in Vision Transformer, MLP-Mixer and related networks
"""
k_ini = initializers.TruncatedNormal(stddev=0.02)
b_ini = initializers.Zeros()
def __init__(self, in_features, mlp_ratio=4.0, drop=0., name=None):
super(MLP, self).__init__(name=name)
self.fc1 = layers.Dense(int(in_features * mlp_ratio),
name="fc1",
kernel_initializer=self.k_ini,
bias_initializer=self.b_ini)
self.act = layers.Activation("gelu")
self.fc2 = layers.Dense(in_features,
name="fc2",
kernel_initializer=self.k_ini,
bias_initializer=self.b_ini)
self.drop = layers.Dropout(drop)
def call(self, x, training=None):
x = self.fc1(x)
x = self.act(x)
x = self.drop(x, training=training)
x = self.fc2(x)
x = self.drop(x, training=training)
return x
def __init__(self, class_num, init_stddev=0.02, **kwargs):
super(PreTrainNextSentencePredictor, self).__init__(**kwargs)
self.dense = layers.Dense(
class_num,
kernel_initializer=initializers.TruncatedNormal(
stddev=init_stddev),
)
def __init__(self, conf, nc):
"""
Parameters
----------
conf: Dictionary
Configuration dictionary.
"""
# Initialize.
self.conf = conf
self.raw_data_path = self.conf['raw_data_path']
self.hps = self.conf['hps']
self.nn_arch = self.conf['nn_arch']
super(Module5, self).__init__()
# Design layers.
self.conv3d_1 = Conv3D(
nc,
kernel_size=3,
strides=1,
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal(),
kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
def build(self, input_shape):
# define a parameter table of relative position bias
# [2*Mh-1 * 2*Mw-1, nH]
self.relative_position_bias_table = self.add_weight(
shape=[
(2 * self.window_size[0] - 1) * (2 * self.window_size[1] - 1),
self.num_heads
],
initializer=initializers.TruncatedNormal(stddev=0.02),
trainable=True,
dtype=tf.float32,
name="relative_position_bias_table")
coords_h = np.arange(self.window_size[0])
coords_w = np.arange(self.window_size[1])
coords = np.stack(np.meshgrid(coords_h, coords_w,
indexing="ij")) # [2, Mh, Mw]
coords_flatten = np.reshape(coords, [2, -1]) # [2, Mh*Mw]
# [2, Mh*Mw, 1] - [2, 1, Mh*Mw]
relative_coords = coords_flatten[:, :,
None] - coords_flatten[:,
None, :] # [2, Mh*Mw, Mh*Mw]
relative_coords = np.transpose(relative_coords,
[1, 2, 0]) # [Mh*Mw, Mh*Mw, 2]
relative_coords[:, :,
0] += self.window_size[0] - 1 # shift to start from 0
relative_coords[:, :, 1] += self.window_size[1] - 1
relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
relative_position_index = relative_coords.sum(-1) # [Mh*Mw, Mh*Mw]
self.relative_position_index = tf.Variable(
tf.convert_to_tensor(relative_position_index),
trainable=False,
dtype=tf.int64,
name="relative_position_index")
def __init__(self, conf, rate, nc):
"""
Parameters
----------
conf: Dictionary
Configuration dictionary.
"""
# Initialize.
self.conf = conf
self.raw_data_path = self.conf['raw_data_path']
self.hps = self.conf['hps']
self.nn_arch = self.conf['nn_arch']
super(Module7, self).__init__()
# Design layers.
self.add_1 = Add()
self.conv3d_1 = Conv3D(
nc,
kernel_size=3,
dilation_rate=(rate[0] * self.nn_arch['conv_rate_multiplier'],
rate[1] * self.nn_arch['conv_rate_multiplier'],
rate[2] * self.nn_arch['conv_rate_multiplier']),
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal())
self.bn_1 = BatchNormalization(momentum=self.hps['bn_momentum'],
scale=self.hps['bn_scale'])
self.act_1 = Activation('relu')
def __init__(self, conf, nc):
"""
Parameters
----------
conf: Dictionary
Configuration dictionary.
"""
# Initialize.
self.conf = conf
self.raw_data_path = self.conf['raw_data_path']
self.hps = self.conf['hps']
self.nn_arch = self.conf['nn_arch']
super(Module6, self).__init__()
# Design layers.
self.upsampling3d_1 = UpSampling3D()
self.conv3d_1 = Conv3D(
nc,
kernel_size=3,
strides=1,
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal(),
kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
self.bn_1 = BatchNormalization(momentum=self.hps['bn_momentum'],
scale=self.hps['bn_scale'])
self.act_1 = Activation('relu')
def make_original_like(conv_filters=16, conv_filter_shape=5, nodes=1024):
seq = start_seq()
seq.add(layers.Conv2D(
conv_filters, conv_filter_shape,
padding='same',
kernel_initializer=initializers.TruncatedNormal(stddev=0.1)))
seq.add(layers.MaxPooling2D((2, 2), (2, 2), padding='same'))
seq.add(layers.Flatten())
seq.add(layers.Dense(
nodes,
kernel_initializer=initializers.TruncatedNormal(),
bias_initializer=initializers.zeros(),
activation=activations.relu))
return end_seq(seq)
def __init__(self, n_class, n_layer, root_filters, kernal_size=3, pool_size=2, use_bn=True, use_res=True,
padding='SAME', concat_or_add='concat'):
super().__init__()
self.dw_layers = dict()
self.up_layers = dict()
self.max_pools = dict()
self.dw1_layers = dict()
for layer in range(n_layer):
filters = 2 ** layer * root_filters
dict_key = str(n_layer - layer - 1)
if layer == 0:
Dw1 = _DownSev(filters, 7, 'dw1_%d' % layer)
self.dw1_layers[dict_key] = Dw1
dw = _DownSampling(filters, kernal_size, 'dw_%d' % layer, use_bn, use_res)
self.dw_layers[dict_key] = dw
if layer < n_layer - 1:
pool = layers.MaxPool2D([2, 2], padding=padding)
self.max_pools[dict_key] = pool
for layer in range(n_layer - 2, -1, -1):
filters = 2 ** (layer + 1) * root_filters
dict_key = str(n_layer - layer - 1)
up = _UpSampling(filters, kernal_size, [2, 2], concat_or_add)
self.up_layers[dict_key] = up
stddev = np.sqrt(2 / (kernal_size ** 2 * root_filters))
self.conv_out = layers.Conv2D(n_class, 1, padding=padding, use_bias=False,
kernel_initializer=initializers.TruncatedNormal(stddev=stddev),
name='conv_out')
def conv_batch_lrelu(input_tensor, numfilter, dim, strides=1):
input_tensor = Conv2D(numfilter, (dim, dim), strides=strides, padding='same',
kernel_regularizer=regularizers.l2(0.0005),
kernel_initializer=initializers.TruncatedNormal(stddev=0.1),
use_bias=False
)(input_tensor)
input_tensor = BatchNormalization()(input_tensor)
return LeakyReLU(alpha=0.1)(input_tensor)
def build(self, input_shape):
he_std = self.gain / np.sqrt(np.prod(input_shape[1:], axis=-1)) #?
init_std = 1.0 / self.lrmul
self.runtime_coeff = he_std * self.lrmul
self.kernel_initializer = initializers.TruncatedNormal(
mean=0.0, stddev=init_std) #?
super(EqualizedLRDense, self).build(input_shape)
def __init__(self,
patch_size=4,
num_classes=1000,
embed_dim=96,
depths=(2, 2, 6, 2),
num_heads=(3, 6, 12, 24),
window_size=7,
mlp_ratio=4.,
qkv_bias=True,
drop_rate=0.,
attn_drop_rate=0.,
drop_path_rate=0.1,
norm_layer=layers.LayerNormalization,
name=None,
**kwargs):
super().__init__(name=name)
self.num_classes = num_classes
self.num_layers = len(depths)
self.embed_dim = embed_dim
self.mlp_ratio = mlp_ratio
# split image into non-overlapping patches
self.patch_embed = PatchEmbed(patch_size=patch_size,
embed_dim=embed_dim,
norm_layer=norm_layer)
self.pos_drop = layers.Dropout(drop_rate)
# stochastic depth decay rule
dpr = [x for x in np.linspace(0, drop_path_rate, sum(depths))]
# build layers
self.stage_layers = []
for i_layer in range(self.num_layers):
# 注意这里构建的stage和论文图中有些差异
# 这里的stage不包含该stage的patch_merging层，包含的是下个stage的
layer = BasicLayer(
dim=int(embed_dim * 2**i_layer),
depth=depths[i_layer],
num_heads=num_heads[i_layer],
window_size=window_size,
mlp_ratio=self.mlp_ratio,
qkv_bias=qkv_bias,
drop=drop_rate,
attn_drop=attn_drop_rate,
drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
downsample=PatchMerging if
(i_layer < self.num_layers - 1) else None,
name=f"layer{i_layer}")
self.stage_layers.append(layer)
self.norm = norm_layer(epsilon=1e-6, name="norm")
self.head = layers.Dense(
num_classes,
kernel_initializer=initializers.TruncatedNormal(stddev=0.02),
bias_initializer=initializers.Zeros(),
name="head")
def build_model(self, input_shape, dim, data_dim):
input = Input(shape=input_shape, batch_size=self.batch_size)
x = Dense(dim,
kernel_initializer=initializers.TruncatedNormal(mean=0.,
stddev=0.5),
activation='relu')(input)
x = Dense(dim * 2, activation='relu')(x)
x = Dense(dim * 4, activation='relu')(x)
x = Dense(data_dim)(x)
return Model(inputs=input, outputs=x)
def ConvLayer(ConvIn, NumFilters, FilterSize=4, StrideLength=2, DropOutRate=False, Activation=True,
BatchNormalizationON=True, Padding="same", Alpha=0.2):
WeightsInitializer = initializers.TruncatedNormal(mean=0.0, stddev=0.02, seed=None)
Layer = Conv2D(NumFilters, FilterSize, StrideLength, padding=Padding, kernel_initializer=WeightsInitializer)(ConvIn)
if BatchNormalizationON:
Layer = BatchNormalization()(Layer)
if Activation:
Layer = LeakyReLU(alpha=Alpha)(Layer)
return Layer
def __init__(self,
dim: int,
norm_layer=layers.LayerNormalization,
name=None):
super(PatchMerging, self).__init__(name=name)
self.dim = dim
self.reduction = layers.Dense(
2 * dim,
use_bias=False,
kernel_initializer=initializers.TruncatedNormal(stddev=0.02),
name="reduction")
self.norm = norm_layer(epsilon=1e-6, name="norm")
def __init__(self,
units,
activation='sigmoid',
use_bias=True,
weight_initializer=None,
bias_initializer='zeros',
connection_initializer=None,
connection_regularizer=None,
connection_constraint=None,
round_input=True,
round_connections=True,
clip_connections=True,
round_bias=True,
constrain_after_train=True,
**kwargs):
super(Tea, self).__init__(**kwargs)
self.units = units
self.activation = activations.get(activation)
self.use_bias = use_bias
if connection_initializer:
self.connection_initializer = connection_initializer
else:
self.connection_initializer = initializers.TruncatedNormal(
mean=0.5)
if weight_initializer:
self.weight_initializer = weight_initializer
else:
self.weight_initializer = tea_weight_initializer
self.bias_initializer = bias_initializer
self.connection_regularizer = connection_regularizer
self.connection_constraint = connection_constraint
self.input_width = None
self.round_input = round_input
self.round_connections = round_connections
self.clip_connections = clip_connections
self.round_bias = round_bias
self.constrain_after_train = constrain_after_train
self.uses_learning_phase = True
def build_model(learning_rate=0.001,
decay=10e-6,
momentum=0.9,
l2_parameter=0.1):
model = models.Sequential()
model.add(
layers.Dense(units=256,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0, stddev=np.sqrt(2. / 1024), seed=None),
activation='relu',
input_shape=(1024, )))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(units=64,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0, stddev=np.sqrt(2. / 256), seed=None),
activation='relu'))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(units=16,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0, stddev=np.sqrt(2. / 128), seed=None),
activation='relu'))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(1,
kernel_regularizer=regularizers.l2(l2_parameter),
kernel_initializer=initializers.TruncatedNormal(
mean=0.0, stddev=np.sqrt(2. / 64), seed=None)))
model.compile(optimizer=optimizers.SGD(lr=learning_rate,
decay=decay,
momentum=0.9,
nesterov=True),
loss='mse',
metrics=['mse'])
return model
def CNN(inputs, filter_size, pool_size):
conv_initializer = initializers.TruncatedNormal(mean=0., stddev=0.12)
fc_initializer = initializers.TruncatedNormal(mean=0., stddev=0.06)
x = Conv1D(filters=32,
kernel_size=filter_size,
padding='valid',
kernel_regularizer=regularizers.l2(5e-5),
bias_regularizer=regularizers.l2(5e-5),
kernel_initializer=conv_initializer)(inputs)
x = GroupNormalization(groups=4, axis=-1)(x)
x = Activation('relu')(x)
#x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling1D(pool_size=pool_size)(x)
x = Dropout(0.25)(x)
x = Flatten()(x)
x = Dense(64,
kernel_regularizer=regularizers.l2(5e-4),
bias_regularizer=regularizers.l2(5e-4),
kernel_initializer=fc_initializer)(x)
x = Activation('relu')(x)
#x = LeakyReLU(alpha=0.1)(x)
x = Dropout(0.25)(x)
return x
def ConvTransLayer(ConvTransIn, NumFilters, FilterSize=4, StrideLength=2, DropOutRate=False, convOut=None,
Activation=True, BatchNormalizationON=True, Padding="same", Alpha=0.2):
WeightsInitializer = initializers.TruncatedNormal(mean=0.0, stddev=0.02, seed=None)
Layer = Conv2DTranspose(NumFilters, FilterSize, StrideLength, padding='same',
kernel_initializer=WeightsInitializer)(
concatenate([ConvTransIn, convOut]) if convOut is not None else ConvTransIn)
if BatchNormalizationON:
Layer = BatchNormalization()(Layer)
if Activation:
Layer = LeakyReLU(alpha=Alpha)(Layer)
if DropOutRate:
Layer = Dropout(rate=DropOutRate)(Layer)
return Layer
def __init__(self, conf, nc):
"""
Parameters
----------
conf: Dictionary
Configuration dictionary.
"""
# Initialize.
self.conf = conf
self.raw_data_path = self.conf['raw_data_path']
self.hps = self.conf['hps']
self.nn_arch = self.conf['nn_arch']
super(Module3, self).__init__()
# Design layers.
self.global_avg_pool3d_1 = GlobalAveragePooling3D() # data_format?
self.reshape_1 = Reshape((1, 1, 1, nc))
self.conv3d_1 = Conv3D(
np.maximum(1, int(nc / 2)),
kernel_size=1,
strides=1,
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal(),
kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
self.conv3d_2 = Conv3D(
nc,
kernel_size=1,
strides=1,
padding='same',
use_bias=False,
kernel_initializer=initializers.TruncatedNormal(),
kernel_regularizer=regularizers.l2(self.hps['weight_decay']))
def conv_block(net, style_weights, filters, kernel_size, strides, activation,
depthwise_separable_conv):
"""
first applies padding with mode=REFLECT,
then a valid conv2d,
then conditional instance norm using style_weights
and finally, activation
TODO: add support to choose between regular and depthwise separable convolutions
"""
pad_0 = (kernel_size[0] - 1) // 2
pad_1 = (kernel_size[1] - 1) // 2
net = Lambda(
lambda t: tf.pad(t, [[0, 0], [pad_0, pad_0], [pad_1, pad_1], [0, 0]],
mode="REFLECT"))(net)
weight_initializer = initializers.TruncatedNormal(mean=0.0,
stddev=0.1,
seed=1)
if depthwise_separable_conv:
net = SeparableConv2D(
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding="valid",
activation=None,
depthwise_initializer=weight_initializer,
pointwise_initializer=weight_initializer,
)(net)
else:
net = Conv2D(
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding="valid",
activation=None,
kernel_initializer=weight_initializer,
)(net)
net = ConditionalInstanceNormalization(
style_weights.shape[-1])([net, style_weights])
if activation != None:
net = Activation(activation)(net)
return net
def build_Embedded(self, action_space=6, dueling=True):
self.network_size = 256
X_input = Input(shape=(self.REM_STEP*99) )
# X_input = Input(shape=(self.REM_STEP*7,))
input_reshape=(self.REM_STEP,99)
X = X_input
truncatedn_init = initializers.TruncatedNormal(0, 1e-2)
x_init = "he_uniform"
y_init = initializers.glorot_uniform()
const_init = initializers.constant(1e-2)
X_reshaped = Reshape(input_reshape)(X_input)
X = LayerNormalization(axis=2)(X_reshaped)
X = LayerNormalization(axis=1)(X)
X = Reshape((2,-1))(X)
X = Dense(self.network_size*2, kernel_initializer=y_init, activation ="relu")(X)
X = Dense(256, kernel_initializer=y_init, activation ="relu")(X)
X = TimeDistributed(Dense(self.network_size/4, kernel_initializer=y_init,activation ="relu"))(X)
X = Flatten()(X)
if dueling:
state_value = Dense(
1, activation ="softmax")(X)
state_value = Lambda(lambda s: K.expand_dims(
s[:, 0], -1), output_shape=(action_space,))(state_value)
action_advantage = Dense(
action_space, activation ="linear") (X)
action_advantage = Lambda(lambda a: a[:, :] - K.mean(
a[:, :], keepdims=True), output_shape=(action_space,))(action_advantage)
X = Add()([state_value, action_advantage])
else:
# Output Layer with # of actions: 2 nodes (left, right)
X = Dense(action_space, activation="relu",
kernel_initializer='he_uniform')(X)
model = Model(inputs=X_input, outputs=X, name='build_Embedded')
model.compile(loss=huber_loss, optimizer=Adam(
lr=self.learning_rate), metrics=["accuracy"])
# model.compile(loss="mean_squared_error", optimizer=Adam(lr=0.00025,epsilon=0.01), metrics=["accuracy"])
model.summary()
return model
def _conv_transpose_block(net, filters, kernel_size, strides, padding,
activation):
# https://github.com/lengstrom/fast-style-transfer/blob/master/src/transform.py#L59-L67
weights_initializer = initializers.TruncatedNormal(mean=0.0,
stddev=0.1,
seed=1)
net = Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
kernel_initializer=weights_initializer)(net)
net = InstanceNormalization()(net)
if activation != None:
net = Activation(activation)(net)
return net
def PolyA_CNN(length, seq_kernel_size=6, cov_kernel_size=12, pool_size=6):
input_shape1 = (length, 4)
input_shape2 = (length, 1)
seq_input = Input(shape=input_shape1, name="seq_input")
cov_input = Input(shape=input_shape2, name="cov_input")
fc_initializer = initializers.TruncatedNormal(mean=0., stddev=0.06)
#
x = CNN(seq_input, seq_kernel_size, pool_size)
y = CNN(cov_input, cov_kernel_size, pool_size)
input_layers = [seq_input, cov_input]
outLayer = Dense(1,
kernel_regularizer=regularizers.l2(5e-4),
bias_regularizer=regularizers.l2(5e-4),
kernel_initializer=fc_initializer,
activation='sigmoid')(concatenate([x, y]))
model = Model(inputs=input_layers, outputs=outLayer)
return model
