def end_seq(seq):
seq.add(layers.Dense(
10,
kernel_initializer=initializers.zeros(),
bias_initializer=initializers.zeros(),
activation=activations.softmax))
return seq
def build(self, input_shape):
input_dimension = input_shape[-1]
units1 = self.output_dimension * self.number_of_mixtures
self.mu_kernel = self.add_weight(name="mu_kernel",
shape = shape(input_dimension, units1),
initializer=initializers.random_normal(),
trainable=True)
self.mu_bias = self.add_weight(name="mu_bias",
shape = shape(units1),
initializer=initializers.zeros(),
trainable=True)
self.sigma_kernel = self.add_weight(name="sigma_kernel",
shape = shape(input_dimension, units1),
initializer=initializers.random_normal(),
trainable=True)
self.sigma_bias = self.add_weight(name="sigma_bias",
shape = shape(units1),
initializer=initializers.zeros(),
trainable=True)
units2 = self.number_of_mixtures
self.pi_kernel = self.add_weight(name="pi_kernel",
shape = shape(input_dimension, units2),
initializer=initializers.random_normal(),
trainable=True)
self.pi_bias = self.add_weight(name="pi_bias",
shape = shape(units2),
initializer=initializers.zeros(),
trainable=True)
def create_mdrnn(self, direction):
kernel_initializer = initializers.zeros()
recurrent_initializer = initializers.constant(2)
bias_initializer = initializers.zeros()
return MDRNN(units=1, input_shape=(None, 1),
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=bias_initializer,
activation=None,
return_sequences=True,
direction=direction)
def __init__(self, **kwargs):
super().__init__()
# should be L1=100 L2=513 z_dim=20
self.L1 = kwargs['L1']
self.L2 = kwargs['L2']
self.z_dim = kwargs['z_dim']
# encoder
self.fc1 = layers.Dense(self.L1)
self.fc21 = layers.Dense(self.z_dim,
kernel_initializer=initializers.zeros())
self.fc22 = layers.Dense(self.z_dim,
kernel_initializer=initializers.zeros())
# decoder
self.fc3 = layers.Dense(self.L2)
def state_to_action_net_zero(inp_shape, filter_layers):
model = Sequential(name='target')
for i in range(len(filter_layers)):
if i == 0:
model.add(
Conv2D(filter_layers[i],
kernel_size=4,
padding='same',
use_bias=False,
input_shape=inp_shape))
model.add(LayerNormalization())
model.add(ReLU())
else:
model.add(
Conv2D(filter_layers[i],
kernel_size=4,
padding='same',
use_bias=False))
model.add(LayerNormalization())
model.add(ReLU())
model.add(Flatten())
model.add(Dropout(0.1))
model.add(Dense(128, use_bias=True))
model.add(Dropout(0.1))
model.add(Dense(7, use_bias=True, kernel_initializer=init.zeros()))
return model
def initialize(self):
w_initializer = glorot_uniform()
zeros_initializer = zeros()
for w, wv in zip(self.weights, self.velocities):
w.assign(w_initializer(w.shape, dtype=self.floatx))
wv.assign(zeros_initializer(wv.shape, dtype=self.floatx))
def get_conv(filters, kernel_size=3, activation=tf.nn.relu):
return Conv2D(filters=filters,
kernel_size=kernel_size,
padding='same',
activation=activation,
data_format='channels_first',
kernel_initializer=initializers.glorot_normal(),
bias_initializer=initializers.zeros(),
kernel_regularizer=l2(0.001),
bias_regularizer=l2(0.001))
def build(self, input_shape):
kernel_shape_f_g = (1, 1) + (self.number_of_channels,
self.number_of_filters_f_g)
kernel_shape_h = (1, 1) + (self.number_of_channels,
self.number_of_filters_h)
self.gamma = self.add_weight(shape=[1],
initializer=initializers.zeros(),
trainable=True,
name="gamma")
self.kernel_f = self.add_weight(
shape=kernel_shape_f_g,
initializer=initializers.glorot_uniform(),
trainable=True,
name="kernel_f")
self.kernel_g = self.add_weight(
shape=kernel_shape_f_g,
initializer=initializers.glorot_uniform(),
trainable=True,
name="kernel_g")
self.kernel_h = self.add_weight(
shape=kernel_shape_h,
initializer=initializers.glorot_uniform(),
trainable=True,
name="kernel_h")
self.bias_f = self.add_weight(shape=(self.number_of_filters_f_g, ),
initializer=initializers.zeros(),
trainable=True,
name="bias_f")
self.bias_g = self.add_weight(shape=(self.number_of_filters_f_g, ),
initializer=initializers.zeros(),
trainable=True,
name="bias_g")
self.bias_h = self.add_weight(shape=(self.number_of_filters_h, ),
initializer=initializers.zeros(),
trainable=True,
name="bias_h")
super(AttentionLayer2D, self).build(input_shape)
self.input_spec = InputSpec(ndim=4, axes={3: input_shape[-1]})
self.built = True
def build(self, input_shape):
if not isinstance(input_shape, list):
input_shape = [input_shape]
self.biases = [
self.add_weight(
name="eabias_{}".format(str(i)),
shape=(shape[-2], ),
regularizer=regularizers.l2(self.bias_lambda),
initializer=initializers.zeros(),
) for i, shape in enumerate(input_shape)
]
self.built = True
def action_block(inputs, hidden_layers, zero=False):
x = inputs
for i in range(len(hidden_layers)):
x = Dense(hidden_layers[i])(x)
x = Dropout(0.1)(x)
x = ReLU()(x)
if not zero:
x = Dense(1)(x)
else:
x = Dense(1, kernel_initializer=init.zeros())(x)
return x
def make_model(filters=160,
blocks=8,
kernels=(5, 1),
rate=0.001,
freeze_batch_norm=False):
input = Input(shape=(NUM_INPUT_CHANNELS, 8, 8), name='input')
# initial convolution
x = get_conv(filters=filters, kernel_size=kernels[0])(input)
# residual blocks
for i in range(blocks):
x = get_residual_block(x, freeze_batch_norm, i)
# value tower
vt = Flatten()(x)
vt = get_dense(40, regu=0.02)(vt)
vt = Dropout(rate=0.5)(vt)
vt = get_norm(freeze_batch_norm, 'batchnorm-vt')(vt)
vt = get_dense(20, regu=0.04)(vt)
vt = Dropout(rate=0.5)(vt)
value = Dense(1,
activation=tf.nn.tanh,
name='value',
kernel_initializer=initializers.glorot_normal(),
bias_initializer=initializers.zeros(),
bias_regularizer=l2(0.2),
kernel_regularizer=l2(0.4),
activity_regularizer=l2(0.1))(vt)
px = get_conv(filters=8 * 8, activation=None, kernel_size=kernels[1])(x)
pf = Flatten()(px)
policy = Softmax(name='policy')(pf)
model = Model(inputs=input, outputs=[value, policy])
losses = {
'value': 'mean_squared_error',
'policy': 'categorical_crossentropy'
}
weights = {'value': 1.0, 'policy': 1.0}
optimizer = Adam(rate)
model.compile(optimizer=optimizer,
loss=losses,
loss_weights=weights,
metrics=[])
print('Model parameters: %d' % model.count_params())
return model
def make_original_like(conv_filters=16, conv_filter_shape=5, nodes=1024):
seq = start_seq()
seq.add(layers.Conv2D(
conv_filters, conv_filter_shape,
padding='same',
kernel_initializer=initializers.TruncatedNormal(stddev=0.1)))
seq.add(layers.MaxPooling2D((2, 2), (2, 2), padding='same'))
seq.add(layers.Flatten())
seq.add(layers.Dense(
nodes,
kernel_initializer=initializers.TruncatedNormal(),
bias_initializer=initializers.zeros(),
activation=activations.relu))
return end_seq(seq)
def build(self, hids):
hids = gf.repeat(hids)
weights = []
zeros_initializer = zeros()
pre_hid = self.num_attrs
for hid in hids + [self.num_classes]:
shape = (pre_hid, hid)
# use zeros_initializer temporary to save time
weight = tf.Variable(zeros_initializer(shape=shape, dtype=self.floatx))
weights.append(weight)
pre_hid = hid
self.weights = weights
self.adj_grad_sum = tf.Variable(tf.zeros_like(self.adj_tensor))
self.x_grad_sum = tf.Variable(tf.zeros_like(self.x_tensor))
self.optimizer = Adam(self.lr, epsilon=1e-8)
def initialize(self):
w_initializer = glorot_uniform()
zeros_initializer = zeros()
for w in self.weights:
w.assign(w_initializer(w.shape, dtype=self.floatx))
if self.structure_attack:
self.adj_grad_sum.assign(zeros_initializer(self.adj_grad_sum.shape, dtype=self.floatx))
if self.feature_attack:
self.x_grad_sum.assign(zeros_initializer(self.x_grad_sum.shape, dtype=self.floatx))
# reset optimizer
for var in self.optimizer.variables():
var.assign(tf.zeros_like(var))
def build(self, input_shape):
self.W = []
self.b = []
for i in range(self.layer_num):
self.W.append(
self.add_weight(name='kernel',
shape=(int(input_shape[-1]), 1),
initializer=glorot_normal(),
regularizer=l2(0),
trainable=True))
self.b.append(
self.add_weight(name='bias',
shape=(int(input_shape[-1]), 1),
initializer=zeros(),
trainable=True))
super(CrossLayer, self).build(input_shape)
def build(self, hids):
hids = gf.repeat(hids)
weights, w_velocities = [], []
zeros_initializer = zeros()
pre_hid = self.num_attrs
for hid in hids + [self.num_classes]:
shape = (pre_hid, hid)
# use zeros_initializer temporary to save time
weight = tf.Variable(zeros_initializer(shape=shape, dtype=self.floatx))
w_velocity = tf.Variable(zeros_initializer(shape=shape, dtype=self.floatx))
weights.append(weight)
w_velocities.append(w_velocity)
pre_hid = hid
self.weights, self.w_velocities = weights, w_velocities
def build(self, hidden_layers):
weights, velocities = [], []
zeros_initializer = zeros()
pre_hid = self.n_attrs
for hid in hidden_layers + [self.n_classes]:
shape = (pre_hid, hid)
# use zeros_initializer temporary to save time
weight = tf.Variable(
zeros_initializer(shape=shape, dtype=self.floatx))
w_velocity = tf.Variable(
zeros_initializer(shape=shape, dtype=self.floatx))
weights.append(weight)
velocities.append(w_velocity)
pre_hid = hid
self.weights, self.velocities = weights, velocities
def res_model_zero(initial_filters, num_res, res_struc, dense):
inputs = Input(shape=(6, 7, 3))
for i in range(len(initial_filters)):
if i == 0:
x = Conv2D(initial_filters[i], 3, padding='same')(inputs)
x = LayerNormalization()(x)
x = ReLU()(x)
else:
x = Conv2D(initial_filters[i], 3, padding='same')(x)
x = LayerNormalization()(x)
x = ReLU()(x)
for j in range(num_res):
x = res_block(x, res_struc, 3)
x = Conv2D(1, 1, padding='same')
x = LayerNormalization()(x)
x = Flatten()(x)
for k in range(len(dense)):
x = Dense(dense[k])(x)
x = ReLU()(x)
x = Dense(7, kernel_initializer=init.zeros())(x)
return Model(inputs, x)
def get_initializers(activation):
"""Returns the kernel and bias initializers for the given activation
function.
Parameters
----------
activation: str
Activation function for a given layer
Returns
-------
kernel_initializer: tf.keras.initializers
Kernel initializer for the given activation function
bias_initializer: tf.keras.initializers
Bias initializer for the given activation function
"""
if activation == "relu":
kernel_initializer = he_normal()
bias_initializer = constant(value=0.01)
else:
kernel_initializer = glorot_normal()
bias_initializer = zeros()
return kernel_initializer, bias_initializer
def build(self, input_shapes):
"""
Builds the layer
Args:
input_shapes (list of int): shapes of the layer's inputs (node features and adjacency matrix)
"""
feat_shape = input_shapes[0]
input_dim = int(feat_shape[-1])
# Variables to support integrated gradients
self.delta = self.add_weight(name="ig_delta",
shape=(),
trainable=False,
initializer=initializers.ones())
self.non_exist_edge = self.add_weight(
name="ig_non_exist_edge",
shape=(),
trainable=False,
initializer=initializers.zeros(),
)
# Initialize weights for each attention head
for head in range(self.attn_heads):
# Layer kernel
kernel = self.add_weight(
shape=(input_dim, self.units),
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
name="kernel_{}".format(head),
)
self.kernels.append(kernel)
# # Layer bias
if self.use_bias:
bias = self.add_weight(
shape=(self.units, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
name="bias_{}".format(head),
)
self.biases.append(bias)
# Attention kernels
attn_kernel_self = self.add_weight(
shape=(self.units, 1),
initializer=self.attn_kernel_initializer,
regularizer=self.attn_kernel_regularizer,
constraint=self.attn_kernel_constraint,
name="attn_kernel_self_{}".format(head),
)
attn_kernel_neighs = self.add_weight(
shape=(self.units, 1),
initializer=self.attn_kernel_initializer,
regularizer=self.attn_kernel_regularizer,
constraint=self.attn_kernel_constraint,
name="attn_kernel_neigh_{}".format(head),
)
self.attn_kernels.append([attn_kernel_self, attn_kernel_neighs])
self.built = True
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras import initializers
import numpy as np
import keras.backend as k
import random
# Define model
initializer = initializers.VarianceScaling(scale=1,mode='fan_avg',distribution='uniform',seed=None)
bias_init = initializers.zeros()
inputs = tf.keras.Input(shape=(260,344,1), dtype='float32', sparse=False)
conv1 = layers.Conv2D(filters=16,kernel_size=(3,3),strides=(1,1),
padding='same',data_format='channels_last',dilation_rate=(1,1),
use_bias=False, kernel_initializer=initializer, bias_initializer=bias_init,
kernel_regularizer=None,bias_regularizer=None,activity_regularizer=None,
activation = 'linear',kernel_constraint = None, bias_constraint=None)(inputs)
activation1 = layers.Activation(trainable=True, dtype='float32', activation='relu')(conv1)
pool1 = layers.MaxPooling2D(pool_size=(2,2), padding='valid', strides=(2,2),
data_format='channels_last')(activation1)
conv2a = layers.Conv2D(filters=32,kernel_size=(3,3),strides=(1,1),
padding='same',data_format='channels_last',dilation_rate=(1,1),
activation='linear',use_bias=False, kernel_initializer=initializer,
bias_initializer=bias_init,kernel_regularizer=None,
bias_regularizer=None,activity_regularizer=None,kernel_constraint=None,
bias_constraint=None)(pool1)
def build_model(self):
input_shape = (self.seq_length,)
inputs = Input(shape=input_shape, dtype='int32')
embedding = layers.Embedding(
len(self.index_to_vector_map),
self.embedding_dimensions,
input_length=self.seq_length,
trainable=self.embedding_training
)
embedding.build((None, len(self.index_to_vector_map)))
embedding.set_weights([self.index_to_vector_map])
concatenate = layers.Concatenate(axis=-1)
flatten = layers.Flatten(data_format='channels_last')
dropout = layers.Dropout(self.dropout_rate)
dense = layers.Dense(self.num_classes, activation='softmax')
pooled_outputs = []
x_embedding = embedding(inputs)
x_embedding = tf.expand_dims(x_embedding, -1)
for filter_size in self.filter_sizes:
filter_shape = (filter_size, self.embedding_dimensions)
conv2d = layers.Conv2D(
self.num_filters_per_size,
filter_shape,
1,
padding='VALID',
activation=self.activation_fn,
kernel_initializer=initializers.RandomUniform(minval=-0.01, maxval=0.01),
bias_initializer=initializers.zeros(),
use_bias=True,
data_format='channels_last',
input_shape=(self.seq_length, self.embedding_dimensions, 1)
)
x = conv2d(x_embedding)
x = tf.nn.max_pool(
x,
ksize=[1, self.seq_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID'
)
x = flatten(x)
pooled_outputs.append(x)
if len(pooled_outputs) == 1:
pooled_outputs = tf.convert_to_tensor(pooled_outputs[0])
else:
pooled_outputs = concatenate(pooled_outputs)
pooled_outputs = dropout(pooled_outputs)
pooled_outputs = dense(pooled_outputs)
model = Model(inputs=inputs, outputs=pooled_outputs, name='sentence_cnn')
model.compile(
loss='categorical_crossentropy',
optimizer=optimizers.Adadelta(
learning_rate=self.learning_rate,
rho=self.lr_decay,
epsilon=1e-6,
),
metrics=['accuracy']
)
return model
plt.show()
model = Sequential([
Dense(
units=4,
input_shape=(4, ),
activation=relu,
trainable=False, # to freeze the layer
kernel_initializer=random_uniform(),
bias_initializer=ones()),
Dense(units=2,
activation=relu,
kernel_initializer=lecun_uniform(),
bias_initializer=zeros()),
Dense(units=4, activation=softmax)
])
model.summary()
W0_layers = get_weights(model)
b0_layers = get_biases(model)
X_train = np.random.random((100, 4))
y_train = X_train
X_test = np.random.random((20, 4))
y_test = X_test
model.get_layer(
def model_builder(self,
filter_size: int = 5,
seed_val: int = 123,
**kwargs) -> tf.keras.Sequential:
"""
Build and compile a 1D-CNN depending on the given hyper params (parameters_hyper.yaml).
Kwargs require a dict like below.
{
"conv1_length": int,
"conv2_length": int,
"extra_conv_layer": bool,
"conv3_length": int,
"dense1_length": int
}
"""
he_norm = he_normal(seed=seed_val)
bias_val = zeros()
model = models.Sequential()
model.add(
layers.Conv1D(filters=kwargs["conv1_length"],
kernel_size=filter_size,
strides=1,
padding="same",
use_bias=True,
input_shape=self.data.shape[1:],
kernel_initializer=he_norm,
bias_initializer=bias_val,
activation="relu"))
model.add(layers.MaxPool1D())
model.add(
layers.Conv1D(filters=kwargs["conv2_length"],
kernel_size=ceil(filter_size / 2),
strides=1,
padding="same",
use_bias=True,
kernel_initializer=he_norm,
bias_initializer=bias_val,
activation="relu"))
model.add(layers.MaxPool1D())
if kwargs["extra_conv_layer"]:
model.add(
layers.Conv1D(filters=kwargs["conv3_length"],
kernel_size=ceil(filter_size / 2),
strides=1,
padding="same",
use_bias=True,
kernel_initializer=he_norm,
bias_initializer=bias_val,
activation="relu"))
model.add(layers.MaxPool1D())
model.add(layers.Flatten())
model.add(
layers.Dense(units=kwargs["dense1_length"],
use_bias=True,
kernel_initializer=he_norm,
bias_initializer=bias_val,
activation="relu"))
model.add(
layers.Dense(units=1,
use_bias=True,
kernel_initializer=he_norm,
bias_initializer=bias_val,
activation="relu"))
model.compile(optimizer=Adam(
learning_rate=parameters.General_Params().initial_lr),
loss=MeanAbsolutePercentageError(name="MAPE"),
metrics=[
MeanAbsoluteError(name="MAE"),
RootMeanSquaredError(name="RMSE")
])
return model
