def end_seq(seq): seq.add(layers.Dense( 10, kernel_initializer=initializers.zeros(), bias_initializer=initializers.zeros(), activation=activations.softmax)) return seq
def build(self, input_shape): input_dimension = input_shape[-1] units1 = self.output_dimension * self.number_of_mixtures self.mu_kernel = self.add_weight(name="mu_kernel", shape = shape(input_dimension, units1), initializer=initializers.random_normal(), trainable=True) self.mu_bias = self.add_weight(name="mu_bias", shape = shape(units1), initializer=initializers.zeros(), trainable=True) self.sigma_kernel = self.add_weight(name="sigma_kernel", shape = shape(input_dimension, units1), initializer=initializers.random_normal(), trainable=True) self.sigma_bias = self.add_weight(name="sigma_bias", shape = shape(units1), initializer=initializers.zeros(), trainable=True) units2 = self.number_of_mixtures self.pi_kernel = self.add_weight(name="pi_kernel", shape = shape(input_dimension, units2), initializer=initializers.random_normal(), trainable=True) self.pi_bias = self.add_weight(name="pi_bias", shape = shape(units2), initializer=initializers.zeros(), trainable=True)
def create_mdrnn(self, direction): kernel_initializer = initializers.zeros() recurrent_initializer = initializers.constant(2) bias_initializer = initializers.zeros() return MDRNN(units=1, input_shape=(None, 1), kernel_initializer=kernel_initializer, recurrent_initializer=recurrent_initializer, bias_initializer=bias_initializer, activation=None, return_sequences=True, direction=direction)
def __init__(self, **kwargs): super().__init__() # should be L1=100 L2=513 z_dim=20 self.L1 = kwargs['L1'] self.L2 = kwargs['L2'] self.z_dim = kwargs['z_dim'] # encoder self.fc1 = layers.Dense(self.L1) self.fc21 = layers.Dense(self.z_dim, kernel_initializer=initializers.zeros()) self.fc22 = layers.Dense(self.z_dim, kernel_initializer=initializers.zeros()) # decoder self.fc3 = layers.Dense(self.L2)
def state_to_action_net_zero(inp_shape, filter_layers): model = Sequential(name='target') for i in range(len(filter_layers)): if i == 0: model.add( Conv2D(filter_layers[i], kernel_size=4, padding='same', use_bias=False, input_shape=inp_shape)) model.add(LayerNormalization()) model.add(ReLU()) else: model.add( Conv2D(filter_layers[i], kernel_size=4, padding='same', use_bias=False)) model.add(LayerNormalization()) model.add(ReLU()) model.add(Flatten()) model.add(Dropout(0.1)) model.add(Dense(128, use_bias=True)) model.add(Dropout(0.1)) model.add(Dense(7, use_bias=True, kernel_initializer=init.zeros())) return model
def initialize(self): w_initializer = glorot_uniform() zeros_initializer = zeros() for w, wv in zip(self.weights, self.velocities): w.assign(w_initializer(w.shape, dtype=self.floatx)) wv.assign(zeros_initializer(wv.shape, dtype=self.floatx))
def get_conv(filters, kernel_size=3, activation=tf.nn.relu): return Conv2D(filters=filters, kernel_size=kernel_size, padding='same', activation=activation, data_format='channels_first', kernel_initializer=initializers.glorot_normal(), bias_initializer=initializers.zeros(), kernel_regularizer=l2(0.001), bias_regularizer=l2(0.001))
def build(self, input_shape): kernel_shape_f_g = (1, 1) + (self.number_of_channels, self.number_of_filters_f_g) kernel_shape_h = (1, 1) + (self.number_of_channels, self.number_of_filters_h) self.gamma = self.add_weight(shape=[1], initializer=initializers.zeros(), trainable=True, name="gamma") self.kernel_f = self.add_weight( shape=kernel_shape_f_g, initializer=initializers.glorot_uniform(), trainable=True, name="kernel_f") self.kernel_g = self.add_weight( shape=kernel_shape_f_g, initializer=initializers.glorot_uniform(), trainable=True, name="kernel_g") self.kernel_h = self.add_weight( shape=kernel_shape_h, initializer=initializers.glorot_uniform(), trainable=True, name="kernel_h") self.bias_f = self.add_weight(shape=(self.number_of_filters_f_g, ), initializer=initializers.zeros(), trainable=True, name="bias_f") self.bias_g = self.add_weight(shape=(self.number_of_filters_f_g, ), initializer=initializers.zeros(), trainable=True, name="bias_g") self.bias_h = self.add_weight(shape=(self.number_of_filters_h, ), initializer=initializers.zeros(), trainable=True, name="bias_h") super(AttentionLayer2D, self).build(input_shape) self.input_spec = InputSpec(ndim=4, axes={3: input_shape[-1]}) self.built = True
def build(self, input_shape): if not isinstance(input_shape, list): input_shape = [input_shape] self.biases = [ self.add_weight( name="eabias_{}".format(str(i)), shape=(shape[-2], ), regularizer=regularizers.l2(self.bias_lambda), initializer=initializers.zeros(), ) for i, shape in enumerate(input_shape) ] self.built = True
def action_block(inputs, hidden_layers, zero=False): x = inputs for i in range(len(hidden_layers)): x = Dense(hidden_layers[i])(x) x = Dropout(0.1)(x) x = ReLU()(x) if not zero: x = Dense(1)(x) else: x = Dense(1, kernel_initializer=init.zeros())(x) return x
def make_model(filters=160, blocks=8, kernels=(5, 1), rate=0.001, freeze_batch_norm=False): input = Input(shape=(NUM_INPUT_CHANNELS, 8, 8), name='input') # initial convolution x = get_conv(filters=filters, kernel_size=kernels[0])(input) # residual blocks for i in range(blocks): x = get_residual_block(x, freeze_batch_norm, i) # value tower vt = Flatten()(x) vt = get_dense(40, regu=0.02)(vt) vt = Dropout(rate=0.5)(vt) vt = get_norm(freeze_batch_norm, 'batchnorm-vt')(vt) vt = get_dense(20, regu=0.04)(vt) vt = Dropout(rate=0.5)(vt) value = Dense(1, activation=tf.nn.tanh, name='value', kernel_initializer=initializers.glorot_normal(), bias_initializer=initializers.zeros(), bias_regularizer=l2(0.2), kernel_regularizer=l2(0.4), activity_regularizer=l2(0.1))(vt) px = get_conv(filters=8 * 8, activation=None, kernel_size=kernels[1])(x) pf = Flatten()(px) policy = Softmax(name='policy')(pf) model = Model(inputs=input, outputs=[value, policy]) losses = { 'value': 'mean_squared_error', 'policy': 'categorical_crossentropy' } weights = {'value': 1.0, 'policy': 1.0} optimizer = Adam(rate) model.compile(optimizer=optimizer, loss=losses, loss_weights=weights, metrics=[]) print('Model parameters: %d' % model.count_params()) return model
def make_original_like(conv_filters=16, conv_filter_shape=5, nodes=1024): seq = start_seq() seq.add(layers.Conv2D( conv_filters, conv_filter_shape, padding='same', kernel_initializer=initializers.TruncatedNormal(stddev=0.1))) seq.add(layers.MaxPooling2D((2, 2), (2, 2), padding='same')) seq.add(layers.Flatten()) seq.add(layers.Dense( nodes, kernel_initializer=initializers.TruncatedNormal(), bias_initializer=initializers.zeros(), activation=activations.relu)) return end_seq(seq)
def build(self, hids): hids = gf.repeat(hids) weights = [] zeros_initializer = zeros() pre_hid = self.num_attrs for hid in hids + [self.num_classes]: shape = (pre_hid, hid) # use zeros_initializer temporary to save time weight = tf.Variable(zeros_initializer(shape=shape, dtype=self.floatx)) weights.append(weight) pre_hid = hid self.weights = weights self.adj_grad_sum = tf.Variable(tf.zeros_like(self.adj_tensor)) self.x_grad_sum = tf.Variable(tf.zeros_like(self.x_tensor)) self.optimizer = Adam(self.lr, epsilon=1e-8)
def initialize(self): w_initializer = glorot_uniform() zeros_initializer = zeros() for w in self.weights: w.assign(w_initializer(w.shape, dtype=self.floatx)) if self.structure_attack: self.adj_grad_sum.assign(zeros_initializer(self.adj_grad_sum.shape, dtype=self.floatx)) if self.feature_attack: self.x_grad_sum.assign(zeros_initializer(self.x_grad_sum.shape, dtype=self.floatx)) # reset optimizer for var in self.optimizer.variables(): var.assign(tf.zeros_like(var))
def build(self, input_shape): self.W = [] self.b = [] for i in range(self.layer_num): self.W.append( self.add_weight(name='kernel', shape=(int(input_shape[-1]), 1), initializer=glorot_normal(), regularizer=l2(0), trainable=True)) self.b.append( self.add_weight(name='bias', shape=(int(input_shape[-1]), 1), initializer=zeros(), trainable=True)) super(CrossLayer, self).build(input_shape)
def build(self, hids): hids = gf.repeat(hids) weights, w_velocities = [], [] zeros_initializer = zeros() pre_hid = self.num_attrs for hid in hids + [self.num_classes]: shape = (pre_hid, hid) # use zeros_initializer temporary to save time weight = tf.Variable(zeros_initializer(shape=shape, dtype=self.floatx)) w_velocity = tf.Variable(zeros_initializer(shape=shape, dtype=self.floatx)) weights.append(weight) w_velocities.append(w_velocity) pre_hid = hid self.weights, self.w_velocities = weights, w_velocities
def build(self, hidden_layers): weights, velocities = [], [] zeros_initializer = zeros() pre_hid = self.n_attrs for hid in hidden_layers + [self.n_classes]: shape = (pre_hid, hid) # use zeros_initializer temporary to save time weight = tf.Variable( zeros_initializer(shape=shape, dtype=self.floatx)) w_velocity = tf.Variable( zeros_initializer(shape=shape, dtype=self.floatx)) weights.append(weight) velocities.append(w_velocity) pre_hid = hid self.weights, self.velocities = weights, velocities
def res_model_zero(initial_filters, num_res, res_struc, dense): inputs = Input(shape=(6, 7, 3)) for i in range(len(initial_filters)): if i == 0: x = Conv2D(initial_filters[i], 3, padding='same')(inputs) x = LayerNormalization()(x) x = ReLU()(x) else: x = Conv2D(initial_filters[i], 3, padding='same')(x) x = LayerNormalization()(x) x = ReLU()(x) for j in range(num_res): x = res_block(x, res_struc, 3) x = Conv2D(1, 1, padding='same') x = LayerNormalization()(x) x = Flatten()(x) for k in range(len(dense)): x = Dense(dense[k])(x) x = ReLU()(x) x = Dense(7, kernel_initializer=init.zeros())(x) return Model(inputs, x)
def get_initializers(activation): """Returns the kernel and bias initializers for the given activation function. Parameters ---------- activation: str Activation function for a given layer Returns ------- kernel_initializer: tf.keras.initializers Kernel initializer for the given activation function bias_initializer: tf.keras.initializers Bias initializer for the given activation function """ if activation == "relu": kernel_initializer = he_normal() bias_initializer = constant(value=0.01) else: kernel_initializer = glorot_normal() bias_initializer = zeros() return kernel_initializer, bias_initializer
def build(self, input_shapes): """ Builds the layer Args: input_shapes (list of int): shapes of the layer's inputs (node features and adjacency matrix) """ feat_shape = input_shapes[0] input_dim = int(feat_shape[-1]) # Variables to support integrated gradients self.delta = self.add_weight(name="ig_delta", shape=(), trainable=False, initializer=initializers.ones()) self.non_exist_edge = self.add_weight( name="ig_non_exist_edge", shape=(), trainable=False, initializer=initializers.zeros(), ) # Initialize weights for each attention head for head in range(self.attn_heads): # Layer kernel kernel = self.add_weight( shape=(input_dim, self.units), initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint, name="kernel_{}".format(head), ) self.kernels.append(kernel) # # Layer bias if self.use_bias: bias = self.add_weight( shape=(self.units, ), initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint, name="bias_{}".format(head), ) self.biases.append(bias) # Attention kernels attn_kernel_self = self.add_weight( shape=(self.units, 1), initializer=self.attn_kernel_initializer, regularizer=self.attn_kernel_regularizer, constraint=self.attn_kernel_constraint, name="attn_kernel_self_{}".format(head), ) attn_kernel_neighs = self.add_weight( shape=(self.units, 1), initializer=self.attn_kernel_initializer, regularizer=self.attn_kernel_regularizer, constraint=self.attn_kernel_constraint, name="attn_kernel_neigh_{}".format(head), ) self.attn_kernels.append([attn_kernel_self, attn_kernel_neighs]) self.built = True
import tensorflow as tf from tensorflow.keras import layers from tensorflow.keras import initializers import numpy as np import keras.backend as k import random # Define model initializer = initializers.VarianceScaling(scale=1,mode='fan_avg',distribution='uniform',seed=None) bias_init = initializers.zeros() inputs = tf.keras.Input(shape=(260,344,1), dtype='float32', sparse=False) conv1 = layers.Conv2D(filters=16,kernel_size=(3,3),strides=(1,1), padding='same',data_format='channels_last',dilation_rate=(1,1), use_bias=False, kernel_initializer=initializer, bias_initializer=bias_init, kernel_regularizer=None,bias_regularizer=None,activity_regularizer=None, activation = 'linear',kernel_constraint = None, bias_constraint=None)(inputs) activation1 = layers.Activation(trainable=True, dtype='float32', activation='relu')(conv1) pool1 = layers.MaxPooling2D(pool_size=(2,2), padding='valid', strides=(2,2), data_format='channels_last')(activation1) conv2a = layers.Conv2D(filters=32,kernel_size=(3,3),strides=(1,1), padding='same',data_format='channels_last',dilation_rate=(1,1), activation='linear',use_bias=False, kernel_initializer=initializer, bias_initializer=bias_init,kernel_regularizer=None, bias_regularizer=None,activity_regularizer=None,kernel_constraint=None, bias_constraint=None)(pool1)
def build_model(self): input_shape = (self.seq_length,) inputs = Input(shape=input_shape, dtype='int32') embedding = layers.Embedding( len(self.index_to_vector_map), self.embedding_dimensions, input_length=self.seq_length, trainable=self.embedding_training ) embedding.build((None, len(self.index_to_vector_map))) embedding.set_weights([self.index_to_vector_map]) concatenate = layers.Concatenate(axis=-1) flatten = layers.Flatten(data_format='channels_last') dropout = layers.Dropout(self.dropout_rate) dense = layers.Dense(self.num_classes, activation='softmax') pooled_outputs = [] x_embedding = embedding(inputs) x_embedding = tf.expand_dims(x_embedding, -1) for filter_size in self.filter_sizes: filter_shape = (filter_size, self.embedding_dimensions) conv2d = layers.Conv2D( self.num_filters_per_size, filter_shape, 1, padding='VALID', activation=self.activation_fn, kernel_initializer=initializers.RandomUniform(minval=-0.01, maxval=0.01), bias_initializer=initializers.zeros(), use_bias=True, data_format='channels_last', input_shape=(self.seq_length, self.embedding_dimensions, 1) ) x = conv2d(x_embedding) x = tf.nn.max_pool( x, ksize=[1, self.seq_length - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID' ) x = flatten(x) pooled_outputs.append(x) if len(pooled_outputs) == 1: pooled_outputs = tf.convert_to_tensor(pooled_outputs[0]) else: pooled_outputs = concatenate(pooled_outputs) pooled_outputs = dropout(pooled_outputs) pooled_outputs = dense(pooled_outputs) model = Model(inputs=inputs, outputs=pooled_outputs, name='sentence_cnn') model.compile( loss='categorical_crossentropy', optimizer=optimizers.Adadelta( learning_rate=self.learning_rate, rho=self.lr_decay, epsilon=1e-6, ), metrics=['accuracy'] ) return model
plt.show() model = Sequential([ Dense( units=4, input_shape=(4, ), activation=relu, trainable=False, # to freeze the layer kernel_initializer=random_uniform(), bias_initializer=ones()), Dense(units=2, activation=relu, kernel_initializer=lecun_uniform(), bias_initializer=zeros()), Dense(units=4, activation=softmax) ]) model.summary() W0_layers = get_weights(model) b0_layers = get_biases(model) X_train = np.random.random((100, 4)) y_train = X_train X_test = np.random.random((20, 4)) y_test = X_test model.get_layer(
def model_builder(self, filter_size: int = 5, seed_val: int = 123, **kwargs) -> tf.keras.Sequential: """ Build and compile a 1D-CNN depending on the given hyper params (parameters_hyper.yaml). Kwargs require a dict like below. { "conv1_length": int, "conv2_length": int, "extra_conv_layer": bool, "conv3_length": int, "dense1_length": int } """ he_norm = he_normal(seed=seed_val) bias_val = zeros() model = models.Sequential() model.add( layers.Conv1D(filters=kwargs["conv1_length"], kernel_size=filter_size, strides=1, padding="same", use_bias=True, input_shape=self.data.shape[1:], kernel_initializer=he_norm, bias_initializer=bias_val, activation="relu")) model.add(layers.MaxPool1D()) model.add( layers.Conv1D(filters=kwargs["conv2_length"], kernel_size=ceil(filter_size / 2), strides=1, padding="same", use_bias=True, kernel_initializer=he_norm, bias_initializer=bias_val, activation="relu")) model.add(layers.MaxPool1D()) if kwargs["extra_conv_layer"]: model.add( layers.Conv1D(filters=kwargs["conv3_length"], kernel_size=ceil(filter_size / 2), strides=1, padding="same", use_bias=True, kernel_initializer=he_norm, bias_initializer=bias_val, activation="relu")) model.add(layers.MaxPool1D()) model.add(layers.Flatten()) model.add( layers.Dense(units=kwargs["dense1_length"], use_bias=True, kernel_initializer=he_norm, bias_initializer=bias_val, activation="relu")) model.add( layers.Dense(units=1, use_bias=True, kernel_initializer=he_norm, bias_initializer=bias_val, activation="relu")) model.compile(optimizer=Adam( learning_rate=parameters.General_Params().initial_lr), loss=MeanAbsolutePercentageError(name="MAPE"), metrics=[ MeanAbsoluteError(name="MAE"), RootMeanSquaredError(name="RMSE") ]) return model