def __build_cnn2D(self): inputs = tf.keras.Input(shape=(self.image_width, self.image_height, self.history_length)) x = layers.Lambda(lambda layer: layer / 255)(inputs) x = layers.Conv2D( filters=16, kernel_size=(4, 4), strides=(2, 2), activation='relu', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) x = layers.MaxPool2D((2, 2))(x) x = layers.Conv2D( filters=8, kernel_size=(2, 2), strides=(1, 1), activation='relu', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) x = layers.MaxPool2D((2, 2))(x) x = layers.Flatten()(x) x = layers.Dense( 64, activation='relu', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) predictions = layers.Dense( self.num_actions, activation='linear', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) model = tf.keras.Model(inputs=inputs, outputs=predictions) model.compile( optimizer=optimizers.Adam(self.learning_rate), loss=losses.Huber() ) #loss to be removed. It is needed in the bugged version installed on Jetson model.summary() return model
def __build_cnn1D(self): inputs = tf.keras.Input(shape=(self.state_size, self.history_length)) x = layers.Conv1D( filters=16, kernel_size=4, strides=2, activation='relu', kernel_initializer=initializers.VarianceScaling(scale=2.))(inputs) x = layers.Conv1D( filters=32, kernel_size=2, strides=1, activation='relu', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) x = layers.Flatten()(x) x = layers.Dense( 64, activation='relu', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) predictions = layers.Dense( self.num_actions, activation='linear', kernel_initializer=initializers.VarianceScaling(scale=2.))(x) model = tf.keras.Model(inputs=inputs, outputs=predictions) model.compile( optimizer=optimizers.Adam(self.learning_rate), loss=losses.Huber() ) #loss to be removed. It is needed in the bugged version installed on Jetson model.summary() return model
def get_quantized_initializer(w_initializer, w_range): """Gets the initializer and scales it by the range.""" if isinstance(w_initializer, six.string_types): if w_initializer == "he_normal": return initializers.VarianceScaling(scale=2 * w_range, mode="fan_in", distribution="normal", seed=None) if w_initializer == "he_uniform": return initializers.VarianceScaling(scale=2 * w_range, mode="fan_in", distribution="uniform", seed=None) elif w_initializer == "glorot_normal": return initializers.VarianceScaling(scale=w_range, mode="fan_avg", distribution="normal", seed=None) elif w_initializer == "glorot_uniform": return initializers.VarianceScaling(scale=w_range, mode="fan_avg", distribution="uniform", seed=None) elif w_initializer == "random_uniform": return initializers.RandomUniform(-w_range, w_range) return w_initializer
def get_initializer(name, **kwargs): """ Return a kernel initializer by name """ custom_inits = { # initializers for EfficientNet 'en_conv': initializers.VarianceScaling(scale=2., mode='fan_out', distribution='untruncated_normal'), 'en_dense': initializers.VarianceScaling(scale=1. / 2., mode='fan_out', distribution='uniform') } if isinstance(name, str): name = name.lower() if name in custom_inits: return custom_inits[name] elif name.lower() == 'orthogonal': gain = kwargs.get('gain', 1.) return initializers.orthogonal(gain) return initializers.get(name) else: return name
def __init__(self, width, depth, num_anchors=9, separable_conv=True, freeze_bn=False, **kwargs): super(BoxNet, self).__init__(**kwargs) self.width = width self.depth = depth self.num_anchors = num_anchors self.separable_conv = separable_conv options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', 'bias_initializer': 'zeros', } if separable_conv: kernel_initializer = { 'depthwise_initializer': initializers.VarianceScaling(), 'pointwise_initializer': initializers.VarianceScaling(), } options.update(kernel_initializer) self.convs = [ layers.SeparableConv2D(filters=width, name=f'{self.name}/box-{i}', **options) for i in range(depth) ] self.head = layers.SeparableConv2D(filters=num_anchors * 4, name=f'{self.name}/box-predict', **options) else: kernel_initializer = { 'kernel_initializer': initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None) } options.update(kernel_initializer) self.convs = [ layers.Conv2D(filters=width, name=f'{self.name}/box-{i}', **options) for i in range(depth) ] self.head = layers.Conv2D(filters=num_anchors * 4, name=f'{self.name}/box-predict', **options) self.bns = [[ layers.BatchNormalization(momentum=MOMENTUM, epsilon=EPSILON, name=f'{self.name}/box-{i}-bn-{j}') for j in range(3, 8) ] for i in range(depth)] # self.bns = [[BatchNormalization(freeze=freeze_bn, name=f'{self.name}/box-{i}-bn-{j}') for j in range(3, 8)] # for i in range(depth)] self.relu = layers.Lambda(lambda x: tf.nn.swish(x)) self.reshape = layers.Reshape((-1, 4)) self.level = 0
def dense_embedding(n_features=6, n_features_cat=2, activation='relu', number_of_pupcandis=100, embedding_input_dim={0: 13, 1: 3}, emb_out_dim=8, with_bias=True, t_mode=0, units=[64, 32, 16]): n_dense_layers = len(units) inputs_cont = Input(shape=(number_of_pupcandis, n_features-2), name='input') pxpy = Input(shape=(number_of_pupcandis, 2), name='input_pxpy') embeddings = [] inputs = [inputs_cont, pxpy] for i_emb in range(n_features_cat): input_cat = Input(shape=(number_of_pupcandis, 1), name='input_cat{}'.format(i_emb)) inputs.append(input_cat) embedding = Embedding( input_dim=embedding_input_dim[i_emb], output_dim=emb_out_dim, embeddings_initializer=initializers.RandomNormal( mean=0, stddev=0.4/emb_out_dim), name='embedding{}'.format(i_emb))(input_cat) embedding = Reshape((number_of_pupcandis, emb_out_dim))(embedding) embeddings.append(embedding) x = Concatenate()([inputs_cont] + [emb for emb in embeddings]) for i_dense in range(n_dense_layers): x = Dense(units[i_dense], activation='linear', kernel_initializer='lecun_uniform')(x) x = BatchNormalization(momentum=0.95)(x) x = Activation(activation=activation)(x) if t_mode == 0: x = GlobalAveragePooling1D(name='pool')(x) x = Dense(2, name='output', activation='linear')(x) if t_mode == 1: if with_bias: b = Dense(2, name='met_bias', activation='linear', kernel_initializer=initializers.VarianceScaling(scale=0.02))(x) pxpy = Add()([pxpy, b]) w = Dense(1, name='met_weight', activation='linear', kernel_initializer=initializers.VarianceScaling(scale=0.02))(x) w = BatchNormalization(trainable=False, name='met_weight_minus_one', epsilon=False)(w) x = Multiply()([w, pxpy]) x = GlobalAveragePooling1D(name='output')(x) outputs = x keras_model = Model(inputs=inputs, outputs=outputs) keras_model.get_layer('met_weight_minus_one').set_weights([np.array([1.]), np.array([-1.]), np.array([0.]), np.array([1.])]) return keras_model
def __init__(self): super(kerasModel, self).__init__() self.layersList = [] self.layersList.append( kl.Dense(9, activation="relu", input_shape=(4, ), use_bias=False, kernel_initializer=ki.VarianceScaling(), name="dense_1")) self.layersList.append( kl.Dense(1, activation="sigmoid", kernel_initializer=ki.VarianceScaling(), use_bias=False, name="out")) self.loss = discountedLoss() self.optimizer = ko.Adam(lr=1e-2) self.train_loss = kme.Mean(name='train_loss') self.validation_loss = kme.Mean(name='val_loss') self.metric = kme.Accuracy(name="accuracy") @tf.function() def predict(x): """ This is where we run through our whole dataset and return it, when training and testing. """ for l in self.layersList: x = l(x) return x self.predict = predict @tf.function() def train_step(x, labels, adv): """ This is a TensorFlow function, run once for each epoch for the whole input. We move forward first, then calculate gradients with Gradient Tape to move backwards. """ with tf.GradientTape() as tape: predictions = self.predict(x) loss = self.loss.call(y_true=labels, y_pred=predictions, adv=adv) gradients = tape.gradient(loss, self.trainable_variables) self.optimizer.apply_gradients( zip(gradients, self.trainable_variables)) self.train_loss(loss) return loss self.train_step = train_step
def create_keras_two_layer_dense_model(*, input_size, output_size, verbose=False, **kwargs ): # ................................................... # Create model model = Sequential() #.. First hidden layer model.add(Dense( input_dim=input_size, units=kwargs['h1_unit_size'], activation=kwargs["h1_activation"], kernel_initializer=initializers.VarianceScaling(scale=2.0, seed=0) )) model.add(tf.keras.layers.Dropout(kwargs["h1_Dropout"])) #.. Output layer model.add(Dense( units=output_size, activation=kwargs["out_activation"], kernel_regularizer=tf.keras.regularizers.l2(0.001), kernel_initializer=initializers.VarianceScaling(scale=1.0, seed=0) )) # Print network summary if verbose==True: print(model.summary()) else: pass # ................................................... # Define Loss Function and Trianing Operation """ # [option]: Use only default values, model.compile( optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['acc']) """ model.compile( optimizer= kwargs["optimizer"], loss= losses.sparse_categorical_crossentropy, metrics= kwargs["metrics"] # even one arg must be in the list ) return model
def block(self, x, iw, ib, filters, upsample=True): if upsample: y = self.upsample(self.upsample_size)(x) else: y = layers.Activation('linear')(x) # Style vector for use in tRGB w_rgb = layers.Dense(filters, kernel_initializer=initializers.VarianceScaling( 200 / y.shape[2]))(iw) # Reshape style vector w = layers.Dense(x.shape[-1], kernel_initializer='he_uniform')(iw) # Crop noise to fit image b = layers.Lambda(self.crop)([ib, y]) # Pass noise through a dense layer to fit filter size d = layers.Dense(filters, kernel_initializer='zeros')(b) y = self.mod_conv2d(filters)([y, w]) y = layers.add([y, d]) y = layers.LeakyReLU(0.2)(y) w = layers.Dense(filters, kernel_initializer='he_uniform')(iw) d = layers.Dense(filters, kernel_initializer='zeros')(b) y = self.mod_conv2d(filters)([y, w]) y = layers.add([y, d]) y = layers.LeakyReLU(0.2)(y) return y, self.tRGB(y, w_rgb)
def conv2d_bn(self, x, nb_filter, num_row, num_col, padding='same', strides=(1, 1), use_bias=False): if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = -1 x = Conv2D(nb_filter, (num_row, num_col), strides=strides, padding=padding, use_bias=use_bias, kernel_regularizer=regularizers.l2(0.00004), kernel_initializer=initializers.VarianceScaling( scale=2.0, mode='fan_in', distribution='normal', seed=None))(x) x = BatchNormalization(axis=channel_axis, momentum=0.9997, scale=False)(x) x = Activation('relu')(x) return x
def build(self, input_shape): self.denses = [ KL.Dense(1024, kernel_initializer=initializers.VarianceScaling(), kernel_regularizer=self._kernel_regularizer, activation='relu') for _ in range(2) ] super().build(input_shape)
def __init__(self, width, depth, num_classes=8, num_anchors=9, freeze_bn=False, **kwargs): super(ClassNet, self).__init__(**kwargs) self.width = width self.depth = depth self.num_classes = num_classes self.num_anchors = num_anchors options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', } kernel_initializer = { 'depthwise_initializer': initializers.VarianceScaling(), 'pointwise_initializer': initializers.VarianceScaling(), } options.update(kernel_initializer) self.convs = [ layers.SeparableConv2D(filters=self.width, bias_initializer='zeros', name=f'{self.name}/class-{i}', **options) for i in range(self.depth) ] self.head = layers.SeparableConv2D( filters=self.num_classes * self.num_anchors, bias_initializer=PriorProbability(probability=0.01), name=f'{self.name}/class-predict', **options) self.bns = [[ BatchNormalization(freeze=freeze_bn, momentum=MOMENTUM, epsilon=EPSILON, name=f'{self.name}/class-{i}-bn-{j}') for j in range(3, 8) ] for i in range(self.depth)] self.activation = layers.Lambda(lambda x: tf.nn.swish(x)) self.reshape = layers.Reshape((-1, self.num_classes)) self.activation_sigmoid = layers.Activation('sigmoid') self.level = 0
def __init__(self, width, depth, num_classes=20, num_anchors=9, separable_conv=True, freeze_bn=False, **kwargs): super(ClassNet, self).__init__(**kwargs) self.width = width self.depth = depth self.num_classes = num_classes self.num_anchors = num_anchors self.separable_conv = separable_conv options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', } if self.separable_conv: kernel_initializer = { 'depthwise_initializer': initializers.VarianceScaling(), 'pointwise_initializer': initializers.VarianceScaling(), } options.update(kernel_initializer) self.convs = [layers.SeparableConv2D(filters=width, bias_initializer='zeros', name=f'{self.name}/class-{i}', **options) for i in range(depth)] self.head = layers.SeparableConv2D(filters=num_classes * num_anchors, bias_initializer=PriorProbability(probability=0.01), name=f'{self.name}/class-predict', **options) else: kernel_initializer = { 'kernel_initializer': initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None) } options.update(kernel_initializer) self.convs = [layers.Conv2D(filters=width, bias_initializer='zeros', name=f'{self.name}/class-{i}', **options) for i in range(depth)] self.head = layers.Conv2D(filters=num_classes * num_anchors, bias_initializer=PriorProbability(probability=0.01), name='class-predict', **options) self.bns = [ [layers.BatchNormalization(momentum=MOMENTUM, epsilon=EPSILON, name=f'{self.name}/class-{i}-bn-{j}') for j in range(3, 8)] for i in range(depth)] # self.bns = [[BatchNormalization(freeze=freeze_bn, name=f'{self.name}/class-{i}-bn-{j}') for j in range(3, 8)] # for i in range(depth)] self.relu = layers.Lambda(swish) self.reshape = layers.Reshape((-1, num_classes)) self.activation = layers.Activation('sigmoid') self.level = 0
def tRGB(self, x, w): size = x.shape[2] scale = self.max_size // size vs = initializers.VarianceScaling(200 / size) x = self.mod_conv2d(3, (1, 1), kernel_initializer=vs, demod=False)([x, w]) x = self.upsample((scale, scale))(x) return x
def build(self, input_shape): self._conv_classification_head = KL.Conv2D( self._multiples * self._num_classes, (1, 1), padding='valid', activation=None, kernel_initializer=self._kernel_initializer_classification_head, kernel_regularizer=self._kernel_regularizer, name=f'{self.name}classification_head') self._conv_box_prediction_head = KL.Conv2D( (self._num_classes - 1) * self._multiples * 4, (1, 1), padding='valid', activation=None, kernel_initializer=self._kernel_initializer_box_prediction_head, kernel_regularizer=self._kernel_regularizer, name=f'{self.name}box_prediction_head') if self._use_mask: self._segmentation_layers = [ KL.Conv2D(256, (3, 3), padding='valid', activation='relu', kernel_initializer=initializers.VarianceScaling( scale=2., mode='fan_out'), kernel_regularizer=self._kernel_regularizer), KL.Conv2DTranspose( 256, (2, 2), strides=(2, 2), padding='valid', activation='relu', kernel_initializer=initializers.VarianceScaling( scale=2., mode='fan_out'), kernel_regularizer=self._kernel_regularizer), KL.Conv2D(self._num_classes, (3, 3), padding='valid', activation='relu', kernel_initializer=initializers.VarianceScaling( scale=2., mode='fan_out'), kernel_regularizer=self._kernel_regularizer) ] super().build(input_shape)
def __init__(self, width, depth, num_anchors=9, name='box_net', **kwargs): self.name = name self.width = width self.depth = depth self.num_anchors = num_anchors options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', 'bias_initializer': 'zeros', 'depthwise_initializer': initializers.VarianceScaling(), 'pointwise_initializer': initializers.VarianceScaling(), } self.convs = [layers.SeparableConv2D(filters=width, name=f'{self.name}/box-{i}', **options) for i in range(depth)] self.head = layers.SeparableConv2D(filters=num_anchors * 4, name=f'{self.name}/box-predict', **options) self.bns = [[layers.BatchNormalization(momentum=MOMENTUM, epsilon=EPSILON, name=f'{self.name}/box-{i}-bn-{j}') for j in range(3, 8)] for i in range(depth)] self.relu = layers.Lambda(lambda x: tf.nn.swish(x)) self.reshape = layers.Reshape((-1, 4))
def tiny_classifier(source_layers, num_priors, num_classes=21): mbox_conf = [] mbox_loc = [] for i, x in enumerate(source_layers): # source_layers # name = x.name.split('/')[0] # name만 추출 (ex: block3b_add) name = x.name.split(':')[0] # name만 추출 (ex: block3b_add) x1 = SeparableConv2D(num_priors[i] * num_classes, 3, padding='same', depthwise_initializer=initializers.VarianceScaling(), pointwise_initializer=initializers.VarianceScaling(), name= name + '_mbox_conf_1')(x) x1 = Flatten(name=name + '_mbox_conf_flat')(x1) mbox_conf.append(x1) x2 = SeparableConv2D(num_priors[i] * 4, 3, padding='same', depthwise_initializer=initializers.VarianceScaling(), pointwise_initializer=initializers.VarianceScaling(), name= name + '_mbox_loc_1')(x) x2 = Flatten(name=name + '_mbox_loc_flat')(x2) mbox_loc.append(x2) mbox_loc = Concatenate(axis=1, name='mbox_loc')(mbox_loc) mbox_loc = Reshape((-1, 4), name='mbox_loc_final')(mbox_loc) mbox_conf = Concatenate(axis=1, name='mbox_conf')(mbox_conf) mbox_conf = Reshape((-1, num_classes), name='mbox_conf_logits')(mbox_conf) predictions = Concatenate(axis=2, name='predictions', dtype=tf.float32)([mbox_loc, mbox_conf]) return predictions
def _get_qfunc(self): inputs = Input(shape=self.obs_shape, dtype='uint8') cast_input = tf.dtypes.cast(inputs, tf.float32) input_scaled = cast_input/255.0 conv1 = layers.Conv2D(filters=32,kernel_size=(8,8),strides=4,kernel_initializer=initializers.VarianceScaling(scale=2,distribution='untruncated_normal'), activation=tf.nn.relu)(input_scaled) conv2 = layers.Conv2D(filters=64,kernel_size=(4,4),strides=2,kernel_initializer=initializers.VarianceScaling(scale=2,distribution='untruncated_normal'), activation=tf.nn.relu)(conv1) conv3 = layers.Conv2D(filters=64,kernel_size=(3,3),strides=1,kernel_initializer=initializers.VarianceScaling(scale=2,distribution='untruncated_normal'), activation=tf.nn.relu)(conv2) flatten = layers.Flatten()(conv3) dense = layers.Dense(units=512,kernel_initializer=initializers.VarianceScaling(scale=2,distribution='untruncated_normal'), activation=tf.nn.relu)(flatten) outputs = layers.Dense(units=self.n_actions)(dense) model = tf.keras.Model(inputs, outputs) return model
def f(x): initializer = initializers.VarianceScaling(2, mode='fan_avg', distribution='normal') d = layers.Dense(size, kernel_initializer=initializer, dtype=config.policy)(x) if dropout_rate: d = layers.Dropout(dropout_rate, dtype=config.policy)(d) d = layers.BatchNormalization(dtype=tf.float32)(d) if activation is not None: d = activation(d) return d
def __init__(self, width, depth, num_classes=20, num_anchors=9, name='class_net', **kwargs): self.name = name self.width = width self.depth = depth self.num_classes = num_classes self.num_anchors = num_anchors options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', 'depthwise_initializer': initializers.VarianceScaling(), 'pointwise_initializer': initializers.VarianceScaling(), } self.convs = [layers.SeparableConv2D(filters=width, bias_initializer='zeros', name=f'{self.name}/class-{i}', **options) for i in range(depth)] self.head = layers.SeparableConv2D(filters=num_classes * num_anchors, bias_initializer=PriorProbability(probability=0.01), name=f'{self.name}/class-predict', **options) self.bns = [[layers.BatchNormalization(momentum=MOMENTUM, epsilon=EPSILON, name=f'{self.name}/class-{i}-bn-{j}') for j in range(3, 8)] for i in range(depth)] self.relu = layers.Lambda(lambda x: tf.nn.swish(x)) self.reshape = layers.Reshape((-1, num_classes)) self.activation = layers.Activation('sigmoid')
def conv2d_bn(self, x, nb_filter, num_row, num_col, padding='same', strides=(1, 1), use_bias=False, activation_use = 'relu', use_bn = True, use_activation = True): x = Conv2D(nb_filter, (num_row, num_col), strides=strides, padding=padding, use_bias=use_bias, kernel_regularizer=regularizers.l2(0.00004), kernel_initializer=initializers.VarianceScaling(scale=2.0, mode='fan_in', distribution='normal', seed=None))(x) if use_bn: x = BatchNormalization(axis=self.channel_axis, momentum=0.9997, scale=False)(x) if use_activation: x = Activation(activation_use)(x) return x
def create_keras_one_layer_dense_model(*, input_size, output_size, verbose=False, **kwargs ): """ Notes: https://www.tensorflow.org/tutorials/keras/save_and_load """ # ................................................... # Create model model = Sequential() #.. add fully connected layer model.add(Dense( input_dim=input_size, # IE 784 PIXELS, ! units=output_size, activation=kwargs["out_activation"], kernel_regularizer=tf.keras.regularizers.l2(0.001), kernel_initializer=initializers.VarianceScaling(scale=1.0, seed=0) )) # Print network summary if verbose==True: print(model.summary()) else: pass # ................................................... # Define Loss Function and Trianing Operation """ # [option]: Use only default values, model.compile( optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['acc']) """ model.compile( optimizer= kwargs["optimizer"], loss= losses.sparse_categorical_crossentropy, metrics= kwargs["metrics"] # even one arg must be in the list ) return model
def run_deep_WaveNet(X_train, y_train, X_val, y_val, X_test, y_test): X_train = np.expand_dims(X_train, axis=2) X_val = np.expand_dims(X_val, axis=2) X_test = np.expand_dims(X_test, axis=2) model = wrappers.scikit_learn.KerasClassifier(build_fn=build_deep_WaveNet) he_avg_init = initializers.VarianceScaling(scale=2., mode='fan_avg', distribution="uniform") opti = [ optimizers.Adam(learning_rate=10), optimizers.Adam(learning_rate=1), optimizers.Adam(learning_rate=0.1), optimizers.Adam(learning_rate=0.01), optimizers.Adam(learning_rate=0.0001), optimizers.Adam(learning_rate=0.00001), ] init = ['he_normal', he_avg_init] epochs = [10000] batches = [25, 50, 150] param_grid = dict(optimizer=opti, epochs=epochs, batch_size=batches, init=init) callback = callbacks.EarlyStopping(monitor='loss', patience=50) grid = GridSearchCV(estimator=model, param_grid=param_grid) grid_result = grid.fit(X_train, y_train, callbacks=[callback]) print("Best: {} using {}".format(grid_result.best_score_, grid_result.best_params_)) model = grid.best_estimator_.model test_loss, test_acc = model.evaluate(X_test, y_test, verbose=2) print("Model Config : {}".format(model.optimizer.get_config())) print("Test Loss : {} Test Acc : {}".format(test_loss, test_acc))
def f(x): padding = 'same' if strides == 1 or config.pad else 'valid' initializer = initializers.VarianceScaling(2, mode='fan_avg', distribution='normal') c = layers.Conv2D(filters, kernel, padding=padding, strides=strides, kernel_initializer=initializer, dtype=config.policy)(x) if dropout_rate: c = layers.Dropout(dropout_rate, noise_shape=[None, 1, 1, None], dtype=config.policy)(c) c = layers.BatchNormalization(dtype=tf.float32)(c) if not norelu: c = layers.ReLU(dtype=config.policy)(c) if pool: c = layers.MaxPooling2D(dtype=config.policy)(c) return c
def build_best_WaveNet(): he_avg_init = initializers.VarianceScaling(scale=2., mode='fan_avg', distribution="uniform") model = models.Sequential() model.add(layers.InputLayer(input_shape=(265, 1))) for rate in (1, 2, 4, 8) * 2: model.add( layers.Conv1D(filters=20, kernel_size=2, activation="relu", dilation_rate=rate)) model.add(layers.Conv1D(filters=10, kernel_size=1)) model.add(layers.Flatten()) model.add( layers.Dense(64, kernel_initializer=he_avg_init, activation='sigmoid')) model.add(layers.Dense(1)) return model
import tensorflow as tf from tensorflow.keras import layers from tensorflow.keras import models from tensorflow.keras import regularizers from tensorflow.keras import initializers import tensorflow.keras.backend as K init = initializers.VarianceScaling(scale=0.5, mode='fan_in', distribution='normal') def dense_bn(x, channels, name=None, l2_reg=1e-4): x = layers.Dense(channels, use_bias=False, name='{}_fc'.format(name), kernel_regularizer=regularizers.l2(l2_reg))(x) x = layers.BatchNormalization(scale=True, name='{}_bn'.format(name))(x) return layers.Activation('relu', name=name)(x) def dense_selu(x, channels, name=None, l2_reg=1e-4): x = layers.Dense(channels, name='{}/fc'.format(name), kernel_initializer=init, kernel_regularizer=regularizers.l2(l2_reg))(x) return layers.Activation('selu', name=name + '/selu')(x) def flatten_pixels(nchannels):
discount_factor = 0.99 n_step = 10 one_step_weight = 1.0 / 2.0 n_step_weight = 1.0 / 2.0 expert_weight = 0.0 model_restore_path = None # network architecture conv_layers = { 'filters': [32, 64, 64, 1024], 'kernel_sizes': [8, 4, 3, 7], 'strides': [4, 2, 1, 1], 'paddings': ['valid' for _ in range(4)], 'activations': ['relu' for _ in range(4)], 'initializers': [initializers.VarianceScaling(scale=2.0) for _ in range(4)], 'names': ['conv_%i' % (i) for i in range(1, 5)] } dense_layers = None # exploration parameters eps_schedule = [[1, 0.1, 1000000], [0.1, 0.01, 5000000], [0.01, 0.001, 5000000]] # training session parameters target_interval = 10000 warmup_steps = 50000 pretrain_steps = None learning_interval = 4 num_steps = 12000000 num_episodes = 1500
def get_model(self): # Input Layer user_input = Input(shape=(1, ), dtype='int32', name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') text_input = Input(shape=(self.args.sim_feature_size, ), dtype='float32', name='text_input') # Embedding layer MF_Embedding_User = Embedding( input_dim=self.num_users, output_dim=self.args.mf_embedding_dim, name='mf_embedding_user', embeddings_initializer=initializers.VarianceScaling( scale=0.01, distribution='normal'), embeddings_regularizer=l2(0.01), input_length=1) MF_Embedding_Item = Embedding( input_dim=self.num_items, output_dim=self.args.mf_embedding_dim, name='mf_embedding_item', embeddings_initializer=initializers.VarianceScaling( scale=0.01, distribution='normal'), embeddings_regularizer=l2(0.01), input_length=1) # MF part mf_user_latent = tf.keras.layers.Flatten()( MF_Embedding_User(user_input)) mf_item_latent = tf.keras.layers.Flatten()( MF_Embedding_Item(item_input)) # why Flatten? mf_vector = concatenate([mf_user_latent, mf_item_latent ]) # element-wise multiply ??? for idx in range(len(self.args.mf_fc_unit_nums)): # 学习非线性关系 layer = Dense(self.args.mf_fc_unit_nums[idx], activation='relu', name="layer%d" % idx) mf_vector = layer(mf_vector) # Text part # text_input = Dense(10, activation='relu', kernel_regularizer=l2(0.01))(text_input) # sim? 需要再使用MLP处理下? # Concatenate MF and TEXT parts predict_vector = concatenate([mf_vector, text_input]) for idx in range(len(self.args.final_MLP_layers)): # 整合后再加上MLP? layer = Dense(self.args.final_MLP_layers[idx], activation='relu') # name="layer%d" % idx predict_vector = layer(predict_vector) predict_vector = tf.keras.layers.Dropout(0.5)( predict_vector) # 使用dropout? if self.args.final_activation == 'softmax': predict_vector = Dense(2, activation='softmax', name="prediction")(predict_vector) elif self.args.final_activation == 'sigmoid': predict_vector = Dense(1, activation='sigmoid', kernel_initializer='lecun_uniform', name="prediction")(predict_vector) # # Final prediction layer # predict_vector = Dense(1, activation='sigmoid', kernel_initializer='lecun_uniform', name="prediction")(predict_vector) model = Model(inputs=[user_input, item_input, text_input], outputs=predict_vector) return model
def get_model(self): num_layer = len(self.layers) # Number of layers in the MLP # Input variables user_input = Input(shape=(1, ), dtype='int32', name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') # Embedding layer MF_Embedding_User = Embedding( input_dim=self.num_users, output_dim=self.args.mf_embedding_dim, name='mf_embedding_user', embeddings_initializer=initializers.VarianceScaling( scale=0.01, distribution='normal'), embeddings_regularizer=l2(self.args.NCF_reg_mf), input_length=1) # MF_Embedding_Item = Embedding( input_dim=self.num_items, output_dim=self.args.mf_embedding_dim, name='mf_embedding_item', embeddings_initializer=initializers.VarianceScaling( scale=0.01, distribution='normal'), embeddings_regularizer=l2(self.args.NCF_reg_mf), input_length=1) # MLP_Embedding_User = Embedding( input_dim=self.num_users, output_dim=int(self.args.mf_fc_unit_nums[0] / 2), name="mlp_embedding_user", embeddings_initializer=initializers.VarianceScaling( scale=0.01, distribution='normal'), embeddings_regularizer=l2(self.args.NCF_reg_layers[0]), input_length=1) # MLP_Embedding_Item = Embedding( input_dim=self.num_items, output_dim=int(self.args.mf_fc_unit_nums[0] / 2), name='mlp_embedding_item', embeddings_initializer=initializers.VarianceScaling( scale=0.01, distribution='normal'), embeddings_regularizer=l2(self.args.NCF_reg_layers[0]), input_length=1) # # MF part mf_user_latent = tf.keras.layers.Flatten()( MF_Embedding_User(user_input)) mf_item_latent = tf.keras.layers.Flatten()( MF_Embedding_Item(item_input)) # mf_vector = merge([mf_user_latent, mf_item_latent], mode='mul') # element-wise multiply mf_vector = Multiply()([mf_user_latent, mf_item_latent]) # MLP part mlp_user_latent = tf.keras.layers.Flatten()( MLP_Embedding_User(user_input)) mlp_item_latent = tf.keras.layers.Flatten()( MLP_Embedding_Item(item_input)) # mlp_vector = merge([mlp_user_latent, mlp_item_latent], mode='concat') mlp_vector = Concatenate()([mlp_user_latent, mlp_item_latent]) for idx in range(1, num_layer): layer = Dense(self.args.mf_fc_unit_nums[idx], activation='relu', name="layer%d" % idx) # kernel_regularizer=l2(reg_layers[idx]), mlp_vector = layer(mlp_vector) # Concatenate MF and MLP parts # mf_vector = Lambda(lambda x: x * alpha)(mf_vector) # mlp_vector = Lambda(lambda x : x * (1-alpha))(mlp_vector) # predict_vector = merge([mf_vector, mlp_vector], mode='concat') predict_vector = Concatenate()([mf_vector, mlp_vector]) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer='lecun_uniform', name="prediction")(predict_vector) model = Model(input=[user_input, item_input], output=prediction) return model
def create_convNN(*, input_size, output_size, kwargs, verbose=False): ''' function to build cnn, with 2 convolutions, one hidden layer and one note: its not mistake with kwags, - i kept it intentionally as packed distionary, to allow to read parameter sourse in the code' ''' run = True K.clear_session() if run == True: # Convolutional Network, ........................ model = keras.Sequential() #.. 1st cnn, layer model.add( keras.layers.Conv2D(filters=kwargs['Conv2D_1__filters'], kernel_size=kwargs['Conv2D_1__kernel_size'], strides=kwargs['Conv2D_1__stride'], activation=kwargs['Conv2D_1__activation'], input_shape=input_size)) #.. maxpool 1. model.add( keras.layers.MaxPool2D(pool_size=kwargs['MaxPool2D_1__pool_size'])) #.. 2nd cnn layer, model.add( keras.layers.Conv2D( filters=kwargs['Conv2D_2__filters'], kernel_size=kwargs['Conv2D_2__kernel_size'], strides=kwargs['Conv2D_2__stride'], activation=kwargs['Conv2D_2__activation'], )) #.. maxpool 2, model.add( keras.layers.MaxPool2D(pool_size=kwargs['MaxPool2D_2__pool_size'])) # flatten the results, ......................... model.add(keras.layers.Flatten()) # dense nn, .................................... if kwargs["model"] == "two_dense_layers": #.. First hidden layer model.add( Dense(units=kwargs['h1_unit_size'], activation=kwargs["h1_activation"], kernel_regularizer=tf.keras.regularizers.l2( kwargs['h1_l2']), kernel_initializer=initializers.VarianceScaling( scale=2.0, seed=0))) model.add(tf.keras.layers.Dropout(kwargs["h1_Dropout"])) else: pass #.. Output layer model.add( Dense(units=output_size, activation=kwargs["out_activation"], kernel_regularizer=tf.keras.regularizers.l2( kwargs['out_l2']), kernel_initializer=initializers.VarianceScaling(scale=1.0, seed=0))) # Define Loss Function and Trianing Operation model.compile( optimizer=kwargs["optimizer"], loss=losses.categorical_crossentropy, metrics=kwargs["metrics"] # even one arg must be in the list ) if verbose == True: model.summary() else: pass return model