def generate( self, V: Variat, input_features_layer: DenseFeatures ) -> Generator[Model, None, None]: model = tf.keras.Sequential([ input_features_layer, layers.Dense(158, activation='selu', kernel_initializer=initializers.lecun_normal()), layers.Dropout(0.2), layers.Dense(168, activation='swish', kernel_initializer=initializers.GlorotNormal()), layers.Dropout(0.2), layers.Dense(178, activation='swish', kernel_initializer=initializers.GlorotNormal()), layers.Dropout(0.2), layers.Dense(188, activation='selu', kernel_initializer=initializers.lecun_normal()), layers.Dropout(0.2), layers.Dense(1, activation="sigmoid", kernel_initializer=initializers.lecun_normal()) ]) yield model
def resnet_blocks(input_res, kernel, name): gen_initializer = lecun_normal() in_res_1 = ReflectPadding3D(padding=1)(input_res) out_res_1 = Conv3D(kernel, 3, strides=1, kernel_initializer=gen_initializer, use_bias=False, name=name + '_conv_a', data_format='channels_first')(in_res_1) out_res_1 = InstanceNormalization3D(name=name + '_isnorm_a')(out_res_1) out_res_1 = Activation('relu')(out_res_1) in_res_2 = ReflectPadding3D(padding=1)(out_res_1) out_res_2 = Conv3D(kernel, 3, strides=1, kernel_initializer=gen_initializer, use_bias=False, name=name + '_conv_b', data_format='channels_first')(in_res_2) out_res_2 = InstanceNormalization3D(name=name + '_isnorm_b')(out_res_2) out_res = Add()([out_res_2, input_res]) return out_res
def __init__(self, model_features): super(cnn, self).__init__() """ Define here the layers used during the forward-pass of the neural network. """ l2_regularization_scale = model_features.l2_regularization_scale dropout_probability = model_features.dropout_probability nodes_layer_1 = model_features.nodes_layer_1 nodes_layer_2 = model_features.nodes_layer_2 input_shape = (-1, 1024, 1) number_filters = model_features.number_filters kernel_size = model_features.kernel_size self.batch_size = model_features.batch_size self.scaler = model_features.scaler # Convolution layers. self.cnn_layer1 = tf.layers.Conv1D(filters=number_filters[0], kernel_size=kernel_size[0], strides=1, padding='valid', activation='relu') self.max_pool1 = tf.layers.MaxPooling1D(pool_size=2, strides=2, padding='valid') self.cnn_layer2 = tf.layers.Conv1D(filters=number_filters[1], kernel_size=kernel_size[1], strides=1, padding='valid', activation='relu') self.max_pool2 = tf.layers.MaxPooling1D(pool_size=2, strides=2, padding='valid') # Fully connected layers. self.flatten1 = tf.layers.Flatten() self.dropout1 = tf.layers.Dropout(dropout_probability) self.dense_layer1 = tf.layers.Dense(nodes_layer_1, kernel_initializer=lecun_normal(), activation=tf.nn.relu) self.dropout2 = tf.layers.Dropout(dropout_probability) self.output_layer = tf.layers.Dense(57, kernel_initializer=lecun_normal(), activation=None)
def get_compiled_mlp_model(num_users, num_items, learning_rate=0.001, layers_num=[20, 10], reg_layers=[0, 0]): assert len(layers_num) == len(reg_layers) num_layer = len(layers_num) # Number of layers in the MLP # Input variables user_input = layers.Input(shape=(1, ), dtype='int32', name='user_input') item_input = layers.Input(shape=(1, ), dtype='int32', name='item_input') user_embedding = layers.Embedding(input_dim=num_users, output_dim=int(layers_num[0] / 2), name='user_embedding', embeddings_regularizer=regularizers.l2( reg_layers[0]), input_length=1) item_embedding = layers.Embedding(input_dim=num_items, output_dim=int(layers_num[0] / 2), name='item_embedding', embeddings_regularizer=regularizers.l2( reg_layers[0]), input_length=1) # Crucial to flatten an embedding vector! user_latent = layers.Flatten()(user_embedding(user_input)) item_latent = layers.Flatten()(item_embedding(item_input)) # The 0-th layer is the concatenation of embedding layers vector = layers.concatenate([user_latent, item_latent]) # MLP layers for idx in range(1, num_layer): layer = layers.Dense(layers_num[idx], kernel_regularizer=regularizers.l2( reg_layers[idx]), activation='relu', name='layer%d' % idx) vector = layer(vector) # Final prediction layer prediction = layers.Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(vector) model_mlp = models.Model(inputs=[user_input, item_input], outputs=prediction) model_mlp.compile(optimizer=optimizers.Adam(lr=learning_rate, clipnorm=0.5), loss='binary_crossentropy') return model_mlp
def __init__(self, repr_length, none_initializer=initializers.zeros, kernel_initializer=initializers.lecun_normal(), bias_initializer=initializers.zeros): super(OptionCase, self).__init__() self.repr_length = repr_length self.none_initializer = none_initializer self.kernel_initializer = kernel_initializer self.bias_initializer = bias_initializer self.supports_masking = False self.input_spec = InputSpec(min_ndim=2)
def Classifier_Net(x1, x2): dense_1 = Dense(128, kernel_initializer=initializers.he_normal()) dense_2 = Dense(128, kernel_initializer=initializers.he_normal()) dense_final = Dense(1, kernel_initializer=initializers.lecun_normal(), activation='sigmoid') x = Add()([dense_1(x1), dense_2(x2)]) x = Activation('relu')(BatchNormalization()(x)) x = dense_final(Dropout(0.4)(x)) # x = dense_final(x) return x
def get_model(): dmf_num_layer = len(userlayers) #Number of layers in the DMF mlp_num_layer = len(layers) #Number of layers in the MLP # Input variables user_rating = Input(shape=(num_items, ), dtype='int32', name='user_input') item_rating = Input(shape=(num_users, ), dtype='int32', name='item_input') # DMF part userlayer = Dense(userlayers[0], activation="linear", name='user_layer0') itemlayer = Dense(itemlayers[0], activation="linear", name='item_layer0') dmf_user_latent = userlayer(user_rating) dmf_item_latent = itemlayer(item_rating) for idx in range(1, dmf_num_layer): userlayer = Dense(userlayers[idx], activation='relu', name='user_layer%d' % idx) itemlayer = Dense(itemlayers[idx], activation='relu', name='item_layer%d' % idx) dmf_user_latent = userlayer(dmf_user_latent) dmf_item_latent = itemlayer(dmf_item_latent) dmf_vector = multiply([dmf_user_latent, dmf_item_latent]) # MLP part MLP_Embedding_User = Dense(layers[0] // 2, activation="linear", name='user_embedding') MLP_Embedding_Item = Dense(layers[0] // 2, activation="linear", name='item_embedding') mlp_user_latent = MLP_Embedding_User(user_rating) mlp_item_latent = MLP_Embedding_Item(item_rating) mlp_vector = concatenate([mlp_user_latent, mlp_item_latent]) for idx in range(1, mlp_num_layer): layer = Dense(layers[idx], activation='relu', name="layer%d" % idx) mlp_vector = layer(mlp_vector) # Concatenate DMF and MLP parts predict_vector = concatenate([dmf_vector, mlp_vector]) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name="prediction")(predict_vector) model_ = Model(inputs=[user_rating, item_rating], outputs=prediction) return model_
def create_prm_initializer(prm): if prm['initializer'] is None: prm['initializer_func'] = None if prm['initializer'] == 'glorot_normal': prm['initializer_func'] = glorot_normal() if prm['initializer'] == 'lecun_uniform': prm['initializer_func'] = lecun_uniform() if prm['initializer'] == 'lecun_normal': prm['initializer_func'] = lecun_normal() return (prm)
def gated_classifier_model(Xv, Xp): mlp_phr = Dense(300, kernel_initializer=initializers.he_normal()) mlp_vis = Dense(300, kernel_initializer=initializers.he_normal()) final_mlp = Dense(1, kernel_initializer=initializers.lecun_normal(), activation='sigmoid', name='final') g_phr, g_vis = multimodal_gate_model(Xv, Xp) h_phr = Activation('tanh')(BatchNormalization()(mlp_phr(Xp))) h_vis = Activation('tanh')(BatchNormalization()(mlp_vis(Xv))) h = Add()([Multiply()([g_phr, h_phr]), Multiply()([g_vis, h_vis])]) h = final_mlp(Dropout(0.4)(h)) h = Flatten()(h) return h
def create_str_to_initialiser_converter(self): """Creates a dictionary which converts strings to initialiser""" str_to_initialiser_converter = { "glorot_normal": initializers.glorot_normal, "glorot_uniform": initializers.glorot_uniform, "xavier_normal": initializers.glorot_normal, "xavier_uniform": initializers.glorot_uniform, "xavier": initializers.glorot_uniform, "he_normal": initializers.he_normal(), "he_uniform": initializers.he_uniform(), "lecun_normal": initializers.lecun_normal(), "lecun_uniform": initializers.lecun_uniform(), "truncated_normal": initializers.TruncatedNormal, "variance_scaling": initializers.VarianceScaling, "default": initializers.glorot_uniform } return str_to_initialiser_converter
def classifier_model(f1, f2): mlp_1 = Dense(128, kernel_initializer=initializers.he_normal(), name='mlp_1') mlp_2 = Dense(128, kernel_initializer=initializers.he_normal(), name='mlp_2') final_mlp = Dense(1, kernel_initializer=initializers.lecun_normal(), activation='sigmoid', name='final') final_fuse = Add()([mlp_1(f1), mlp_2(f2)]) h = Activation('relu')(BatchNormalization()(final_fuse)) h = final_mlp(Dropout(0.4)(h)) h = Flatten()(h) return h
def get_compiled_gmf_model(num_users, num_items, latent_dim=8, learning_rate=0.001, reg=[0, 0]): # Input variables user_input = layers.Input(shape=(1, ), dtype='int32', name='user_input') item_input = layers.Input(shape=(1, ), dtype='int32', name='item_input') user_embedding = layers.Embedding( input_dim=num_users, output_dim=latent_dim, embeddings_initializer=initializers.RandomNormal(), embeddings_regularizer=regularizers.l2(reg[0]), input_length=1, name='user_embedding') item_embedding = layers.Embedding( input_dim=num_items, output_dim=latent_dim, embeddings_initializer=initializers.RandomNormal(), embeddings_regularizer=regularizers.l2(reg[1]), input_length=1, name='item_embedding') # Crucial to flatten an embedding vector! user_latent = layers.Flatten()(user_embedding(user_input)) item_latent = layers.Flatten()(item_embedding(item_input)) # Element-wise product of user and item embeddings predict_vector = layers.multiply([user_latent, item_latent]) prediction = layers.Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(predict_vector) model_gmf = models.Model(inputs=[user_input, item_input], outputs=prediction) model_gmf.compile(optimizer=optimizers.Adam(lr=learning_rate, clipnorm=0.5), loss='binary_crossentropy') #model_gmf.compile(optimizer=Accoptimizers.Adam(lr=learning_rate, clipnorm=0.5), loss='binary_crossentropy') return model_gmf
def __init__(self, ncomp=10, nfeat=154, nunits=50, kernel="RBF", sigma=1., quasiRandom=True, cosOnly=False, inputd=None, outputd=None, nsteps=200, weights=None): self.ncomp = ncomp # number of mixture components self.nunits = nunits # number of output units for LSTM self.inputd = inputd # dimensionality of the input self.nsteps = nsteps # number of time steps in the self.outputd = outputd # dimensionality of the output self.weights = weights # sample weights self.nfeat = nfeat # number of RFFs self.quasiRandom = quasiRandom self.cosOnly = cosOnly self.sigma = sigma * np.ones(self.nunits) self.kernel = kernel self.rff_tf = RFF_TF(self.nfeat, self.nunits, self.sigma, self.cosOnly, self.quasiRandom, self.kernel) # Set for GPU use config = tf.ConfigProto(device_count={'CPU': 2, 'GPU': 2}) config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) def elu_modif(x, a=1.): e = 1e-15 return ELU(alpha=a)(x) + 1. + e # Note: The output size will be (outputd + 2) * ncomp # For reference, not used at the moment def log_sum_exp(x, axis=None): """Log-sum-exp trick implementation""" x_max = K.max(x, axis=axis, keepdims=True) return K.log(K.sum(K.exp(x - x_max), axis=axis, keepdims=True)) + x_max def tril_matrix(elements): tfd = tfp.distributions tril_m = tfd.fill_triangular(elements) tf.matrix_set_diag(tril_m, tf.exp(tf.matrix_diag_part(tril_m))) return tril_m def mean_log_Gaussian_like(y_true, parameters): # This version uses tensorflow_probability components = K.reshape(parameters, [ -1, self.outputd + int(0.5 * (self.outputd + 1) * self.outputd) + 1, self.ncomp ]) mu = components[:, :self.outputd, :] mu = K.reshape(mu, [-1, self.ncomp, self.outputd]) sigma = components[:, self.outputd:self.outputd + int(0.5 * (self.outputd + 1) * self.outputd), :] sigma = K.reshape( sigma, [-1, self.ncomp, int(0.5 * (self.outputd + 1) * self.outputd)]) alpha = components[:, -1, :] # alpha = K.softmax(K.clip(alpha,1e-8,1.)) alpha = K.clip(alpha, 1e-8, 1.) tfd = tfp.distributions mix = tfd.MixtureSameFamily( mixture_distribution=tfd.Categorical(probs=alpha), components_distribution=tfd.MultivariateNormalTriL( loc=mu, scale_tril=tril_matrix(sigma))) log_gauss = mix.log_prob(y_true) res = -K.mean(log_gauss) return res # This returns a tensor inputs = Input(shape=(None, self.inputd)) # Initializer with a particular seed initializer = lecun_normal(seed=1.) # Add the LSTM layer nn = CuDNNLSTM(self.nunits)(inputs) # Computes the random Fourier features rff = Lambda(self.rff_tf.toFeatures)(nn) FC_mus = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_mus')(rff) FC_sigmas_d = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_sigmas_d')( rff) # K.exp, W_regularizer=l2(1e-3) FC_sigmas = Dense( units=int(0.5 * (self.outputd - 1) * self.outputd * self.ncomp), activation='linear', kernel_initializer=initializer, name='FC_sigmas')(rff) # K.exp, W_regularizer=l2(1e-3) FC_alphas = Dense(units=self.ncomp, activation='softmax', kernel_initializer=initializer, name='FC_alphas')(rff) output = concatenate([FC_mus, FC_sigmas_d, FC_sigmas, FC_alphas], axis=1) self.model = Model(inputs=inputs, outputs=output) # Note: Replace 'rmsprop' by 'adam' depending on your needs. self.model.compile('adam', loss=mean_log_Gaussian_like)
def __init__(self, ncomp=10, nhidden=2, nunits=[50, 24, 24], inputd=None, outputd=None, nsteps=200, weights=None): self.ncomp = ncomp # number of mixture components self.nhidden = nhidden # number of hidden layers self.nunits = nunits # number of units per hidden layer (integer or array) self.inputd = inputd # dimensionality of the input self.nsteps = nsteps # number of time steps in the self.outputd = outputd # dimensionality of the output self.weights = weights # sample weights # Set for GPU use config = tf.ConfigProto(device_count={'CPU': 2, 'GPU': 2}) config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) def elu_modif(x, a=1.): e = 1e-15 return ELU(alpha=a)(x) + 1. + e # Note: The output size will be (outputd + 2) * ncomp # For reference, not used at the moment def log_sum_exp(x, axis=None): """Log-sum-exp trick implementation""" x_max = K.max(x, axis=axis, keepdims=True) return K.log(K.sum(K.exp(x - x_max), axis=axis, keepdims=True)) + x_max def tril_matrix(elements): tfd = tfp.distributions tril_m = tfd.fill_triangular(elements) tf.matrix_set_diag(tril_m, tf.exp(tf.matrix_diag_part(tril_m))) return tril_m def mean_log_Gaussian_like(y_true, parameters): # This version uses tensorflow_probability components = K.reshape(parameters, [ -1, self.outputd + int(0.5 * (self.outputd + 1) * self.outputd) + 1, self.ncomp ]) mu = components[:, :self.outputd, :] mu = K.reshape(mu, [-1, self.ncomp, self.outputd]) sigma = components[:, self.outputd:self.outputd + int(0.5 * (self.outputd + 1) * self.outputd), :] sigma = K.reshape( sigma, [-1, self.ncomp, int(0.5 * (self.outputd + 1) * self.outputd)]) alpha = components[:, -1, :] # alpha = K.softmax(K.clip(alpha,1e-8,1.)) alpha = K.clip(alpha, 1e-8, 1.) tfd = tfp.distributions mix = tfd.MixtureSameFamily( mixture_distribution=tfd.Categorical(probs=alpha), components_distribution=tfd.MultivariateNormalTriL( loc=mu, scale_tril=tril_matrix(sigma))) log_gauss = mix.log_prob(y_true) res = -K.mean(log_gauss) return res # This returns a tensor inputs = Input(shape=(self.nsteps, self.inputd)) # Initializer with a particular seed initializer = lecun_normal(seed=1.) # Add the LSTM layer # nn = CuDNNLSTM(self.nunits[0])(inputs) nn = LSTM(self.nunits[0])(inputs) # a layer instance is callable on a tensor, and returns a tensor # nn = Dense(self.nunits[0], activation='tanh', kernel_initializer=initializer)(nn) # nn = Dropout(0.05)(nn) # for i in range(self.nhidden - 2): # nn = Dense(self.nunits[i + 1], activation='tanh', kernel_initializer=initializer)(nn) # # nn = Dropout(0.05)(nn) FC_mus = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_mus')(nn) FC_sigmas_d = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_sigmas_d')( nn) # K.exp, W_regularizer=l2(1e-3) FC_sigmas = Dense( units=int(0.5 * (self.outputd - 1) * self.outputd * self.ncomp), activation='linear', kernel_initializer=initializer, name='FC_sigmas')(nn) # K.exp, W_regularizer=l2(1e-3) FC_alphas = Dense(units=self.ncomp, activation='softmax', kernel_initializer=initializer, name='FC_alphas')(nn) output = concatenate([FC_mus, FC_sigmas_d, FC_sigmas, FC_alphas], axis=1) self.model = Model(inputs=inputs, outputs=output) # Note: Replace 'rmsprop' by 'adam' depending on your needs. self.model.compile('adam', loss=mean_log_Gaussian_like) self.model.summary()
def __init__(self, ncomp=10, nfeat=500, inputd=None, cosOnly=False, kernel="RBF", sigma=1, quasiRandom=True, outputd=None, weights=None): self.ncomp = ncomp # number of mixture components self.nfeat = nfeat # number of features self.inputd = inputd # dimensionality of the input self.outputd = outputd # dimensionality of the output self.quasiRandom = quasiRandom self.cosOnly = cosOnly self.sigma = sigma * np.ones(self.inputd) self.kernel = kernel self.rff = RFF(self.nfeat, self.inputd, self.sigma, self.cosOnly, self.quasiRandom, self.kernel) self.weights = weights # weight function self.scaler = StandardScaler() # self.scaler = MinMaxScaler() # self.scaler = MaxAbsScaler() # self.scaler = Normalizer(norm='l2') # self.scaler = QuantileTransformer(output_distribution='normal', random_state=0) # Set up for CPU. Only when doing LSTM training we will use the GPU config = tf.ConfigProto(device_count={'CPU': 12, 'GPU': 0}) config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) def elu_modif(x, a=1.): e = 1e-15 return ELU(alpha=a)(x) + 1. + e # Note: The output size will be (outputd + 2) * ncomp # For reference, not used at the moment def log_sum_exp(x, axis=None): """Log-sum-exp trick implementation""" x_max = K.max(x, axis=axis, keepdims=True) return K.log(K.sum(K.exp(x - x_max), axis=axis, keepdims=True)) + x_max def tril_matrix(elements): tfd = tfp.distributions tril_m = tfd.fill_triangular(elements) tf.matrix_set_diag(tril_m, tf.exp(tf.matrix_diag_part(tril_m))) return tril_m def mean_log_Gaussian_like(y_true, parameters): # This version uses tensorflow_probability components = K.reshape(parameters, [ -1, self.outputd + int(0.5 * (self.outputd + 1) * self.outputd) + 1, self.ncomp ]) mu = components[:, :self.outputd, :] mu = K.reshape(mu, [-1, self.ncomp, self.outputd]) sigma = components[:, self.outputd:self.outputd + int(0.5 * (self.outputd + 1) * self.outputd), :] sigma = K.reshape( sigma, [-1, self.ncomp, int(0.5 * (self.outputd + 1) * self.outputd)]) alpha = components[:, -1, :] # alpha = K.softmax(K.clip(alpha,1e-8,1.)) alpha = K.clip(alpha, 1e-8, 1.) tfd = tfp.distributions mix = tfd.MixtureSameFamily( mixture_distribution=tfd.Categorical(probs=alpha), components_distribution=tfd.MultivariateNormalTriL( loc=mu, scale_tril=tril_matrix(sigma))) log_gauss = mix.log_prob(y_true) res = -K.mean(log_gauss) return res # This returns a tensor inputs = Input(shape=(self.nfeat, )) # Initializer with a particular seed # initializer = RandomUniform(minval=0., maxval=1.) initializer = lecun_normal(seed=1.) # Note: Replace 'rmsprop' by 'adam' depending on your needs. FC_mus = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_mus')(inputs) FC_sigmas_d = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer='Ones', kernel_regularizer=l2(10.), name='FC_sigmas_d')( inputs) # K.exp, W_regularizer=l2(1e-3) FC_sigmas = Dense( units=int(0.5 * (self.outputd - 1) * self.outputd * self.ncomp), activation='linear', kernel_initializer='Ones', kernel_regularizer=l2(1.), name='FC_sigmas')(inputs) # K.exp, W_regularizer=l2(1e-3) FC_alphas = Dense(units=self.ncomp, activation='softmax', kernel_initializer='Ones', name='FC_alphas')(inputs) # Adding a small constant to the main diagonal to avoid numerical problems output = concatenate( [FC_mus, FC_sigmas_d + 1e-4, FC_sigmas, FC_alphas], axis=1) self.model = Model(inputs=inputs, outputs=output) self.model.compile('adam', loss=mean_log_Gaussian_like)
def __init__(self, ncomp=10, nhidden=2, nunits=[24, 24], inputd=None, outputd=None, weights=None): self.ncomp = ncomp # number of mixture components self.nhidden = nhidden # number of hidden layers self.nunits = nunits # number of units per hidden layer (integer or array) self.inputd = inputd # dimensionality of the input self.outputd = outputd # dimensionality of the output self.weights = weights # sample weights # This returns a tensor inputs = Input(shape=(self.inputd, )) # Initializer with a particular seed initializer = lecun_normal(seed=1.) # a layer instance is callable on a tensor, and returns a tensor nn = Dense(self.nunits[0], activation='tanh', kernel_initializer=initializer)(inputs) # nn = Dropout(0.05)(nn) for i in range(self.nhidden - 2): nn = Dense(self.nunits[i + 1], activation='tanh', kernel_initializer=initializer)(nn) # nn = Dropout(0.05)(nn) FC_mus = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_mus')(nn) FC_sigmas_d = Dense(units=self.outputd * self.ncomp, activation='linear', kernel_initializer=initializer, name='FC_sigmas_d')( nn) # K.exp, W_regularizer=l2(1e-3) FC_sigmas = Dense( units=int(0.5 * (self.outputd - 1) * self.outputd * self.ncomp), activation='linear', kernel_initializer=initializer, name='FC_sigmas')(nn) # K.exp, W_regularizer=l2(1e-3) FC_alphas = Dense(units=self.ncomp, activation='softmax', kernel_initializer=initializer, name='FC_alphas')(nn) # Adding a small constant to the main diagonal to avoid numerical problems output = concatenate( [FC_mus, FC_sigmas_d + 1e-4, FC_sigmas, FC_alphas], axis=1) self.model = Model(inputs=inputs, outputs=output) # Note: Replace 'rmsprop' by 'adam' depending on your needs. self.model.compile('adam', loss=MDNN.mean_log_gaussian_loss( self.outputd, self.ncomp))
def __init__(self, model_features): super(cnn, self).__init__() """ Define here the layers used during the forward-pass of the neural network. """ self.l2_regularization_scale = model_features.l2_regularization_scale dropout_probability = model_features.dropout_probability self.batch_size = model_features.batch_size self.dense_nodes = model_features.dense_nodes regularizer = tf.contrib.layers.l2_regularizer( scale=self.l2_regularization_scale) number_filters = model_features.number_filters kernel_length = model_features.kernel_length kernel_strides = model_features.kernel_strides pool_size = model_features.pool_size pool_strides = model_features.pool_strides self.scaler = model_features.scaler cnn_activation_function = model_features.cnn_activation_function dnn_activation_function = model_features.dnn_activation_function output_size = model_features.output_size cnn_trainable = model_features.cnn_trainable # ################### # ##### 1D Conv ##### # ################### self.cnn_1d = {} for filter_index in range(len(kernel_length[0])): self.cnn_1d[str(filter_index)] = tf.layers.Conv1D( filters=number_filters[0], kernel_size=kernel_length[0][filter_index], strides=kernel_strides[0][filter_index], padding='same', activation=cnn_activation_function, kernel_initializer=tf.initializers.he_normal(), trainable=cnn_trainable) # ################### # ##### 2D Conv ##### # ################### self.cnn_2d = {} self.cnn_2d['0'] = tf.layers.Conv2D( filters=number_filters[1], kernel_size=(kernel_length[1], number_filters[0] * len(kernel_length[0])), strides=kernel_strides[1], padding='valid', activation=cnn_activation_function, kernel_initializer=tf.initializers.he_normal(), trainable=cnn_trainable) self.max_pooling2d = {} self.max_pooling2d['0'] = tf.layers.MaxPooling2D( pool_size=pool_size[1], strides=pool_strides[1], padding='same') # ################# # ##### Dense ##### # ################# self.flatten = tf.layers.Flatten() self.dense_layers = {} self.dropout_layers = {} for layer, nodes in enumerate(self.dense_nodes): self.dense_layers[str(layer)] = tf.layers.Dense( nodes, activation=dnn_activation_function, kernel_initializer=lecun_normal(), kernel_regularizer=regularizer) self.dropout_layers[str(layer)] = tf.layers.Dropout( dropout_probability) self.output_layer = tf.layers.Dense(output_size, activation=None)
def get_compiled_neumf_model(num_users, num_items, learning_rate=0.001, mf_dim=10, layers_num=[10], reg_layers=[0], reg_mf=0): assert len(layers_num) == len(reg_layers) num_layer = len(layers_num) #Number of layers in the MLP # Input variables user_input = layers.Input(shape=(1, ), dtype='int32', name='user_input') item_input = layers.Input(shape=(1, ), dtype='int32', name='item_input') # Embedding layer mf_embedding_user = layers.Embedding( input_dim=num_users, output_dim=mf_dim, name='mf_embedding_user', embeddings_initializer=initializers.RandomNormal(), embeddings_regularizer=regularizers.l2(reg_mf), input_length=1) mf_embedding_item = layers.Embedding( input_dim=num_items, output_dim=mf_dim, name='mf_embedding_item', embeddings_initializer=initializers.RandomNormal(), embeddings_regularizer=regularizers.l2(reg_mf), input_length=1) mlp_embedding_user = layers.Embedding( input_dim=num_users, output_dim=int(layers_num[0] / 2), name="mlp_embedding_user", embeddings_initializer=initializers.RandomNormal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), input_length=1) mlp_embedding_item = layers.Embedding( input_dim=num_items, output_dim=int(layers_num[0] / 2), name='mlp_embedding_item', embeddings_initializer=initializers.RandomNormal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), input_length=1) # MF part mf_user_latent = layers.Flatten()(mf_embedding_user(user_input)) mf_item_latent = layers.Flatten()(mf_embedding_item(item_input)) mf_vector = layers.multiply([mf_user_latent, mf_item_latent]) # MLP part mlp_user_latent = layers.Flatten()(mlp_embedding_user(user_input)) mlp_item_latent = layers.Flatten()(mlp_embedding_item(item_input)) mlp_vector = layers.concatenate([mlp_user_latent, mlp_item_latent]) for idx in range(1, num_layer): layer = layers.Dense(layers_num[idx], kernel_regularizer=regularizers.l2( reg_layers[idx]), activation='relu', name="layer%d" % idx) mlp_vector = layer(mlp_vector) # Concatenate MF and MLP parts predict_vector = layers.concatenate([mf_vector, mlp_vector]) # Final prediction layer prediction = layers.Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name="prediction")(predict_vector) model_nuemf = models.Model(inputs=[user_input, item_input], outputs=prediction) model_nuemf.compile(optimizer=optimizers.Adam(lr=learning_rate, clipnorm=0.5), loss='binary_crossentropy') return model_nuemf
def get_test_and_validation(csv_processed, train_size, test_size): train, test = train_test_split(csv_processed, train_size=train_size, test_size=test_size) batch_size = 892 train_ds = DataframeHelper.df_to_dataset(train, batch_size=batch_size) val_ds = DataframeHelper.df_to_dataset(test, batch_size=batch_size) return train_ds, val_ds feature_layer = tf.keras.layers.DenseFeatures( create_input_features_layer()) model = tf.keras.Sequential([ feature_layer, layers.Dense(158, activation='selu', kernel_initializer=initializers.lecun_normal()), layers.Dropout(0.2), layers.Dense(168, activation='swish', kernel_initializer=initializers.GlorotNormal()), layers.Dropout(0.2), layers.Dense(178, activation='swish', kernel_initializer=initializers.GlorotNormal()), layers.Dropout(0.2), layers.Dense(188, activation='selu', kernel_initializer=initializers.lecun_normal()), layers.Dropout(0.2), layers.Dense(1, activation="sigmoid",
def segsrgan_generator_block(name: str, shape: tuple, kernel: int): gen_initializer = lecun_normal() inputs = Input(shape=(1, shape[0], shape[1], shape[2])) # Representation gennet = ReflectPadding3D(padding=3)(inputs) gennet = Conv3D(kernel, 7, strides=1, kernel_initializer=gen_initializer, use_bias=False, name=name + '_gen_conv1', data_format='channels_first')(gennet) gennet = InstanceNormalization3D(name=name + '_gen_isnorm_conv1')(gennet) gennet = Activation('relu')(gennet) # Downsampling 1 gennet = ReflectPadding3D(padding=1)(gennet) gennet = Conv3D(kernel * 2, 3, strides=2, kernel_initializer=gen_initializer, use_bias=False, name=name + '_gen_conv2', data_format='channels_first')(gennet) gennet = InstanceNormalization3D(name=name + '_gen_isnorm_conv2')(gennet) gennet = Activation('relu')(gennet) # Downsampling 2 gennet = ReflectPadding3D(padding=1)(gennet) gennet = Conv3D(kernel * 4, 3, strides=2, kernel_initializer=gen_initializer, use_bias=False, name=name + '_gen_conv3', data_format='channels_first')(gennet) gennet = InstanceNormalization3D(name=name + '_gen_isnorm_conv3')(gennet) gennet = Activation('relu')(gennet) # Resnet blocks : 6, 8*4 = 32 gennet = resnet_blocks(gennet, kernel * 4, name=name + '_gen_block1') gennet = resnet_blocks(gennet, kernel * 4, name=name + '_gen_block2') gennet = resnet_blocks(gennet, kernel * 4, name=name + '_gen_block3') gennet = resnet_blocks(gennet, kernel * 4, name=name + '_gen_block4') gennet = resnet_blocks(gennet, kernel * 4, name=name + '_gen_block5') gennet = resnet_blocks(gennet, kernel * 4, name=name + '_gen_block6') # Upsampling 1 gennet = UpSampling3D(size=(2, 2, 2), data_format='channels_first')(gennet) gennet = ReflectPadding3D(padding=1)(gennet) gennet = Conv3D(kernel * 2, 3, strides=1, kernel_initializer=gen_initializer, use_bias=False, name=name + '_gen_deconv1', data_format='channels_first')(gennet) gennet = InstanceNormalization3D(name=name + '_gen_isnorm_deconv1')(gennet) gennet = Activation('relu')(gennet) # Upsampling 2 gennet = UpSampling3D(size=(2, 2, 2), data_format='channels_first')(gennet) gennet = ReflectPadding3D(padding=1)(gennet) gennet = Conv3D(kernel, 3, strides=1, kernel_initializer=gen_initializer, use_bias=False, name=name + '_gen_deconv2', data_format='channels_first')(gennet) gennet = InstanceNormalization3D(name=name + '_gen_isnorm_deconv2')(gennet) gennet = Activation('relu')(gennet) # Reconstruction gennet = ReflectPadding3D(padding=3)(gennet) gennet = Conv3D(1, 7, strides=1, kernel_initializer=gen_initializer, use_bias=False, name=name + '_gen_1conv', data_format='channels_first')(gennet) predictions = gennet model = Model(inputs=inputs, outputs=predictions, name=name) return model