def create_V2A_network(A_dim, V_dim): A_input = Input(shape=A_dim) AP = AveragePooling1D(pool_size=pool, strides=stride, padding='valid')(A_input) V_input = Input(shape=V_dim) VP = AveragePooling1D(pool_size=pool, strides=stride, padding='valid')(V_input) VL = LSTM(units=1024, return_sequences=True, stateful=False, dropout=0.2, recurrent_dropout=0.2, kernel_initializer=initializers.lecun_normal(), recurrent_initializer=initializers.lecun_uniform())(VP) VL = TimeDistributed( Dense(units=128, kernel_initializer=initializers.lecun_normal(), activation='tanh'))(VL) VT = TimeDistributed( Dense(units=128, kernel_initializer=initializers.lecun_normal(), activation='tanh'))(VP) VL = Average()([VL, VT]) distance = Lambda(QQ, output_shape=[L, 128])([VL, AP]) res_model = Model(inputs=[A_input, V_input], outputs=distance) #my_model.summary() return res_model
def network(categorical_columns_item, num_deep_numeric_feature, num_wide_numeric_feature, bias): input_layers = list() embedding_layers = list() # net categorical deep feature for col, num in categorical_columns_item.items(): input_deep_cat_layer = Input(shape=(1, ), name=col + "_categorical_deep_input") embedding_layer = Embedding( input_dim=num, output_dim=min(10, num // 2), embeddings_initializer=truncated_normal(mean=0, stddev=1 / np.sqrt(num)), input_length=1, name=col + "_deep_embedding")(input_deep_cat_layer) embedding_layer = (Reshape(target_shape=(min(10, num // 2), ), name=col + "_deep_reshape")(embedding_layer)) embedding_layer = Dropout(rate=0.15, noise_shape=(None, 1), name=col + "_deep_dropout")(embedding_layer) input_layers.append(input_deep_cat_layer) embedding_layers.append(embedding_layer) # net numeric deep feature input_deep_num_layer = Input(shape=(num_deep_numeric_feature, ), name="numeric_deep_input") input_layers.append(input_deep_num_layer) # net numeric wide feature input_wide_num_layer = Input(shape=(num_wide_numeric_feature, ), name="numeric_wide_input") input_layers.append(input_wide_num_layer) hidden_layer = Dense(units=32, kernel_initializer=lecun_normal(), activation="selu")(Concatenate()([ Concatenate()(embedding_layers), Dropout(rate=0.15)(input_deep_num_layer) ])) hidden_layer = Dense(units=16, kernel_initializer=lecun_normal(), activation="selu")(hidden_layer) hidden_layer = Dense(units=8, kernel_initializer=lecun_normal(), activation="selu")(hidden_layer) hidden_layer = Concatenate()([hidden_layer, input_wide_num_layer]) output_layer = Dense(units=1, kernel_initializer=lecun_normal(), bias_initializer=constant(logit(bias)), activation="sigmoid", name="output_layer")(hidden_layer) return Model(input_layers, output_layer)
def build_model(self, priors_corr, prior_test, Pi, input_shape, mode): self.prior_test = prior_test self.priors_corr = priors_corr self.Pi = Pi input = Input(shape=input_shape) x = Dropout(0.2, input_shape=input_shape)(input) x = Dense(300, use_bias=False, input_shape=input_shape, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(input) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dropout(0.2)(x) x = Dense(300, use_bias=False, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dropout(0.2)(x) x = Dense(300, use_bias=False, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dropout(0.2)(x) x = Dense(300, use_bias=False, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(x) x = BatchNormalization()(x) x = Activation('relu')(x) g = Dense(1, use_bias=True, kernel_initializer=initializers.lecun_normal(seed=1), activation='sigmoid', name="base_model")(x) base_model = Model(inputs=input, outputs=g) g_bar = Lambda(self.adaption_layer, name="last_layer")(g) model = Model(inputs=input, outputs=g_bar) self.compile_model( model=model, base_model=base_model, Pi=self.Pi, priors_corr=priors_corr, prior_test=prior_test, mode=self.mode, )
def double_nn(opt, dropout, a_func, reg_all, preds_reg, embed_dim, dense_units): user_embeddings = Embedding(u_cnt, embed_dim, embeddings_initializer=RandomNormal(mean=0.0, stddev=0.1), embeddings_regularizer=l2(reg_all), input_length=1, trainable=True, name='user_embeddings') song_embeddings = Embedding(s_cnt, embed_dim, embeddings_initializer=RandomNormal(mean=0.0, stddev=0.1), embeddings_regularizer=l2(reg_all), input_length=1, trainable=True,name='item_embeddings') uid_input = Input(shape=(1,), dtype='int32') embedded_usr = user_embeddings(uid_input) embedded_usr = Reshape((embed_dim,),name='user_embeddings_reshaped')(embedded_usr) ub = create_bias(uid_input, u_cnt, reg_all,'user_biases') sid_input = Input(shape=(1,), dtype='int32') embedded_song = song_embeddings(sid_input) embedded_song = Reshape((embed_dim,),name='item_embeddings_reshaped')(embedded_song) mb = create_bias(sid_input, s_cnt, reg_all,'item_biases') preds = concatenate([embedded_usr, embedded_song],name='concatenated_embeddings_all') if (a_func == 'relu') or (a_func == 'elu'): preds = Dense(dense_units, use_bias=True, activation=a_func, kernel_initializer=RandomNormal(mean=0.0, stddev=0.1), kernel_regularizer=regularizers.l2(preds_reg), bias_initializer=RandomNormal(mean=0.0, stddev=0.1),bias_regularizer=regularizers.l2(preds_reg))(preds) preds=Dropout(dropout)(preds) preds = Dense(1, use_bias=True, activation=a_func, kernel_initializer=RandomNormal(mean=0.0, stddev=0.1), kernel_regularizer=regularizers.l2(preds_reg), bias_initializer=RandomNormal(mean=0.0, stddev=0.1),bias_regularizer=regularizers.l2(preds_reg))(preds) preds=Dropout(dropout)(preds) elif a_func == 'selu': preds = Dense(dense_units, activation=a_func, kernel_initializer=lecun_normal(), name='main_hidden')(preds) preds=AlphaDropout(dropout)(preds) preds = Dense(1, activation=a_func, kernel_initializer=lecun_normal(), name='main_output')(preds) preds=AlphaDropout(dropout)(preds) preds = add([ub, preds]) preds = add([mb, preds]) model = Model(inputs=[uid_input, sid_input], outputs=preds) opt = eval(opt) model.compile(loss='mse', optimizer=opt, metrics=[mae]) return model
def get_model(num_users, num_items, latent_dim, regs=[0, 0]): # Input variables user_input = Input(shape=(1,), dtype='int32', name='user_input') item_input = Input(shape=(1,), dtype='int32', name='item_input') # MF_Embedding_User = Embedding(input_dim = num_users, output_dim = latent_dim, name = 'user_embedding', init = init_normal, W_regularizer = l2(regs[0]), input_length=1) MF_Embedding_User = Embedding(input_dim=num_users, output_dim=latent_dim, embeddings_initializer=initializers.random_normal(), embeddings_regularizer=l2(regs[0]), input_length=1, name='user_embedding') # MF_Embedding_Item = Embedding(input_dim = num_items, output_dim = latent_dim, name = 'item_embedding', init = init_normal, W_regularizer = l2(regs[1]), input_length=1) MF_Embedding_Item = Embedding(input_dim=num_items, output_dim=latent_dim, embeddings_initializer=initializers.random_normal(), embeddings_regularizer=l2(regs[1]), input_length=1, name='item_embedding') # Crucial to flatten an embedding vector! user_latent = Flatten()(MF_Embedding_User(user_input)) item_latent = Flatten()(MF_Embedding_Item(item_input)) # Element-wise product of user and item embeddings # predict_vector = merge([user_latent, item_latent], mode = 'mul') predict_vector = multiply([user_latent, item_latent]) # Final prediction layer # prediction = Lambda(lambda x: K.sigmoid(K.sum(x)), output_shape=(1,))(predict_vector) prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(predict_vector) model_ = Model(input=[user_input, item_input], output=prediction) return model_
def get_model(train, num_users, num_items, layers=[20, 10, 5, 2]): num_layer = len(layers) # Number of layers in the MLP user_matrix = K.constant(getTrainMatrix(train)) item_matrix = K.constant(getTrainMatrix(train).T) # Input variables user_input = Input(shape=(1,), dtype='int32', name='user_input') item_input = Input(shape=(1,), dtype='int32', name='item_input') user_rating = Lambda(lambda x: tf.gather(user_matrix, tf.to_int32(x)))(user_input) item_rating = Lambda(lambda x: tf.gather(item_matrix, tf.to_int32(x)))(item_input) user_rating = Reshape((num_items, ))(user_rating) item_rating = Reshape((num_users, ))(item_rating) MLP_Embedding_User = Dense(layers[0]//2, activation="linear" , name='user_embedding') MLP_Embedding_Item = Dense(layers[0]//2, activation="linear" , name='item_embedding') user_latent = MLP_Embedding_User(user_rating) item_latent = MLP_Embedding_Item(item_rating) # The 0-th layer is the concatenation of embedding layers vector = concatenate([user_latent, item_latent]) # MLP layers for idx in range(1, num_layer): layer = Dense(layers[idx], activation='relu', name='layer%d' % idx) vector = layer(vector) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(vector) model_ = Model(inputs=[user_input, item_input], outputs=prediction) return model_
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]): assert len(layers) == len(reg_layers) num_layer = len(layers) # Number of layers in the MLP # Input variables user_input = Input(shape=(1,), dtype='int32', name='user_input') item_input = Input(shape=(1,), dtype='int32', name='item_input') MLP_Embedding_User = Embedding(input_dim=num_users, output_dim=int(layers[0]/2), name='user_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=1) MLP_Embedding_Item = Embedding(input_dim=num_items, output_dim=int(layers[0]/2), name='item_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=1) # Crucial to flatten an embedding vector! user_latent = Flatten()(MLP_Embedding_User(user_input)) item_latent = Flatten()(MLP_Embedding_Item(item_input)) # The 0-th layer is the concatenation of embedding layers # vector = merge([user_latent, item_latent], mode = 'concat') vector = concatenate([user_latent, item_latent]) # MLP layers for idx in range(1, num_layer): layer = Dense(layers[idx], kernel_regularizer=l2(reg_layers[idx]), activation='relu', name='layer%d' % idx) vector = layer(vector) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(vector) model_ = Model(inputs=[user_input, item_input], outputs=prediction) return model_
def build_initializer(type, kerasDefaults, seed=None, constant=0.): if type == 'constant': return initializers.Constant(value=constant) elif type == 'uniform': return initializers.RandomUniform( minval=kerasDefaults['minval_uniform'], maxval=kerasDefaults['maxval_uniform'], seed=seed) elif type == 'normal': return initializers.RandomNormal(mean=kerasDefaults['mean_normal'], stddev=kerasDefaults['stddev_normal'], seed=seed) # Not generally available # elif type == 'glorot_normal': # return initializers.glorot_normal(seed=seed) elif type == 'glorot_uniform': return initializers.glorot_uniform(seed=seed) elif type == 'lecun_uniform': return initializers.lecun_uniform(seed=seed) elif type == 'lecun_normal': return initializers.lecun_normal(seed=seed) elif type == 'he_normal': return initializers.he_normal(seed=seed)
def test_lecun_normal(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) _runner(initializers.lecun_normal(), tensor_shape, target_mean=0., target_std=std)
def res_unit(input_layer, dense_num): tp1 = PReLU()(input_layer) tp2 = Dropout(0.2)(BatchNormalization()(tp1)) tp3 = Dense(dense_num, kernel_initializer=initializers.lecun_normal())(tp2) union_layer = Add()([input_layer, tp3]) return union_layer
def reference_lstm_nodense_medium(input_tensor, k_init=lecun_normal(seed=1337), k_reg=l1(), rec_reg=l1(), sf=False, imp=2): '''reference BiLSTM with batchnorm and NO dense layers. Output is already batch-normed. Expects ORIGINAL input batch size so pad/adjust window size accordingly!''' h = LSTM(100, kernel_initializer=k_init, return_sequences=True, recurrent_regularizer=rec_reg, kernel_regularizer=k_reg, implementation=imp, stateful=sf)(input_tensor) i = BatchNormalization()(h) j = LSTM(100, kernel_initializer=k_init, return_sequences=True, recurrent_regularizer=rec_reg, kernel_regularizer=k_reg, implementation=imp, stateful=sf)(i) j = BatchNormalization()(j) out = j return out
def reference_bilstm_nodense_big(input_tensor, k_init=lecun_normal(seed=1337), k_reg=l1(), rec_reg=l1(), sf=False, imp=2, dense_act='tanh'): '''reference SMALL BiLSTM with batchnorm and TD-dense. Expects ORIGINAL input batch size so pad/adjust window size accordingly!''' h = Bidirectional( LSTM(200, kernel_initializer=k_init, return_sequences=True, recurrent_regularizer=rec_reg, kernel_regularizer=k_reg, implementation=imp, stateful=sf))(input_tensor) i = BatchNormalization()(h) j = Bidirectional( LSTM(200, kernel_initializer=k_init, return_sequences=True, recurrent_regularizer=rec_reg, kernel_regularizer=k_reg, implementation=imp, stateful=sf))(i) j = BatchNormalization()(j) out = j return out
def build_model(shape, gru_layers, reg_layers, drop_layers): # Embedding Layer user_input = Input(shape=(1, ), dtype='int32', name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') time_input = Input(shape=(1, ), dtype='int32', name='time_input') user_embedding = Flatten()(Embedding( input_dim=shape[0], output_dim=gru_layers[0], embeddings_initializer=initializers.random_normal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), input_length=1, name='gru_user_embedding')(user_input)) item_embedding = Flatten()(Embedding( input_dim=shape[1], output_dim=gru_layers[0], embeddings_initializer=initializers.random_normal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), input_length=1, name='gru_item_embedding')(item_input)) time_embedding = Flatten()(Embedding( input_dim=shape[2], output_dim=gru_layers[1], embeddings_initializer=initializers.random_normal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), input_length=1, name='gru_time_embedding')(time_input)) user_embedding = Dropout(drop_layers[0])(user_embedding) item_embedding = Dropout(drop_layers[0])(item_embedding) time_embedding = Dropout(drop_layers[0])(time_embedding) gru_vector = Concatenate(axis=1)([user_embedding, item_embedding]) gru_vector = Reshape(target_shape=(int(gru_layers[1]), -1))(gru_vector) for index in range(1, len(gru_layers) - 1): layers = GRU(units=gru_layers[index], kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(reg_layers[index]), activation='tanh', recurrent_activation='hard_sigmoid', dropout=drop_layers[index], return_sequences=(index != (len(gru_layers) - 2)), name='gru_layer_%d' % index) gru_vector = layers([gru_vector, time_embedding]) gru_vector = Dropout(drop_layers[-1])(gru_vector) prediction = Dense(units=gru_layers[-1], activation='relu', kernel_initializer=initializers.lecun_normal(), kernel_regularizer=regularizers.l2(reg_layers[-1]), name='gru_prediction')(gru_vector) _model = Model(inputs=[user_input, item_input, time_input], outputs=prediction) return _model
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]): assert len(layers) == len(reg_layers) num_layer = len(layers) # Number of layers in the MLP # Input variables # user_input = Input(shape=(1,), dtype='int32', name='user_input') user_input = Input(shape=(1, ), name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') user_xz_input = Input(shape=(19, ), name='user_xz_input') MLP_Embedding_User = Embedding(input_dim=num_items, output_dim=int(layers[0] / 2), name='user_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=1) MLP_Embedding_Item = Embedding(input_dim=num_items, output_dim=int(layers[0] / 2), name='item_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=1) MLP_Embedding_User_xz = Embedding(input_dim=num_users, output_dim=int(layers[0]), name='user_xz_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=19) user_ = MLP_Embedding_User(user_input) item_ = MLP_Embedding_Item(item_input) ui_ = MLP_Embedding_User_xz(user_xz_input) lstm_1 = LSTM(64, activation='relu')(ui_) user_flatten = Flatten()(user_) item_flatten = Flatten()(item_) u_i_con = concatenate([user_flatten, item_flatten], name='u_i_con') dense_1 = Dense(128, activation='relu', name='dense_1')(u_i_con) dense_2 = Dense(64, activation='relu', name='dense_2')(dense_1) add_1 = add([dense_2, lstm_1], name='add_1') dense_3 = Dense(32, activation='relu', name='dense_3')(add_1) dense_4 = Dense(16, activation='relu', name='dense_4')(dense_3) dense_5 = Dense(8, activation='relu', name='dense_5')(dense_4) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(dense_5) model_ = Model(inputs=[user_xz_input, user_input, item_input], outputs=prediction) print(model_.summary()) return model_
def get_model(train, num_users, num_items, userlayers, itemlayers, layers): dmf_num_layer = len(userlayers) #Number of layers in the DMF mlp_num_layer = len(layers) #Number of layers in the MLP user_matrix = K.constant(getTrainMatrix(train)) item_matrix = K.constant(getTrainMatrix(train).T) # Input variables user_input = Input(shape=(1, ), dtype='int32', name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') # Embedding layer user_rating = Lambda( lambda x: tf.gather(user_matrix, tf.compat.v1.to_int32(x)))(user_input) item_rating = Lambda( lambda x: tf.gather(item_matrix, tf.compat.v1.to_int32(x)))(item_input) user_rating = Reshape((num_items, ))(user_rating) item_rating = Reshape((num_users, ))(item_rating) # DMF part userlayer = Dense(userlayers[0], activation="linear", name='user_layer0') itemlayer = Dense(itemlayers[0], activation="linear", name='item_layer0') dmf_user_latent = userlayer(user_rating) dmf_item_latent = itemlayer(item_rating) for idx in range(1, dmf_num_layer): userlayer = Dense(userlayers[idx], activation='relu', name='user_layer%d' % idx) itemlayer = Dense(itemlayers[idx], activation='relu', name='item_layer%d' % idx) dmf_user_latent = userlayer(dmf_user_latent) dmf_item_latent = itemlayer(dmf_item_latent) dmf_vector = multiply([dmf_user_latent, dmf_item_latent]) # MLP part MLP_Embedding_User = Dense(layers[0] // 2, activation="linear", name='user_embedding') MLP_Embedding_Item = Dense(layers[0] // 2, activation="linear", name='item_embedding') mlp_user_latent = MLP_Embedding_User(user_rating) mlp_item_latent = MLP_Embedding_Item(item_rating) mlp_vector = concatenate([mlp_user_latent, mlp_item_latent]) for idx in range(1, mlp_num_layer): layer = Dense(layers[idx], activation='relu', name="layer%d" % idx) mlp_vector = layer(mlp_vector) # Concatenate DMF and MLP parts predict_vector = concatenate([dmf_vector, mlp_vector]) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name="prediction")(predict_vector) model_ = Model(inputs=[user_input, item_input], outputs=prediction) return model_
def build_snn_model( x_shape=(100, 1, 10), n_layers=1, n_units=64, kernel_reg=1e-9, activity_reg=1e-9, bias_reg=1e-9, dropout_rate=0.5, optimizer='nadam', lr_rate=1e-5, gauss_noise_std=1e-3, n_gpus=0, ): """build snn model""" opts_map = { 'adam': opts.Adam, 'nadam': opts.Nadam, 'adamax': opts.Adamax, 'sgd': opts.SGD, 'rmsprop': opts.RMSprop } snn_cfg = { 'units': int(n_units), #'batch_input_shape': (batch_size, x_shape[1], x_shape[2]), #'batch_size': batch_size, 'input_shape': x_shape, 'kernel_regularizer': regularizers.l2(kernel_reg), 'activity_regularizer': regularizers.l2(activity_reg), 'bias_regularizer': regularizers.l2(bias_reg), 'kernel_initializer': initializers.lecun_normal(seed=cfg.data_cfg['random_seed']), 'activation': 'selu', } model = Sequential() model.add(Dense(**snn_cfg)) model.add(GaussianNoise(gauss_noise_std)) model.add(AlphaDropout(dropout_rate)) if n_layers > 1: for i in range(n_layers - 1): snn_cfg.pop('batch_input_shape', None) model.add(Dense(**snn_cfg)) model.add(GaussianNoise(gauss_noise_std)) model.add(AlphaDropout(dropout_rate)) model.add(Flatten()) # todo: WHy lol model.add(Dense(len(cfg.data_cfg['Target_param_names']))) opt = opts_map[optimizer](lr=lr_rate) model.compile(optimizer=opt, loss='mse') return model
def hint_model_1(number_of_labels, first_filters=16, input_shape=(224, 224, 3)): model = Sequential() # first layer model.add( Conv2D(filters=first_filters, input_shape=input_shape, kernel_size=2, strides=1, activation='relu', kernel_initializer=lecun_normal())) model.add(MaxPooling2D(pool_size=2, strides=2)) # second layer model.add( Conv2D(filters=first_filters * 2, kernel_size=2, strides=1, activation='relu', kernel_initializer=lecun_normal())) model.add(MaxPooling2D(pool_size=2, strides=2)) # third layer model.add( Conv2D(filters=first_filters * 4, kernel_size=2, strides=1, activation='relu', kernel_initializer=lecun_normal())) model.add(MaxPooling2D(pool_size=2, strides=2)) # model.add(GlobalAveragePooling2D()) model.add( Dense(units=number_of_labels, activation='softmax', kernel_initializer=lecun_normal())) # set the filepath for saving the model's weights save_filepath = "saved_models/weights.best.hint_model_1.hdf5" return model, save_filepath
def __conv_block(self, x, filters, kernel_size): out = Conv2D(filters, kernel_size=kernel_size, padding='same', activation='linear', kernel_initializer=initializers.lecun_normal(), kernel_regularizer=regularizers.l2(self.l2_const))(x) out = BatchNormalization(axis=1, momentum=0.9)(out) out = LeakyReLU(alpha=0.1)(out) return out
def hint_model_2_nin_bn(number_of_labels, first_filters=16, input_shape=(224, 224, 3)): ks = 2 # kernel + stride example ks=2 means (2,2) kernel and stride of 2 model = Sequential() model.add( Conv2D(filters=first_filters, kernel_size=1, strides=1, kernel_initializer=lecun_normal(), input_shape=input_shape)) model.add( Conv2D(filters=first_filters, kernel_size=ks, strides=ks, kernel_initializer=lecun_normal())) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=2, strides=2)) model.add( Conv2D(filters=first_filters * 2, kernel_size=1, strides=1, kernel_initializer=lecun_normal())) model.add( Conv2D(filters=first_filters * 2, kernel_size=ks, strides=ks, kernel_initializer=lecun_normal())) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=2, strides=2)) model.add( Conv2D(filters=first_filters * 4, kernel_size=1, strides=1, kernel_initializer=lecun_normal())) model.add( Conv2D(filters=first_filters * 4, kernel_size=ks, strides=ks, kernel_initializer=lecun_normal())) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=2, strides=2)) model.add(GlobalAveragePooling2D()) model.add( Dense(units=number_of_labels, activation='softmax', kernel_initializer=lecun_normal())) filepath = "saved_models/weights.best.hint_model_2_nin_bn.hdf5" return model, filepath
def res_model1(input_shape1, input_shape2): x_input = Input(shape=(input_shape1, ), name='x_input') output_1 = Dropout(0.3)(PReLU()(x_input)) output0 = Dense(input_shape2, kernel_initializer=initializers.lecun_normal())(output_1) output1 = res_unit(output0, input_shape2) # output7=Dense(output_shape,kernel_initializer=initializers.lecun_normal())(Dropout(0.3)(output6)) model = Model(inputs=x_input, outputs=output1) return model
def build_model(self, prior, input_shape): input = Input(shape=input_shape) x = Dense(300, use_bias=False, input_shape=input_shape, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(input) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dense(300, use_bias=False, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dense(300, use_bias=False, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dense(300, use_bias=False, kernel_initializer=initializers.lecun_normal(seed=1), kernel_regularizer=regularizers.l2(self.weight_decay))(x) x = BatchNormalization()(x) x = Activation('relu')(x) output = Dense(1, use_bias=True, kernel_initializer=initializers.lecun_normal(seed=1))(x) model = Model(inputs=input, outputs=output) self.compile_model(model=model, loss_type=self.loss_type, theta1=self.theta1, theta2=self.theta2, prior=prior, mode=self.mode)
def build_model(self, num_users, num_item, layers, reg_layers): assert len(layers) == len(reg_layers) # Input Layer user_id_input = Input(shape=(1,), dtype='int64', name='user_id_input') user_lc_input = Input(shape=(2,), dtype='int64', name='user_lc_input') item_id_input = Input(shape=(1,), dtype='int64', name='item_id_input') item_lc_input = Input(shape=(2,), dtype='int64', name='item_lc_input') user_id_embedding = self.getEmbedding(num_users, int(layers[0] / 4), 1, reg_layers[0], 'user_id_embedding') user_lc_embedding = self.getEmbedding(num_users, int(layers[0] / 4), 2, reg_layers[0], 'user_lc_embedding') item_id_embedding = self.getEmbedding(num_item, int(layers[0] / 4), 1, reg_layers[0], 'item_id_embedding') item_lc_embedding = self.getEmbedding(num_item, int(layers[0] / 4), 2, reg_layers[0], 'item_lc_embedding') user_id_latent = Flatten()(user_id_embedding(user_id_input)) user_lc_latent = Flatten()(user_lc_embedding(user_lc_input)) item_id_latent = Flatten()(item_id_embedding(item_id_input)) item_lc_latent = Flatten()(item_lc_embedding(item_lc_input)) # concatenate predict_user_vector = concatenate([user_id_latent, user_lc_latent]) predict_item_vector = concatenate([item_id_latent, item_lc_latent]) mlp_vector = concatenate([predict_user_vector, predict_item_vector]) # AC-COS cosine_vector = dot([user_lc_latent, item_lc_latent], axes=1, normalize=True) # AC_EUC #euclidean_vector = Lambda(self.euclidean_distance, output_shape=self.eucl_dist_output_shape)([user_lc_latent, item_lc_latent]) # Middle Layer for index in range(1, len(layers) - 1): layer = Dense(units=layers[index], kernel_initializer=initializers.random_normal(), kernel_regularizer=l2(reg_layers[index]), activation='relu', name='mlpLayer%d' % index) mlp_vector = layer(mlp_vector) predict_vector = concatenate([mlp_vector, cosine_vector]) # Output layer prediction = Dense(units=layers[-1], activation='linear', kernel_initializer=initializers.lecun_normal(), kernel_regularizer=l2(reg_layers[-1]), name='prediction')(predict_vector) _model = Model(inputs=[user_id_input, user_lc_input, item_id_input, item_lc_input], outputs=prediction) plot_model(_model, to_file='model.png') return _model
def make_cnn_lstm(word_index, max_seq): embeds, embed_dim = read_embeds(EFILE, word_index) embedding_layer = Embedding(len(word_index) + 1, embed_dim, weights=[embeds], input_length=max_seq, trainable=False) sequence_input = Input(shape=(max_seq, ), dtype='int32') embedded_sequences = embedding_layer(sequence_input) x = Reshape((-1, max_seq, embed_dim))(embedded_sequences) x = TimeDistributed( Conv1D(100, 3, activation='relu', padding='valid', kernel_initializer=lecun_normal(seed=None), input_shape=(1, max_seq, embed_dim)))(x) x = TimeDistributed(MaxPooling1D())(x) x = TimeDistributed(Flatten())(x) x = LSTM(300, activation="relu", kernel_initializer=lecun_normal(seed=None))(x) x = Dropout(0.2)(x) x = Dense(150, activation='relu', kernel_initializer=lecun_normal(seed=None))(x) x = Dense(75, activation='relu', kernel_initializer=lecun_normal(seed=None))(x) preds = Dense(1, activation='sigmoid')(x) return sequence_input, x, preds
def get_model(train, num_users, num_items, userlayers=[512, 64], itemlayers=[1024, 64]): num_layer = len(userlayers) # Number of layers in the MLP user_matrix = K.constant(getTrainMatrix(train)) item_matrix = K.constant(getTrainMatrix(train).T) # Input variables user = Input(shape=(1, ), dtype='int32', name='user_input') item = Input(shape=(1, ), dtype='int32', name='item_input') # Multi-hot User representation and Item representation user_input = Lambda(lambda x: tf.gather(user_matrix, tf.to_int32(x)))(user) item_input = Lambda(lambda x: tf.gather(item_matrix, tf.to_int32(x)))(item) user_input = Reshape((num_items, ))(user_input) item_input = Reshape((num_users, ))(item_input) print(user_input.shape, item_input.shape) # DMF part userlayer = Dense(userlayers[0], activation="linear", name='user_layer0') itemlayer = Dense(itemlayers[0], activation="linear", name='item_layer0') user_latent_vector = userlayer(user_input) item_latent_vector = itemlayer(item_input) print(user_latent_vector.shape, item_latent_vector.shape) for idx in range(1, num_layer): userlayer = Dense(userlayers[idx], activation='relu', name='user_layer%d' % idx) itemlayer = Dense(itemlayers[idx], activation='relu', name='item_layer%d' % idx) user_latent_vector = userlayer(user_latent_vector) item_latent_vector = itemlayer(item_latent_vector) print(user_latent_vector.shape, item_latent_vector.shape) predict_vector = multiply([user_latent_vector, item_latent_vector]) prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(predict_vector) print(prediction.shape) model_ = Model(inputs=[user, item], outputs=prediction) return model_
def model_3(input_shape): x_input1 = Input(shape=(input_shape, ), name='x_input1') x_input2 = Input(shape=(input_shape, ), name='x_input2') check_layer1 = Subtract()([x_input1, x_input2]) check_layer2 = Lambda(lambda x: K.square(x))(check_layer1) check_layer3 = Reshape((input_shape, 1))(check_layer2) check_layer4 = Conv1D( filters=1, kernel_size=input_shape, use_bias=False, kernel_constraint=Unit_Abs(), kernel_initializer=initializers.lecun_normal())(check_layer3) # output7=Dense(output_shape,kernel_initializer=initializers.lecun_normal())(Dropout(0.3)(output6)) check_layer5 = Flatten()(check_layer4) model = Model(inputs=[x_input1, x_input2], outputs=check_layer5) return model
def get_model(num_users, num_items, mf_dim, layers): num_layer = len(layers) # Number of layers in the MLP # Input variables user_input = Input(shape=(1,), dtype='int32', name='user_input') item_input = Input(shape=(1,), dtype='int32', name='item_input') # Embedding layer MF_Embedding_User = Embedding(input_dim=num_users, output_dim=mf_dim, name='mf_embedding_user', embeddings_initializer=initializers.random_normal(), input_length=1) MF_Embedding_Item = Embedding(input_dim=num_items, output_dim=mf_dim, name='mf_embedding_item', embeddings_initializer=initializers.random_normal(), input_length=1) MLP_Embedding_User = Embedding(input_dim=num_users, output_dim=int(layers[0] / 2), name="mlp_embedding_user", embeddings_initializer=initializers.random_normal(), input_length=1) MLP_Embedding_Item = Embedding(input_dim=num_items, output_dim=int(layers[0] / 2), name='mlp_embedding_item', embeddings_initializer=initializers.random_normal(), input_length=1) # MF part mf_user_latent = Flatten()(MF_Embedding_User(user_input)) mf_item_latent = Flatten()(MF_Embedding_Item(item_input)) # mf_vector = merge([mf_user_latent, mf_item_latent], mode = 'mul') # element-wise multiply mf_vector = multiply([mf_user_latent, mf_item_latent]) # MLP part mlp_user_latent = Flatten()(MLP_Embedding_User(user_input)) mlp_item_latent = Flatten()(MLP_Embedding_Item(item_input)) # mlp_vector = merge([mlp_user_latent, mlp_item_latent], mode = 'concat') mlp_vector = concatenate([mlp_user_latent, mlp_item_latent]) for idx in range(1, num_layer): layer = Dense(layers[idx], activation='relu', name="layer%d" % idx) mlp_vector = layer(mlp_vector) predict_vector = concatenate([mf_vector, mlp_vector]) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name="prediction")(predict_vector) model_ = Model(inputs=[user_input, item_input], outputs=prediction) return model_
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]): assert len(layers) == len(reg_layers) num_layer = len(layers) # Number of layers in the MLP # Input variables user_input = Input(shape=(1, ), dtype='int32', name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') MLP_Embedding_User = Embedding(input_dim=num_users, output_dim=int(layers[0] / 2), name='user_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=1) MLP_Embedding_Item = Embedding(input_dim=num_items, output_dim=int(layers[0] / 2), name='item_embedding', embeddings_regularizer=l2(reg_layers[0]), input_length=1) # Crucial to flatten an embedding vector! user_latent = Flatten()(MLP_Embedding_User(user_input)) item_latent = Flatten()(MLP_Embedding_Item(item_input)) # The 0-th layer is the concatenation of embedding layers # vector = merge([user_latent, item_latent], mode = 'concat') vector = concatenate([user_latent, item_latent]) vector_re = Reshape((1, 64))(vector) conv_1 = Conv1D(filters=32, kernel_size=1, padding='same')(vector_re) conv_2 = Conv1D(filters=16, kernel_size=1, padding='same')(conv_1) conv_3 = Conv1D(filters=8, kernel_size=1, padding='same')(conv_2) flatten_1 = Flatten()(conv_3) # Final prediction layer prediction = Dense(1, activation='sigmoid', kernel_initializer=initializers.lecun_normal(), name='prediction')(flatten_1) model_ = Model(inputs=[user_input, item_input], outputs=prediction) print(model_.summary()) return model_
def fit(self, X, Y, lambd, keep_prob, learning_rate, xavier=True, iterations=1000, seed=1, gradient_check=False, print_cost=False): ''' Trains the model given a X and Y and learning paramaters. Args: X (ndarry): Samples as columns, features in rows. Y(ndarry): Labels. lambd(float): If not None or 0, you will get L2 regularization with L2 penalty. keep_prob(float): If less than 1.0, dropout regularization will be implemented. learning_rate(float): Learning rate. xavier(boolean): True for Xavier initialization otherwise random initialization. iterations(int): Number of iterations. seed(int): Ramdom number generator seed. gradient_check(boolean): Switches off dropout to allow checking gradient with a numerical check. print_cost(boolean): True to print cost as you train. ''' if type(keep_prob) == list: assert len(keep_prob) == len(self.activations), 'keep_prob array must much activation dimension''' if type(keep_prob) != list: self.keep_prob = np.ones(len(self.activations)) * (keep_prob) else: self.keep_prob = [x for x in keep_prob] inputs = Input(shape=[self.L[0], ]) x = inputs for i in range(1, len(self.L)): name = str(i) print(name, self.activations[i - 1], self.keep_prob[i - 1]) x = Dense(self.L[i], name=name + 'Z', kernel_initializer=lecun_normal(seed=seed), kernel_regularizer=l2(lambd))(x) x = Activation(self.activations[i - 1], name=name + 'A')(x) x = Dropout(1 - self.keep_prob[i - 1], seed=seed, name=name + 'D')(x) output_layer = x self.model = Model(input=[inputs], output=[output_layer]) self.model.compile(optimizer=SGD(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy']) self.model.fit(x=X, y=Y, verbose=0, epochs=iterations, batch_size=X.shape[0], shuffle=False)
def build_model(shape, gtf_layers, reg_layers, drop_layers): # Embedding Layer user_input = Input(shape=(1,), dtype='int32', name='user_input') item_input = Input(shape=(1,), dtype='int32', name='item_input') time_input = Input(shape=(1,), dtype='int32', name='time_input') user_embedding = Flatten()(Embedding(input_dim=shape[0], output_dim=gtf_layers[0], input_length=1, embeddings_initializer=initializers.random_normal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), name='gtf_user_embedding')(user_input)) item_embedding = Flatten()(Embedding(input_dim=shape[1], output_dim=gtf_layers[0], input_length=1, embeddings_initializer=initializers.random_normal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), name='gtf_item_embedding')(item_input)) time_embedding = Flatten()(Embedding(input_dim=shape[2], output_dim=gtf_layers[0], input_length=1, embeddings_initializer=initializers.random_normal(), embeddings_regularizer=regularizers.l2(reg_layers[0]), name='gtf_time_embedding')(time_input)) user_embedding = Dropout(drop_layers[0])(user_embedding) item_embedding = Dropout(drop_layers[0])(item_embedding) time_embedding = Dropout(drop_layers[0])(time_embedding) us = Dot(axes=-1)([user_embedding, item_embedding]) ut = Dot(axes=-1)([user_embedding, time_embedding]) st = Dot(axes=-1)([item_embedding, time_embedding]) mf_vector = Add()([us, ut, st]) mf_vector = Dropout(drop_layers[-1])(mf_vector) prediction = Dense(units=gtf_layers[-1], activation='relu', use_bias=True, kernel_initializer=initializers.lecun_normal(), kernel_regularizer=regularizers.l2(reg_layers[-1]), name='gtf_prediction')(mf_vector) _model = Model(inputs=[user_input, item_input, time_input], outputs=prediction) return _model
def fit_model(X, Y, Z=None, num_nodes=0, a=1, b=0.5, size=2, kernel='power', epochs=50, batch_size=100, optimizer='adamax', loss="mse", seed=1): from keras.layers import Input, Embedding, BatchNormalization from keras.models import Model from keras.initializers import lecun_normal from .layers import get_distance_layer walk_len = X.shape[1] # layer specification inp = Input(shape=(walk_len, )) if Z is not None: embedding = Embedding(num_nodes, size, weights=[Z])(inp) else: embedding = Embedding(num_nodes, size, embeddings_initializer=lecun_normal(seed))(inp) batchn = BatchNormalization()(embedding) distance = get_distance_layer(kernel, batchn, walk_len, a, b, seed) # build and compile model model = Model(inp, distance) model.compile(optimizer, loss) model.fit(X, Y, epochs=epochs, batch_size=batch_size) Z = model.layers[1].get_weights()[0] return Z
def test_lecun_normal(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) _runner(initializers.lecun_normal(), tensor_shape, target_mean=0., target_std=std)