def H(self, inputs, num_filters): ''' THIS FUNCTION SIMULES THE BEHAVIOR OF EXPRESSION PRESENT IN PAPER: - xl=H(xl−1)+xl−1 * H: represents a composite function which takes in an image/feature map ( x ) and performs some operations on it. * x → Batch Normalization → ReLU → Zero Padding → 3×3 Convolution → Dropout :param inputs: previous layer :param num_filters: integer: number of filters of CNN layer :return: Convolution Layer output ''' conv_out = BatchNormalization(axis=3)(inputs) conv_out = Activation(config.RELU_FUNCTION)(conv_out) bootleneck_filters = num_filters * 4 ## paper conv_out = Conv2D(bootleneck_filters, kernel_size=(1, 1), use_bias=False, padding=config.SAME_PADDING, kernel_initializer=he_uniform( config.HE_SEED))(conv_out) #conv_out = Dropout(0.2)(conv_out) conv_out = BatchNormalization(axis=3)(conv_out) conv_out = Activation(config.RELU_FUNCTION)(conv_out) #conv_out = ZeroPadding2D((1, 1))(conv_out) conv_out = Conv2D(num_filters, kernel_size=(3, 3), use_bias=False, padding=config.SAME_PADDING, kernel_initializer=he_uniform( config.HE_SEED))(conv_out) #conv_out = Dropout(0.2)(conv_out) return conv_out
def unit_2(in_layer, n1=64, n2=64, n3=256, p2=1, d2=1): ''' Shortcut Unit :param in_layer: :return: ''' x = Conv2D(n1, (1, 1), strides=(1, 1), padding='valid', kernel_initializer=he_uniform(), use_bias=False)(in_layer) x = BatchNormalization(momentum=0.95)(x) x = Activation('relu')(x) x = ZeroPadding2D(padding=(p2, p2))(x) x = Conv2D(n2, (3, 3), strides=(1, 1), padding='valid', dilation_rate=(d2, d2), kernel_initializer=he_uniform(), use_bias=False)(x) x = BatchNormalization(momentum=0.95)(x) x = Activation('relu')(x) x = Conv2D(n3, (1, 1), strides=(1, 1), padding='valid', kernel_initializer=he_uniform(), use_bias=False)(x) x = BatchNormalization(momentum=0.95)(x) x = add([in_layer, x]) x = Activation('relu')(x) return x
def create_critic_network(self, state_size, action_dim): ActionInputs = [] StateInputs = [] for i in range(self.N): ActionInputs.append(Input(shape=(action_dim[i], ))) StateInputs.append(Input(shape=(state_size[i], ))) S = concatenate(StateInputs) A = concatenate(ActionInputs) a1 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(S) h1 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(A) h2 = concatenate([h1, a1]) h3 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h2) h4 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h3) V = Dense(1, activation='linear', kernel_initializer=he_uniform(seed=0))(h4) model = Model(inputs=StateInputs + ActionInputs, outputs=V) adam = Adam(lr=self.LEARNING_RATE) model.compile(loss='mse', optimizer=adam) return model, ActionInputs, StateInputs
def sparseNN(): sparse_data = Input(shape=[train_keras["sparse_data"].shape[1]], dtype='float32', sparse=True, name='sparse_data') item_condition = Input(shape=[1], name="item_condition") shipping = Input(shape=[1], name="shipping") temp = Input(shape=[1], name="temp") temp2 = Input(shape=[1], name="temp2") x = Dense(200, kernel_initializer=he_uniform(seed=0))(sparse_data) x = PReLU()(x) x = concatenate([x, item_condition, shipping, temp, temp2]) x = Dense(200, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = Dense(100, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = Dense(1)(x) model = Model([sparse_data, item_condition, shipping, temp, temp2], x) optimizer = Adam(.0011) model.compile(loss="mse", optimizer=optimizer) return model
def identity_block(X, f, filters, stage, block): """ Implementation of the identity block Arguments: X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev) f -- integer, specifying the shape of the middle CONV's window for the main path filters -- python list of integers, defining the number of filters in the CONV layers of the main path stage -- integer, used to name the layers, depending on their position in the network block -- string/character, used to name the layers, depending on their position in the network Returns: X -- output of the identity block, tensor of shape (n_H, n_W, n_C) """ # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value. You'll need this later to add back to the main path. X_shortcut = X # First component of main pathonvolved with the layer input to produce X = Conv2D(filters=F1, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2a', kernel_initializer=he_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X) X = Activation('relu')(X) # Second component of main path X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=he_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=he_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X) # Add shortcut value to main path X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X
def convolutional_block(X, f, filters, stage, block, s=2): """ Implementation of the convolutional block Arguments: X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev) f -- integer, specifying the shape of the middle CONV's window for the main path filters -- python list of integers, defining the number of filters in the CONV layers of the main path stage -- integer, used to name the layers, depending on their position in the network block -- string/character, used to name the layers, depending on their position in the network s -- Integer, specifying the stride to be used Returns: X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C) """ # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2 = filters # Save the input value X_shortcut = X ##### MAIN PATH ##### X = Conv2D(filters=F2, kernel_size=(f, f), strides=(s, s), padding='same', name=conv_name_base + '2a', kernel_initializer=he_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X) X = Activation('relu')(X) X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=he_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X) ##### SHORTCUT PATH #### X_shortcut = Conv2D(filters=F1, kernel_size=(1, 1), strides=(s, s), padding='valid', name=conv_name_base + '1', kernel_initializer=he_uniform(seed=0))(X_shortcut) X_shortcut = BatchNormalization(axis=3, name=bn_name_base + '1')(X_shortcut) # Add shortcut value to main path X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X
def create_actor_network(self, state_size, action_dim): S = Input(shape=(state_size,)) h0 = Dense(HIDDEN1_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(S) h1 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h0) h2 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h1) V = Dense(action_dim, activation='tanh', kernel_initializer=he_uniform(seed=0))(h2) model = Model(inputs=S, outputs=V) return model, model.trainable_weights, S
def convolution_block(self, tensor_input, *args): ''' THIS FUNCTIONS REPRESENTS THE CONCEPT OF CONVOLUTION BLOCK ON RESNET, COMBINING MAIN PATH AND SHORTCUT paper: https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf residual model image (outout same): https://www.youtube.com/watch?v=wqkc-sj5H94 :param tensor_input: input_tensor result of previous block application on cnn architecture (conv_block or identity_block) :param args: number of filters to populate conv2d layers :return: tensor merge of path created using convs and final shortcut ''' try: ## save copy input, because i need to apply alteration on tensor_input parameter, and in final i need to merge this two tensors shortcut_path = tensor_input tensor_input = Conv2D( filters=args[0], padding=config.SAME_PADDING, kernel_size=(3, 3), strides=args[1], # in paper 1 conv layer in 1 conv_block have stride=1, i continue with stride=2, in order to reduce computacional costs) kernel_initializer=he_uniform(config.HE_SEED), kernel_regularizer=l2(config.DECAY))(tensor_input) tensor_input = BatchNormalization(axis=3)( tensor_input ) ## perform batch normalization alongside channels axis [samples, width, height, channels] tensor_input = Activation(config.RELU_FUNCTION)(tensor_input) tensor_input = Conv2D( filters=args[0], padding=config.SAME_PADDING, kernel_size=(3, 3), strides=1, kernel_initializer=he_uniform(config.HE_SEED), kernel_regularizer=l2(config.DECAY))(tensor_input) tensor_input = BatchNormalization(axis=3)( tensor_input ) ## perform batch normalization alongside channels axis [samples, width, height, channels] tensor_input = Activation(config.RELU_FUNCTION)(tensor_input) ## definition of shortcut path shortcut_path = Conv2D( filters=args[0], kernel_size=(1, 1), strides=args[1], padding=config.SAME_PADDING, kernel_initializer=he_uniform(config.HE_SEED), kernel_regularizer=l2(config.DECAY))(shortcut_path) shortcut_path = BatchNormalization(axis=3)(shortcut_path) ## now i need to merge conv path and shortcut path, this is passed to activation function tensor_input = Add()([tensor_input, shortcut_path]) tensor_input = Activation(config.RELU_FUNCTION)(tensor_input) return tensor_input except: raise CustomError.ErrorCreationModel(config.ERROR_ON_CONV_BLOCK)
def build_model(): sparse_params = Input(shape=[X_train['sparse_params'].shape[1]], dtype='float32', sparse=True, name='sparse_params') categorical_inputs = [] for cat in cat_cols: categorical_inputs.append(Input(shape=[1], name=cat)) categorical_embeddings = [] for i, cat in enumerate(cat_cols): categorical_embeddings.append( Embedding(embed_sizes[i], 10, embeddings_regularizer=l2(0.00001))(categorical_inputs[i])) categorical_logits = Concatenate()([Flatten()(cat_emb) for cat_emb in categorical_embeddings]) categorical_logits = prense(categorical_logits, 256) categorical_logits = prense(categorical_logits, 128) numerical_inputs = Input(shape=[len(num_cols)], name='numerical') numerical_logits = numerical_inputs numerical_logits = BatchNormalization()(numerical_logits) numerical_logits = prense(numerical_logits, 256) numerical_logits = prense(numerical_logits, 128) params_logits = prense(sparse_params, 64) params_logits = prense(params_logits, 32) desc_inp = Input(shape=[max_len_desc], name='desc') title_inp = Input(shape=[max_len_title], name='title') embedding = Embedding(nb_words, embed_size, weights=[embedding_matrix], trainable=False) # nb_words emb_desc = embedding(desc_inp) emb_title = embedding(title_inp) emb_text = Concatenate(axis=1)([emb_desc,emb_title]) text_logits = SpatialDropout1D(0.2)(emb_text) text_logits = Bidirectional(CuDNNLSTM(128, return_sequences=True))(text_logits) text_logits = Conv1D(64, kernel_size=3, padding="valid", kernel_initializer="glorot_uniform")(text_logits) avg_pool = GlobalAveragePooling1D()(text_logits) max_pool = GlobalMaxPool1D()(text_logits) text_logits = Concatenate()([avg_pool, max_pool]) x = Dropout(0.2)(text_logits) x = Concatenate()([categorical_logits, text_logits]) x = BatchNormalization()(x) x = Concatenate()([x, params_logits, numerical_logits]) x = Dense(512, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = Dense(256, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = Dense(128, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = LayerNorm1D()(x) out = Dense(1, activation='sigmoid')(x) model = Model(inputs=[desc_inp] + [title_inp] + [sparse_params] + categorical_inputs + [numerical_inputs], outputs=out) model.compile(optimizer=Adam(lr=0.0005, clipnorm=0.5), loss='mean_squared_error', metrics=[root_mean_squared_error]) return model
def build(self, *args, trainedModel=None) -> Sequential: ''' THIS FUNCTION IS RESPONSIBLE FOR THE INITIALIZATION OF SEQUENTIAL ALEXNET MODEL Reference: https://arxiv.org/pdf/1608.06993.pdf --> Original Paper Reference: https://github.com/liuzhuang13/DenseNet/blob/master/models/densenet.lua --> Original Author of DenseNet Paper :param args: list integers, in logical order --> to populate cnn (filters) and dense (neurons) :return: Sequential: AlexNet MODEL ''' try: #IF USER ALREADY HAVE A TRAINED MODEL, AND NO WANTS TO BUILD AGAIN A NEW MODEL if trainedModel != None: return trainedModel input_shape = (config.WIDTH, config.HEIGHT, config.CHANNELS) input = Input(shape=(input_shape)) x = Conv2D(args[0], kernel_size=(5, 5), use_bias=False, kernel_initializer=he_uniform(config.HE_SEED), strides=2, padding=config.SAME_PADDING, kernel_regularizer=regularizers.l2(1e-4))(input) x = BatchNormalization(axis=3)(x) x = Activation(config.RELU_FUNCTION)(x) x = MaxPooling2D(pool_size=(3, 3), strides=2, padding='same')(x) nFilters = args[0] for i in range(args[1]): x = self.dense_block( x, args[2], args[3], args[3] ) # initial number of filters is equal to growth rate, and all conv's uses all same number of filters: growth rate if i < (args[1] - 1): x = self.transition( x, args[4] ) ## in last block (final step doesn't apply transition logic, global average pooling, made this x = BatchNormalization(axis=3)(x) x = Activation(config.RELU_FUNCTION)(x) x = GlobalAveragePooling2D()(x) x = Dense(config.NUMBER_CLASSES, kernel_initializer=he_uniform(config.HE_SEED), kernel_regularizer=regularizers.l2(1e-4))( x) # Num Classes for CIFAR-10 outputs = Activation(config.SOFTMAX_FUNCTION)(x) model = mp(input, outputs) if config.BUILD_SUMMARY == 1: model.summary() return model except: raise CustomError.ErrorCreationModel(config.ERROR_ON_BUILD)
def conv2d_compress_block(input_tensor, n_filters, init_seed=None): x = Conv2D(filters=n_filters, kernel_size=(1, 1), kernel_initializer=he_uniform(seed=init_seed), bias_initializer=he_uniform(seed=init_seed), padding='same')(input_tensor) x = LeakyReLU()(x) x = BatchNormalization()(x) return x
def vgg_block(filters, layers, x_input): x = Conv2D(filters, (3, 3), padding='same', kernel_initializer=he_uniform())(x_input) x = BatchNormalization()(x) x = Activation('relu')(x) for _ in range(layers-1): x = Conv2D(filters, (3, 3), padding='same', kernel_initializer=he_uniform())(x) x = BatchNormalization()(x) x = Activation('relu')(x) return x
def build_discriminator(self): # Input Layer inputs = [] # Embedding Layer embeddings = [] for idx, key in enumerate(self.keys): if key == 'mask': continue if key == 'lat_lon': i = Input(shape=(self.max_length, self.vocab_size[key]), name='input_' + key) unstacked = Lambda(lambda x: tf.unstack(x, axis=1))(i) d = Dense(units=64, use_bias=True, activation='relu', kernel_initializer=he_uniform(seed=1), name='emb_' + key) dense_latlon = [d(x) for x in unstacked] e = Lambda(lambda x: tf.stack(x, axis=1))(dense_latlon) else: i = Input(shape=(self.max_length, self.vocab_size[key]), name='input_' + key) unstacked = Lambda(lambda x: tf.unstack(x, axis=1))(i) d = Dense(units=self.vocab_size[key], use_bias=True, activation='relu', kernel_initializer=he_uniform(seed=1), name='emb_' + key) dense_attr = [d(x) for x in unstacked] e = Lambda(lambda x: tf.stack(x, axis=1))(dense_attr) inputs.append(i) embeddings.append(e) # Feature Fusion Layer concat_input = Concatenate(axis=2)(embeddings) unstacked = Lambda(lambda x: tf.unstack(x, axis=1))(concat_input) d = Dense(units=100, use_bias=True, activation='relu', kernel_initializer=he_uniform(seed=1), name='emb_trajpoint') dense_outputs = [d(x) for x in unstacked] emb_traj = Lambda(lambda x: tf.stack(x, axis=1))(dense_outputs) # LSTM Modeling Layer (many-to-one) lstm_cell = LSTM(units=100, recurrent_regularizer=l1(0.02))(emb_traj) # Output sigmoid = Dense(1, activation='sigmoid')(lstm_cell) return Model(inputs=inputs, outputs=sigmoid)
def create_critic_network(self, state_size, action_size): S = Input(shape=(state_size,)) h1 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(S) h2 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h1) h3 = Dense(HIDDEN2_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h2) h4 = Dense(HIDDEN1_UNITS, activation='relu', kernel_initializer=he_uniform(seed=0))(h3) # h5 = Dense(action_size + 1, activation='linear')(h4) # V = Lambda(lambda i: K.expand_dims(i[:, 0], -1) + i[:, 1:] - K.mean(i[:, 1:], keepdims=True), # output_shape=(self.action_size,))(h5) V = Dense(action_size, activation='linear', kernel_initializer=he_uniform(seed=0))(h4) model = Model(inputs=S, outputs=V) adam = Adam(lr=self.LEARNING_RATE) model.compile(loss='mse', optimizer=adam) return model, S
def build_model(self): '''以Model(input,output)的方式建立模型。''' image = Input(self.in_shape) # input image # ============= # initial block # ============= conv_i = Conv2D(self.channels_1st_conv, self.first_conv_params["fsize"], strides=self.first_conv_params["stride"], padding="SAME", use_bias=False, kernel_regularizer=l2(self.wd), kernel_initializer=he_uniform()) bn_i = BatchNormalization(gamma_regularizer=l2(self.wd), beta_regularizer=l2(self.wd)) act_i = Activation("relu") if self.first_conv_params["maxpool"] is False: x = conv_i(act_i(bn_i(image))) else: max_pool_i = MaxPooling2D(pool_size=3, strides=2) x = max_pool_i(conv_i(act_i(bn_i(image)))) # ========================= # dense +transition blocks # ========================= channels = self.channels_1st_conv for i, num_layers in enumerate(self.block_layers): x = self.dense_block(num_layers, self.growth_rate, x) channels += num_layers * self.growth_rate if i != len(self.block_layers) - 1: x = self.trans_block(channels // 2, x) # ============= # final block # ============= bn_f = BatchNormalization(gamma_regularizer=l2(self.wd), beta_regularizer=l2(self.wd)) act1_f = Activation("relu") glb_avg_pool_f = GlobalAveragePooling2D() dense_f = Dense(self.out_classes, kernel_regularizer=l2(self.wd), bias_regularizer=l2(self.wd), kernel_initializer=he_uniform()) act2_f = Activation("softmax") x = act2_f(dense_f(glb_avg_pool_f(act1_f(bn_f(x))))) # ========================================================== # connect the input & output tensor in order to form a model # ========================================================== model = Model(image, x) return model
def _set_default_initializer(self, kwargs): """ Sets the default initializer for convolution 2D and Seperable convolution 2D layers to Convolutional Aware or he_uniform. if a specific initializer has been passed in from the model plugin, then the specified initializer will be used rather than the default. Parameters ---------- kwargs: dict The keyword arguments for the current layer Returns ------- dict The keyword arguments for the current layer with the initializer updated to the select default value """ if "kernel_initializer" in kwargs: logger.debug("Using model specified initializer: %s", kwargs["kernel_initializer"]) return kwargs if self.use_convaware_init: default = ConvolutionAware() if self.first_run: # Indicate the Convolutional Aware should be calculated on first run default._init = True # pylint:disable=protected-access else: default = he_uniform() if kwargs.get("kernel_initializer", None) != default: kwargs["kernel_initializer"] = default logger.debug("Set default kernel_initializer to: %s", kwargs["kernel_initializer"]) return kwargs
def training_rpn_model(feature_extractor, anchors_per_loc, seed): conv_init = he_uniform(seed) cls_reg_init = glorot_uniform(seed) conv = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_initializer=conv_init, activation='relu', name='RPN_conv')(feature_extractor.output) cls = Conv2D(filters=1 * anchors_per_loc, kernel_size=(1, 1), kernel_initializer=cls_reg_init, activation='sigmoid', name='RPN_cls')(conv) reg = Conv2D(filters=4 * anchors_per_loc, kernel_size=(1, 1), kernel_initializer=cls_reg_init, activation=expanded_sigmoid, name='RPN_reg')(conv) cls = Reshape(target_shape=(-1, ), name='bbox_cls')(cls) reg = Reshape(target_shape=(-1, 4), name='bbox_reg')(reg) return Model(inputs=feature_extractor.input, outputs=[cls, reg, feature_extractor.output], name='RPN')
def keras_setup(): #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras init = initializers.he_uniform(seed=seed) # he initializer를 seed 없이 매번 random하게 사용 -> seed 줌 input1 = []; [input1.append([]) for u in range(rowsize*colsize)] input2 = []; [input2.append([]) for u in range(rowsize*colsize)] for u in range(rowsize*colsize): input1[u] = keras.layers.Input(shape=(sequencesize, fn)) # 각 병렬 layer shape에 따라 input 받음 input2[u] = Bidirectional(LSTM(n_hidden))(input1[u]) # biRNN -> 시계열에서 단일 value로 나감 input2[u] = Dense(layer_1, kernel_initializer = init, activation='relu')(input2[u]) # fully conneted layers, relu input2[u] = Dropout(dropout_rate1)(input2[u]) # dropout input2[u] = Dense(layer_1, kernel_initializer = init, activation='relu')(input2[u]) # fully conneted layers, relu input2[u] = Dropout(dropout_rate1)(input2[u]) # dropout added = keras.layers.Add()(input2) # 병렬구조를 여기서 모두 합침 merge1 = Dense(layer_1, kernel_initializer = init, activation='relu', \ kernel_regularizer=regularizers.l2(l2_rate))(added) # fully conneted layers, relu merge1 = Dropout(dropout_rate2)(merge1) # dropout merge1 = Dense(layer_1, kernel_initializer = init, activation='relu', \ kernel_regularizer=regularizers.l2(l2_rate))(merge1) # fully conneted layers, sigmoid merge2 = Dense(n_out, kernel_initializer = init, activation='sigmoid')(merge1) # fully conneted layers, relu merge2 = Activation('softmax')(merge2) # activation as softmax function model = keras.models.Model(inputs=input1, outputs=merge2) # input output 선언 model.compile(loss='categorical_crossentropy', \ optimizer=Adam(lr=lr, beta_1=0.9, beta_2=0.999), \ metrics=['accuracy']) # optimizer #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras #### keras return model
def prepare_model(): global scaler data = import_data(ticker, timeframe, start_train_date, end_train_date, calculate_input, lookback, calculate_output, lookforward, split=(100, 0, 0)) scaler = StandardScaler() # Creating an instance of a scaler. scaler.fit(data['train_input']) # Fitting the scaler. data_scaled = scaler.transform(data['train_input']) # Normalizing data m = Sequential() m.add( Dense(units=num_features, activation='tanh', input_dim=num_features, kernel_initializer=he_uniform(1))) m.add(Dense(num_features, activation='tanh')) m.add(Dense(1, activation='linear')) m.compile(loss='mean_squared_error', optimizer='sgd') m.fit(data_scaled, data['train_output'], epochs=num_epochs) return m
def mlp(n_layers, input_dim, units, lr_rate, drop_rate, seed, opt): """define a model stacked LSTM layers n_layers: number of LSTM layers in the architecture units: number of neurons in each LSTM layer lr_rate: learning rate for the optimizer drop_rate: seed: opt: """ model = Sequential() model.add(InputLayer(input_shape=(input_dim, ))) for _ in range(n_layers): model.add( Dense(units, kernel_initializer=he_uniform(seed=seed), activation='relu')) model.add(Dropout(rate=drop_rate, seed=seed)) model.add(Dense(1, activation='sigmoid')) if opt == 'adam': model.compile(optimizer=Adam(lr=lr_rate), loss='binary_crossentropy', metrics=['binary_accuracy']) if opt == 'sgd': model.compile(optimizer=SGD(lr=lr_rate, nesterov=True), loss='binary_crossentropy', metrics=['binary_accuracy']) model.summary() return model
def build_model(embedding_matrix, nb_words, embedding_size=300): inp = Input(shape=(max_length, )) x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False)(inp) x = SpatialDropout1D(0.35, seed=seed_nb)(x) x1 = Bidirectional( CuDNNLSTM(256, kernel_initializer=glorot_uniform(seed=seed_nb), return_sequences=True))(x) x2 = Bidirectional( CuDNNGRU(128, kernel_initializer=glorot_uniform(seed=seed_nb), return_sequences=True))(x1) max_pool1 = GlobalMaxPooling1D()(x1) max_pool2 = GlobalMaxPooling1D()(x2) conc = Concatenate()([max_pool1, max_pool2]) predictions = Dense(58, activation='softmax', kernel_initializer=he_uniform(seed=seed_nb))(conc) model = Model(inputs=inp, outputs=predictions) adam = optimizers.Adam(lr=learning_rate) model.compile(optimizer=adam, loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model
def convolution(self, x, n_filters, pool=None, name='conv'): # define common parameters params = dict(kernel_initializer=he_uniform(seed=42), use_bias=False, padding='same') # apply correct convolutional operation op = None if pool == 'up': op = Conv3DTranspose(filters=n_filters, kernel_size=3, strides=2, name=name + '_conv', **params) if pool is None: op = Conv3D(filters=n_filters, kernel_size=3, name=name + '_conv', **params) if pool == 'down': op = Conv3D(filters=n_filters, kernel_size=3, strides=2, name=name + '_conv', **params) # normalize and non linearlity x = op(x) x = GroupNormalization(groups=8, name=name + '_norm')(x) x = ReLU(name=name + '_relu')(x) return x
def prepare_model(): data = import_data(ticker, timeframe, start_train_date, end_train_date, calculate_input, lookback, calculate_output, lookforward, split=(100, 0, 0)) # Creating a model... model = Sequential() model.add( Dense(units=num_features * 2, activation='tanh', input_dim=num_features, kernel_initializer=he_uniform(1))) model.add(Dense(num_features * 2, activation='tanh')) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) one_hot_train_outputs = keras.utils.to_categorical( data['train_output'], num_classes=num_classes) # Performing one-hot encoding model.fit(data['train_input'], one_hot_train_outputs, epochs=num_epochs) # Training the model return model
def test_he_uniform(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) _runner(initializers.he_uniform(), tensor_shape, target_mean=0., target_std=std)
def sparseNN(): sparse_data = Input(shape=[x_train.shape[1]], dtype='float32', sparse=True, name='sparse_data') x = Dense(200, kernel_initializer=he_uniform(seed=0))(sparse_data) x = PReLU()(x) x = Dense(200, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = Dense(100, kernel_initializer=he_uniform(seed=0))(x) x = PReLU()(x) x = Dense(1)(x) model = Model([sparse_data], x) optimizer = Adam(.001) model.compile(loss="mse", optimizer=optimizer) return model
def _get_default_initializer(cls, initializer): """ Returns a default initializer of Convolutional Aware or he_uniform for convolutional layers. Parameters ---------- initializer: :class:`keras.initializers.Initializer` or None The initializer that has been passed into the model. If this value is ``None`` then a default initializer will be returned based on the configuration choices, otherwise the given initializer will be returned. Returns ------- :class:`keras.initializers.Initializer` The kernel initializer to use for this convolutional layer. Either the original given initializer, he_uniform or convolutional aware (if selected in config options) """ if initializer is None: retval = ConvolutionAware( ) if _CONFIG["conv_aware_init"] else he_uniform() logger.debug("Set default kernel_initializer: %s", retval) else: retval = initializer logger.debug("Using model supplied initializer: %s", retval) return retval
def create_nn(num_features, num_classes=2, optimizer=None, activation=None, units_multiplier=None, num_hidden_layers=None): if isinstance(num_features, str): return ( True ) # Returns "True" if the model requires one-hot encoding and "False" otherwise] if units_multiplier is None: units_multiplier = 1 if optimizer is None: optimizer = 'adam' if activation is None: activation = 'tanh' if num_hidden_layers is None: num_hidden_layers = 1 m = Sequential() m.add( Dense(units=num_features * units_multiplier, activation=activation, input_dim=num_features, kernel_initializer=he_uniform(1))) for l in range(num_hidden_layers): m.add(Dense(num_features * units_multiplier, activation=activation)) m.add(Dense(num_classes, activation='softmax')) # Compiling the model #m.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) m.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) return m
def _get_default_initializer( initializer: keras.initializers.Initializer ) -> keras.initializers.Initializer: """ Returns a default initializer of Convolutional Aware or he_uniform for convolutional layers. Parameters ---------- initializer: :class:`keras.initializers.Initializer` or None The initializer that has been passed into the model. If this value is ``None`` then a default initializer will be set to 'he_uniform'. If Convolutional Aware initialization has been enabled, then any passed through initializer will be replaced with the Convolutional Aware initializer. Returns ------- :class:`keras.initializers.Initializer` The kernel initializer to use for this convolutional layer. Either the original given initializer, he_uniform or convolutional aware (if selected in config options) """ if _CONFIG["conv_aware_init"]: retval = ConvolutionAware() elif initializer is None: retval = he_uniform() else: retval = initializer logger.debug("Using model supplied initializer: %s", retval) logger.debug("Set default kernel_initializer: (original: %s current: %s)", initializer, retval) return retval
def create_model(): momentum = 0.9 lr = 0.001 decay = 0.0005 reg = regularizers.l2(decay) kernel_init = initializers.he_uniform() model = Sequential() model.add( E2E_conv(2, 4, (2, 70), kernel_regularizer=reg, input_shape=(70, 70, 1), data_format="channels_last")) model.add(LeakyReLU(alpha=0.33)) # model.add(E2E_conv(2,32,(2,90),kernel_regularizer=reg,input_shape=(90,90,1),data_format="channels_last")) # model.add(LeakyReLU(alpha=0.33)) # model.add(E2E_conv(2,64,(2,90),kernel_regularizer=reg,input_shape=(90,90,1),data_format="channels_last")) # model.add(LeakyReLU(alpha=0.33)) model.add( Convolution2D(4, (1, 70), kernel_regularizer=reg, data_format="channels_last")) model.add(LeakyReLU(alpha=0.33)) model.add(Flatten()) model.add(Dropout(0.8)) model.add( Dense(2, activation="softmax", kernel_regularizer=reg, kernel_initializer=kernel_init)) model.add(Flatten()) model.compile(optimizer=SGD(nesterov=True, lr=lr), loss='categorical_crossentropy', metrics=['accuracy']) return model
def build_fasttext(self): """ 可用的initialization方法:random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform() :return: """ initial_dict = { 'orthogonal': orthogonal(), 'he_n': he_normal(), 'he_u': he_uniform() } # model.add(Embedding(output_dim=self.hidden_layer, input_length=self.max_sequence_length, # input_dim=self.max_nb_words, embeddings_initializer=initial_dict['he_n'], trainable=True)) # model.add(GlobalAveragePooling1D()) # model.add(Dense(self.n_classes, activation='softmax', kernel_initializer=initial_dict['he_n'])) self.model.add( Embedding(output_dim=self.hidden_layer, input_length=self.max_sequence_length, input_dim=self.max_nb_words, trainable=True)) self.model.add(GlobalAveragePooling1D()) self.model.add(Dense(self.n_classes, activation='softmax')) print(self.model.summary()) self.compile()
def cross_autoencoder(train, train_noisy, test, encode, decode, keepprob, slr, batch_size, tuning=True, best_epoch=None): from keras.layers import Input, Dense, Dropout, BatchNormalization from keras.layers.advanced_activations import PReLU from sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error from keras.optimizers import Nadam, Adam,SGD,Adagrad,Adadelta,RMSprop from keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping from keras.models import Model from keras import backend as K from keras import metrics, regularizers, initializers import gc # NOTE: Requires latest Keras version 2.2.4 (To extract best model at EarlyStopping) print("Encoding layers") # Start SEED = 1234 input_shape = Input(shape=(train.shape[1],)) print(encode) e = encode[0] # In case single layer vars()["encoded_%s"%e] = Dense(encode[0], activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(input_shape) vars()["e_batch_%s"%e] = BatchNormalization()(vars()["encoded_%s"%e]) vars()["e_drop_%s"%e] = Dropout(keepprob)(vars()["e_batch_%s"%e]) # Encoding layers prev_layer = encode[0] print("prev_layer: ", prev_layer) for e in encode[1:]: print(e) vars()["encoded_%s"%e] = Dense(e, activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(vars()["e_drop_%s"%prev_layer]) vars()["e_batch_%s"%e] = BatchNormalization()(vars()["encoded_%s"%e]) vars()["e_drop_%s"%e] = Dropout(keepprob)(vars()["e_batch_%s"%e]) prev_layer = e # Decoding layers vars()["d_drop_%s"%prev_layer] = vars()["e_drop_%s"%e] d = prev_layer # In case single layer for d in decode[:-1]: print(d, vars()["d_drop_%s"%prev_layer]) vars()["decoded_{}".format(d)] = Dense(d, activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(vars()["d_drop_{}".format(prev_layer)]) vars()["d_batch_%s"%d] = BatchNormalization()(vars()["decoded_%s"%d]) vars()["d_drop_%s"%d] = Dropout(keepprob)(vars()["d_batch_%s"%d]) prev_layer = d vars()["last"] = Dense(decode[-1], activation="sigmoid")(vars()["d_drop_%s"%d]) # Map the autoencoder autoencoder = Model(input_shape, vars()["last"]) # Create encoding layer encoder = Model(input_shape, vars()["encoded_%s"%e]) # Optimizer optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999) # Compile autoencoder.compile(optimizer=optimizer, loss="mean_squared_error", metrics=["mse"]) # Run model depending on need if tuning==True: early_stopping = EarlyStopping(monitor='val_loss', patience=100, restore_best_weights=True) print("Hyperparameter tunning....") print(autoencoder.summary()) results = autoencoder.fit(x=train_noisy, y=train, validation_split=0.2, #validation_data=(test, test), batch_size=batch_size, epochs=2000, verbose=2, callbacks=[early_stopping]) best_metric = np.min(results.history["val_loss"]) train_loss = results.history["loss"][results.history["val_loss"].index(best_metric)] best_epoch = np.argmin(results.history["val_loss"]) K.clear_session() return best_epoch, best_metric, train_loss else: print("Best Epoch found at {}".format(best_epoch)) print("Building model....") results = autoencoder.fit(x=train_noisy, y=train, epochs=best_epoch+1, batch_size=batch_size, verbose=2) # Obtain prediction train_output = encoder.predict(train) test_output = encoder.predict(test) K.clear_session() return train_output, test_output
def dae(cell_train, cell_test, gsea, arch, keepprob, slr, batch_size, init_decomp, self_valid=True): from keras.layers import Input, Dense, Dropout, BatchNormalization, concatenate, Lambda from keras.layers.advanced_activations import PReLU from sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error from keras.optimizers import Nadam, Adam,SGD,Adagrad,Adadelta,RMSprop from keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping from keras.models import Model from tensorflow import gather from keras import backend as K from keras import metrics, regularizers, initializers import gc # Trains autoencoder using tensorflow # Prepare data g_sets = list(set(gsea.Gene_set)) all_genes = list(set(gsea.genes)) exp_cols = list(cell_train.columns) index_list = [] for i in g_sets: genes = list(gsea.query("Gene_set==@i").genes) index = [exp_cols.index(i) for i in genes] index_list.append(index) train_feat = np.asarray(cell_train) #[samples, genes] test_feat = np.asarray(cell_test) log_table = [] # Standardize initial inputs to 0-1 range # train_feat, test_feat = scale_0_1_multiple(train_feat, test_feat) train_feat, test_feat = scale_standard_multiple(train_feat, test_feat) print("Number of gene sets used for parallel encoding: {}".format(len(g_sets))) # Specify architecture layers = arch_layers(len(index_list)*init_decomp, arch) rev_layers = [i for i in reversed(layers)] print("layers: ", layers) print("rev_layers:", rev_layers) # BUILD KERAS MODEL - API SEED = 1234 recon_feat = len(all_genes) input_shape = Input(shape=(train_feat.shape[1],)) # Split data based on index list for il in xrange(len(index_list)): vars()["x_%s"%il] = Lambda(lambda x: gather(x, index_list[il], axis=1 ))(input_shape) # Learn each individually for il in xrange(len(index_list)): vars()["x_%s_prel"%il] = Dense(init_decomp, activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(vars()["x_%s"%il]) # Merge for first layer encoded_merge = concatenate([vars()["x_%s_prel"%il] for il in xrange(len(index_list))]) encoded_bn = BatchNormalization()(encoded_merge) vars()["encode_%s_drop"%(0)] = Dropout(keepprob)(encoded_bn) # Add additional encoded layers [200,100] for l in xrange(len(layers)-1): vars()["encode_%s_prel"%(l+1)] = Dense(layers[(l+1)], activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(vars()["encode_%s_drop"%l]) vars()["encode_%s_bn"%(l+1)] = BatchNormalization()(vars()["encode_%s_prel"%(l+1)]) vars()["encode_%s_drop"%(l+1)] = Dropout(keepprob)(vars()["encode_%s_bn"%(l+1)]) # DECODING [100,200] # Decoding first layer vars()["decode_0_prel"] = Dense(rev_layers[1], activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(vars()["encode_%s_drop"%(l+1)]) vars()["decode_0_bn"] = BatchNormalization()(vars()["decode_0_prel"]) vars()["decode_0_drop"] = Dropout(keepprob)(vars()["decode_0_bn"]) # Decoding additional layers d=-1 for d in xrange(len(rev_layers)-2): vars()["decode_%s_prel"%(d+1)] = Dense(rev_layers[(d+2)], activation=PReLU(), kernel_initializer = initializers.he_uniform(seed=SEED))(vars()["decode_%s_drop"%d]) vars()["decode_%s_bn"%(d+1)] = BatchNormalization()(vars()["decode_%s_prel"%(d+1)]) vars()["decode_%s_drop"%(d+1)] = Dropout(keepprob)(vars()["decode_%s_bn"%(d+1)]) # Reconstruct vars()["last"] = Dense(recon_feat, activation="sigmoid")(vars()["decode_%s_drop"%(d+1)]) # Map the autoencoder autoencoder = Model(input_shape, vars()["last"]) # Create encoding layer (Latent space) encoder = Model(input_shape, vars()["encode_%s_prel"%(l+1)]) # Optimizer optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999) # Compile autoencoder.compile(optimizer=optimizer, loss="mean_squared_error", metrics=["mse"]) # Run model early_stopping = EarlyStopping(monitor='val_loss', patience=15, restore_best_weights=True) print(autoencoder.summary()) # How are we approaching validation - On a split of train set or validating on external set if self_valid==True: results = autoencoder.fit(x=train_feat, y=train_feat, validation_split=0.4, #validation_data=(test_feat, test_feat), batch_size=batch_size, verbose=2, epochs=500, callbacks=[early_stopping]) else: results = autoencoder.fit(x=train_feat, y=train_feat, validation_data=(test_feat, test_feat), batch_size=batch_size, verbose=2, epochs=500, callbacks=[early_stopping]) # Obtain prediction train_output = encoder.predict(train_feat) test_output = encoder.predict(test_feat) # Clean up K.clear_session() del autoencoder gc.collect() print("Done modeling GSEA-DAE") # Return return train_output, test_output, results
def test_he_uniform(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) _runner(initializers.he_uniform(), tensor_shape, target_mean=0., target_std=std)
def update_kwargs(kwargs): """ Set the default kernel initializer to he_uniform() """ kwargs["kernel_initializer"] = kwargs.get("kernel_initializer", he_uniform()) return kwargs
def test_he_uniform(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) scale = np.sqrt(6. / fan_in) _runner(initializers.he_uniform(), tensor_shape, target_mean=0., target_max=scale, target_min=-scale)
def keras_mlp(train_feat, train_target, test_feat, test_target, batch_size, slr, keepprob, arch, l2_reg, patience=15, v_split=0.1, tuning=True, best_epoch=None): import random, math, gc from sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error # Deep Learning Libraries import tensorflow as tf from keras.models import Sequential, load_model from keras.layers import Dense, Dropout, Flatten from keras.layers import Conv2D, MaxPooling2D, BatchNormalization from keras.layers.advanced_activations import PReLU from keras.optimizers import Adam,Nadam,SGD,Adagrad,Adadelta,RMSprop from keras.preprocessing.image import ImageDataGenerator from keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping from keras.utils import to_categorical from keras import backend as K from keras import metrics, regularizers, initializers # NOTE: Requires latest Keras version 2.2.4 (To extract best model at EarlyStopping) # Prep data train_feat, test_feat = np.asarray(train_feat), np.asarray(test_feat) train_target = np.asarray(train_target.value) # Model as regression # Standardize initial inputs to 0-1 range # train_feat, test_feat = scale_0_1_multiple(train_feat, test_feat) # train_feat, test_feat = scale_standard_multiple(train_feat, test_feat) # Architecture init_features = train_feat.shape[1] print("init arch: ", init_features, arch) layers = arch_layers(init_features, arch) print(layers) # Build KERAS Fully connected network SEED = 1234 model = Sequential() model.add(Dense(layers[1], input_dim=layers[0], kernel_regularizer=regularizers.l2(l2_reg), kernel_initializer = initializers.he_uniform(seed=SEED))) model.add(PReLU()) model.add(BatchNormalization()) model.add(Dropout(keepprob)) for l in layers[2:]: model.add(Dense(l, kernel_regularizer=regularizers.l2(l2_reg), kernel_initializer = initializers.he_uniform(seed=SEED))) model.add(PReLU()) model.add(BatchNormalization()) model.add(Dropout(keepprob)) # Add the output layer and compile model.add(Dense(1, kernel_initializer = initializers.he_uniform(seed=SEED))) # Optimizer optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999) # optimizer = Adam(lr=slr, beta_1=0.9, beta_2=0.999) # Compile model.compile(optimizer=optimizer, loss="mse", metrics=["mse"]) print(K.tensorflow_backend._get_available_gpus()) sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) # Running model depending on need if tuning==True: early_stopping = EarlyStopping(monitor='val_mean_squared_error', patience=patience, restore_best_weights=True) print("Hyperparameter tunning....") print(model.summary()) results = model.fit(x=train_feat, y=train_target, batch_size=batch_size, epochs=4000, validation_split=v_split, verbose=2, shuffle=True, callbacks=[early_stopping]) # Get stats best_metric = np.min(results.history["val_mean_squared_error"]) train_loss = results.history["mean_squared_error"][results.history["val_mean_squared_error"].index(best_metric)] best_epoch = np.argmin(results.history["val_mean_squared_error"]) log_table = pd.DataFrame([[batch_size, keepprob, train_loss, best_metric]], columns=["bs","keep_prob", "train_error", "test_error"]) print("Best Epoch found at {}".format(best_epoch)) # Clean up K.clear_session() del model del results gc.collect() return best_epoch, log_table else: print("Using Best Epoch found at {}".format(best_epoch)) print("Building model....") print(model.summary()) results = model.fit(x=train_feat, y=train_target, batch_size=batch_size, epochs=best_epoch+1, verbose=2, shuffle=True) # Predict predictions = model.predict(test_feat) test_target = test_target.assign(prediction=predictions) # Clean up K.clear_session() del model del results gc.collect() return test_target