def ResNet(input_shape, output_shape, optimizer='adam', leak=0.0): """Summary Args: input_shape (TYPE): Description output_shape (TYPE): Description optimizer (str, optional): Description leak (float, optional): Description Returns: TYPE: Description """ inputs = Input(shape=input_shape) x = Conv1D(64, 3, border_mode='same')(inputs) x = BatchNormalization()(x) x = Activation('relu')(x) #x = MaxPooling1D(pool_length=2, border_mode='same')(x) x = res_block_1d(x, [64, 64, 256], leak=leak, shortcut=True) x = res_block_1d(x, [64, 64, 256], leak=leak, shortcut=False) x = res_block_1d(x, [64, 64, 256], leak=leak, shortcut=False) x = res_block_1d(x, [128, 128, 512], leak=leak, shortcut=True) x = res_block_1d(x, [128, 128, 512], leak=leak, shortcut=False) x = res_block_1d(x, [128, 128, 512], leak=leak, shortcut=False) x = res_block_1d(x, [128, 128, 512], leak=leak, shortcut=False) x = res_block_1d(x, [256, 256, 1024], leak=leak, shortcut=True) x = res_block_1d(x, [256, 256, 1024], leak=leak, shortcut=False) x = res_block_1d(x, [256, 256, 1024], leak=leak, shortcut=False) x = res_block_1d(x, [256, 256, 1024], leak=leak, shortcut=False) # x = res_block_1d(x, [256, 256, 1024], leak=leak, shortcut=False) # x = res_block_1d(x, [256, 256, 1024], leak=leak, shortcut=False) # x = res_block_1d(x, [512, 512, 2048], leak=leak, shortcut=True) # x = res_block_1d(x, [512, 512, 2048], leak=leak, shortcut=False) # x = res_block_1d(x, [512, 512, 2048], leak=leak, shortcut=False) x = AveragePooling1D(pool_length=4)(x) x = Flatten()(x) outputs = Dense(output_shape, activation='softmax')(x) model = Model(inputs, outputs) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) return model
def conv_filt_v2(input_a, no_filters, filt_width): ''' base module for text data Inputs: input_a -- input tensor no_filters -- number of filter maps used in conv layer (details) filt_width -- convolution window width outputs: output tensor of base module ''' # 20000 is the vocabulary size. It should be greater than or equal to your tokenizer vocabulary size embed = Embedding(vocab_size, embed_size)(input_a) conv = Conv1D(no_filters, filt_width, strides=1)(embed) conv = Activation('relu')(conv) conv = AveragePooling1D(pool_size=2)(conv) conv = Dropout(0.5)(conv) conv = GlobalAveragePooling1D()(conv) return conv
def build_model(): data_shape = (300, 5) # (batch, channels, steps) def square(x): return K.square(x) def log(x): return K.log(K.clip(x, min_value=1e-7, max_value=10000)) model = Sequential() model.add(Conv1D(40, 15, data_format='channels_last', input_shape=data_shape)) model.add(Conv1D(40, 5, use_bias=False, data_format='channels_first')) model.add(BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)) model.add(Activation(square)) model.add(AveragePooling1D(pool_size=40, strides=15)) model.add(Activation(log)) model.add(LSTM(64, activation='sigmoid', recurrent_activation='hard_sigmoid', \ use_bias=True, kernel_initializer='glorot_uniform', \ recurrent_initializer='orthogonal', \ unit_forget_bias=True, kernel_regularizer=None, \ recurrent_regularizer=None, \ bias_regularizer=None, activity_regularizer=None, \ kernel_constraint=None, recurrent_constraint=None, \ bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, \ implementation=1, return_sequences=True, return_state=False, \ go_backwards=True, stateful=False, unroll=False)) model.add(Dropout(0.2)) model.add(LSTM(32, activation='sigmoid', recurrent_activation='hard_sigmoid', \ use_bias=True, kernel_initializer='glorot_uniform', \ recurrent_initializer='orthogonal', \ unit_forget_bias=True, kernel_regularizer=None, \ recurrent_regularizer=None, \ bias_regularizer=None, activity_regularizer=None, \ kernel_constraint=None, recurrent_constraint=None, \ bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, \ implementation=1, return_sequences=True, return_state=False, \ go_backwards=True, stateful=False, unroll=False)) model.add(Dropout(0.5)) model.add(Flatten()) model.add(Dense(3, activation='softmax', kernel_constraint=max_norm(0.5))) start = time.time() model.compile(loss="categorical_crossentropy", optimizer=optimizers.Adagrad(), metrics=['accuracy']) print("Compilation time: ", time.time(), '-', start) return model
def structureSubModel(ssInput): ssConv = Conv1D(filters=256, kernel_size=12, padding="valid", activation="relu", strides=1)(ssInput) ssPool = AveragePooling1D(pool_size=5, strides=5)(ssConv) ssDout1 = Dropout(rate=0.7)(ssPool) seqBiLstm = Bidirectional(LSTM(units=128, return_sequences=True))(ssDout1) seqDout2 = Dropout(rate=0.7)(seqBiLstm) ssFlat = Flatten()(seqDout2) ssDen1 = Dense(256, kernel_initializer='glorot_uniform', activation='relu')(ssFlat) ssDout2 = Dropout(rate=0.7)(ssDen1) return ssDout2
def attentionWrap(self, hid_dim, input_length, input_dim): inputs = Input(shape=(input_length, input_dim)) lstm = GRU(hid_dim, return_sequences=True, dropout=0.2, recurrent_dropout=0.2, input_shape=(input_length, input_dim))(inputs) att = TimeDistributed(Dense(1, activation='tanh'))(lstm) att = Flatten()(att) att = Activation(activation='softmax')(att) att = RepeatVector(hid_dim)(att) att = Permute((2, 1))(att) mer = multiply([lstm, att]) hid = AveragePooling1D(pool_size=input_length)(mer) hid = Flatten()(hid) return inputs, hid
def build_model(self): main_input = Input(shape=(None, ), dtype='int32', name='main_input') x = Embedding(output_dim=self.embedding_dimension, input_dim=n_vocab)(main_input) x = Convolution1D(64, 5, padding='same', activation='relu')(x) if self.dropout_parameter > 0.0: x = Dropout(self.dropout_parameter)(x) if self.rnn_type is 'GRU': rnn = GRU(self.rnn_units, return_sequences=True) elif self.rnn_type is 'LSTM': rnn = LSTM(self.rnn_units, return_sequences=True) else: rnn = SimpleRNN(self.rnn_units) if self.bidirectional: x = Bidirectional(rnn)(x) else: x = rnn(x) if self.maxPooling: x = MaxPooling1D(strides=1, padding='same')(x) print("Using MaxPooling") elif self.averagePooling: x = AveragePooling1D(strides=1, padding='same')(x) print("Using AveragePooling") slot_output = TimeDistributed(Dense(n_slots, activation='softmax'), name='slot_output')(x) intent_output = TimeDistributed(Dense(n_classes, activation='softmax'), name='intent_output')(x) model = kerasModel(inputs=[main_input], outputs=[intent_output, slot_output]) # rmsprop is recommended for RNNs https://stats.stackexchange.com/questions/315743/rmsprop-and-adam-vs-sgd model.compile(optimizer='rmsprop', loss={ 'intent_output': 'categorical_crossentropy', 'slot_output': 'categorical_crossentropy' }) plot_model(model, 'models/' + self.name + '.png') self.model = model return
def conv1d_nn(input_dim=4, window=3): conv1d = Sequential() # conv1d.add(Dense(8, # input_shape=(3, 1))) # conv1d.add(ZeroPadding1D(padding=1)) # above commented, no padding --> pool_size=1 conv1d.add( Conv1D(filters=input_dim, input_shape=(window, 1), kernel_size=window, strides=1, use_bias=True, padding='same')) conv1d.add(AveragePooling1D(pool_size=window, strides=1)) conv1d.add(Flatten()) conv1d.add(Dense(1)) conv1d.compile(loss='mean_squared_error', optimizer='adam') return conv1d
def structureModel(InputShape): conv1 = Conv1D(filters=256, kernel_size=12, padding='valid', activation='relu', strides=1)(InputShape) mapool = AveragePooling1D(pool_size=5, strides=5)(conv1) dout1 = Dropout(rate=0.2)(mapool) structBiLstm = Bidirectional(LSTM(units=128, return_sequences=True))(dout1) seqDout2 = Dropout(rate=0.4)(structBiLstm) flat = Flatten()(seqDout2) den1 = Dense(256, kernel_initializer='glorot_uniform')(flat) activa2 = PReLU(alpha_initializer='zero', weights=None)(den1) dout2 = Dropout(rate=0.5)(activa2) return dout2
def structureModel(structInput): structCov = Conv1D(filters = 16, kernel_size = 12, padding = 'valid', activation = 'relu', strides = 1)(structInput) structPool = AveragePooling1D(pool_size = 20, strides = 10)(structCov) structPoolDout = Dropout(rate=0.2)(structPool) structBiLstm = Bidirectional(LSTM(units = 8, return_sequences = True))(structPoolDout) structBiLstmDout = Dropout(rate = 0.5)(structBiLstm) structFlat = Flatten()(structBiLstmDout) structDen1 = Dense(2, kernel_initializer='glorot_uniform')(structFlat) structActivaDen1 = PReLU(alpha_initializer = 'zero', weights= None)(structDen1) structDout1 = Dropout(rate=0.9)(structActivaDen1) return structDout1
def resnet_block(input_tensor, final_layer_output=220, append='n'): x = Conv1D(64, 7, strides=2, padding='same', name='conv1' + append)(input_tensor) x = BatchNormalization(name='bn_conv1' + append)(x) x = Activation('relu')(x) x = MaxPooling1D(3, strides=2)(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a' + append, strides=1) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b' + append) x = identity_block(x, 3, [64, 64, 256], stage=2, block='c' + append) x = conv_block(x, 3, [128, 128, 512], stage=3, block='a' + append) x = identity_block(x, 3, [128, 128, 512], stage=3, block='b' + append) x = identity_block(x, 3, [128, 128, 512], stage=3, block='c' + append) x = identity_block(x, 3, [128, 128, 512], stage=3, block='d' + append) x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='g' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='h' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='i' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='j' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='k' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='l' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='m' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='n' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='o' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='p' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='q' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='r' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='s' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='t' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='u' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='v' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='w' + append) x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a' + append) x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b' + append) x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c' + append) x = AveragePooling1D(final_layer_output, name='avg_pool' + append)(x) x = Flatten()(x) return x
def ResNet18_1d(input_tensor, final_layer_output=220, append='n'): x = Conv1D(64, 7, strides=2, padding='same', name='conv1' + append)(input_tensor) x = BatchNormalization(name='bn_conv1' + append)(x) x = Activation('relu')(x) x = MaxPooling1D(3, strides=2)(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a' + append, strides=1) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b' + append) x = identity_block(x, 3, [64, 64, 256], stage=2, block='c' + append) x = conv_block(x, 3, [128, 128, 512], stage=3, block='a' + append) x = identity_block(x, 3, [128, 128, 512], stage=3, block='b' + append) x = identity_block(x, 3, [128, 128, 512], stage=3, block='c' + append) x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b' + append) x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c' + append) x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a' + append) x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b' + append) x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c' + append) x = AveragePooling1D(final_layer_output, name='avg_pool' + append)(x) x = Flatten()(x) return x
def conv_filt(input_dim, no_filters, filt_sizes): ''' returns a list of parallel convolutonal layers. We will concatenate them in concat_conv_filt() ''' filt_models = [] print(no_filters, filt_sizes) repeat_threshold = int(len(no_filters)/2) for filt_ind, filt in enumerate(no_filters): model = Sequential() # Anything more than vocabulary size to be considered is fine. I used 20000 here model.add(Embedding(20000, 300, input_length=global_vars.MaxLen)) model.add(Conv1D(no_filters[filt_ind], filt_sizes[filt_ind], strides=1)) model.add(Activation('relu')) if filt_ind < repeat_threshold: pool_sizes = 2 else: pool_sizes = 7 print(pool_sizes) model.add(AveragePooling1D(pool_size=pool_sizes)) model.add(Dropout(0.5)) model.add(GlobalAveragePooling1D()) filt_models.append(model) return filt_models
def get(input_shape, num_classes, batchnorm=True, activation='softmax'): input_layer = Input(shape=input_shape) layer = input_layer for i in range(3): layer = Conv1D(filters=256, kernel_size=4, strides=2, padding='same', kernel_initializer='glorot_normal')(layer) layer = Activation('relu')(layer) layer = MaxPooling1D(2)(layer) avg_pool = AveragePooling1D(pool_size=4)(layer) max_pool = MaxPooling1D(pool_size=4)(layer) layer = concatenate([avg_pool, max_pool]) layer = Flatten()(layer) layer = Dense(units=2048, activation='relu')(layer) layer = Dropout(rate=0.5)(layer) layer = Dense(units=num_classes)(layer) layer = Dropout(rate=0.5)(layer) layer = Activation(activation)(layer) return Model(inputs=input_layer, outputs=layer)
def visual_model(self): # when input is visual feature model = Sequential() model.add( BatchNormalization(input_shape=(self.seq_length, 4805), name='visual_BN_1')) model.add(AveragePooling1D(pool_size=2, name='visual_average')) # lstm layer model.add(LSTM(64, name='visual_lstm')) model.add(Activation('relu', name='visual_activation1')) model.add(BatchNormalization(name='visual_BN_2')) model.add(Dropout(0.5, name='visual_dropout_1')) # the hidden layer model = self.add_hidden_layer(model, 'visual') # the decision layer model.add(self.decision_layer('visual')) return model
def __call__(self, window_size, n_channels=4, regression=False, dense=False): input = Input(shape=(window_size, n_channels)) x = Conv1D(64, n_channels, padding='same', activation='relu')(input) for i in range(5): output = Conv1D(64, 3, padding='same', activation='relu')(x) output = Conv1D(64, 3, padding='same')(output) output = Add()([x, output]) output = Activation('relu')(output) x = output output = AveragePooling1D(2)(output) output = Flatten()(output) output_size = window_size if dense else 1 output = Dense(output_size)(output) if not regression: output = Activation('sigmoid')(output) model = Model(inputs=[input], outputs=[output]) return model
def bimodal_audio_visual(self): # audio model audio_feature = Sequential() audio_feature.add( BatchNormalization(input_shape=(self.audio_feature_f_dim, ), name='av_audio_BN_1')) audio_feature = self.add_hidden_layer(audio_feature, 'av_audio') #visual model face_fusion = Sequential() face_fusion.add( BatchNormalization(input_shape=(self.seq_length, self.face_fusion_f_dim), name='av_visual_BN_1')) face_fusion.add(AveragePooling1D(pool_size=2, name='av_visual_average')) face_fusion.add(LSTM(64, name='av_visual_lstm')) face_fusion.add(Activation('relu', name='av_visual_activation1')) face_fusion.add(BatchNormalization(name='av_visual_BN_2')) face_fusion.add(Dropout(0.5, name='av_visual_dropout_1')) face_fusion = self.add_hidden_layer(face_fusion, 'av_visual') audio_feature_input = audio_feature.input audio_feature_output = audio_feature.layers[-1].output face_fusion_input = face_fusion.input face_fusion_output = face_fusion.layers[-1].output concat_layer = concatenate([audio_feature_output, face_fusion_output]) x = Dense(1024)(concat_layer) x = Activation('relu')(x) x = BatchNormalization()(x) x = Dropout(0.5)(x) out = self.decision_layer('bimodal-av')(x) fusion = Model([audio_feature_input, face_fusion_input], out) return fusion
def _pool(self, p=3, strides=1, padding='same', type="max"): """ Function for applying a pooling layer. To call: _pool(p, strides, padding, type) Parameters: p pooling parameter strides padding type 'max' or 'avg' """ # ================================================== # Check to see if a model has already # been initialized. If not, create the # model and add the input # ================================================== try: self.model_ except: self.model_ = [] self.model_.append(Input(shape=self.trainX_.shape[1:])) # ================================================== # Apply the pooling, depending on which # type was chosen (max is default) # ================================================== if type.lower() == 'avg': self.model_.append( AveragePooling1D(pool_size=p, strides=strides, padding=padding)(self.model_[-1])) else: self.model_.append( MaxPooling1D(pool_size=p, strides=strides, padding=padding)(self.model_[-1]))
def bimodal_visual_word(self): #visual model face_fusion = Sequential() face_fusion.add( BatchNormalization(input_shape=(self.seq_length, self.face_fusion_f_dim), name='vw_visual_BN_1')) face_fusion.add(AveragePooling1D(pool_size=2, name='vw_visual_average')) face_fusion.add(LSTM(64, name='vw_visual_lstm')) face_fusion.add(Activation('relu', name='vw_visual_activation1')) face_fusion.add(BatchNormalization(name='vw_visual_BN_2')) face_fusion.add(Dropout(0.5, name='vw_visual_dropout_1')) face_fusion = self.add_hidden_layer(face_fusion, 'vw_visual') #visual model word_fusion = Sequential() word_fusion.add( BatchNormalization(input_shape=(self.word_fusion_f_dim, ), name='vw_word_BN_1')) # add the hidden layer word_fusion = self.add_hidden_layer(word_fusion, 'vw_word') face_fusion_input = face_fusion.input face_fusion_output = face_fusion.layers[-1].output word_input = word_fusion.input word_output = word_fusion.layers[-1].output concat_layer = concatenate([face_fusion_output, word_output]) x = Dense(1024)(concat_layer) x = Activation('relu')(x) x = BatchNormalization()(x) x = Dropout(0.5)(x) out = self.decision_layer('bimodal-vw')(x) fusion = Model([face_fusion_input, word_input], out) return fusion
def body_fusion(self): # when input is visual feature rand = self.get_random() model = Sequential() model.add( BatchNormalization(input_shape=(self.seq_length, self.body_fusion_f_dim), name='body_fusion_BN_1' + rand)) model.add( AveragePooling1D(pool_size=2, name='body_fusion_average' + rand)) # lstm layer model.add(LSTM(64, name='body_fusion_lstm' + rand)) model.add(Activation('relu', name='body_fusion_activation1' + rand)) model.add(BatchNormalization(name='body_fusion_BN_2' + rand)) model.add(Dropout(0.5, name='body_fusion_dropout_1' + rand)) # the hidden layer model = self.add_hidden_layer(model, 'body_fusion_hidden' + rand) # the decision layer model.add(self.decision_layer('body_fusion' + rand)) return model
def train_model(): # 定义conv1D神经网络层结构 input_layer = Input(shape=(time_steps, 1), dtype='float32') zeropadding_layer = ZeroPadding1D(padding=1)(input_layer) conv1D_layer1 = Conv1D(64, 3, strides=1, use_bias=True)(zeropadding_layer) avgpooling_layer = AveragePooling1D(pool_size=3, strides=1)(conv1D_layer1) flatten_layer = Flatten()(avgpooling_layer) dropout_layer = Dropout(.45)(flatten_layer) output_layer = Dense(1, activation='tanh')(dropout_layer) ts_model = Model(inputs=input_layer, outputs=output_layer) # 编译模型 ts_model.compile(loss='mean_absolute_error', optimizer='adam') # ts_model.summary() # 输出模型层结构 # 存储损失函数最小值时的模型为hdf文件 save_best = ModelCheckpoint( '%s_conv1D_weights.{epoch:02d}-{val_loss:.4f}.h5' % stock_code, monitor='val_loss', verbose=2, save_best_only=True, save_weights_only=False, mode='min', period=1) # 开始训练模型(拟合) ts_model.fit(x=X_train, y=y_train, batch_size=16, epochs=45, verbose=2, callbacks=[save_best], validation_data=(X_val, y_val), shuffle=True)
def bimodal_model_audio_visual(self): # audio model audio_model = Sequential() audio_model.add( BatchNormalization(input_shape=(1582, ), name='av_audio_BN_1')) audio_model = self.add_hidden_layer(audio_model, 'av_audio') #visual model visual_model = Sequential() visual_model.add( BatchNormalization(input_shape=(self.seq_length, 4805), name='av_visual_BN_1')) visual_model.add( AveragePooling1D(pool_size=2, name='av_visual_average')) visual_model.add(LSTM(64, name='av_visual_lstm')) visual_model.add(Activation('relu', name='av_visual_activation1')) visual_model.add(BatchNormalization(name='av_visual_BN_2')) visual_model.add(Dropout(0.5, name='av_visual_dropout_1')) visual_model = self.add_hidden_layer(visual_model, 'av_visual') audio_input = audio_model.input audio_output = audio_model.layers[-1].output visual_input = visual_model.input visual_output = visual_model.layers[-1].output concat_layer = concatenate([audio_output, visual_output]) x = Dense(1024)(concat_layer) x = Activation('relu')(x) x = BatchNormalization()(x) x = Dropout(0.5)(x) out = self.decision_layer('bimodal-av')(x) fusion_model = Model([audio_input, visual_input], out) return fusion_model
def conv_filt(input_dim, no_filters, filt_sizes): filt_models = [] strides_list = [1] * len(no_filters) print(no_filters, filt_sizes, strides_list) repeat_threshold = int(len(no_filters) / 2) for filt_ind, filt in enumerate(no_filters): model = Sequential() model.add(Embedding(20000, 300, input_length=MaxLen)) model.add( Conv1D(no_filters[filt_ind], filt_sizes[filt_ind], strides=strides_list[filt_ind])) model.add(Activation('relu')) if filt_ind < repeat_threshold: pool_sizes = 2 else: pool_sizes = 7 print(pool_sizes) model.add(AveragePooling1D(pool_size=pool_sizes)) model.add(Dropout(0.5)) model.add(GlobalAveragePooling1D()) filt_models.append(model) return filt_models
def __init__(self, input_shape, classes=7): x_input = Input(input_shape) x = Reshape((600, 2), input_shape=(input_shape,))(x_input) x = Conv1D(64, 7, strides=2, padding='same', name='conv1')(x) x = BatchNormalization(name='bn_conv1')(x) x = Activation('relu')(x) x = MaxPooling1D(3, strides=2)(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=1) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') # x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') # x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') # x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') # x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') # x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') # x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') # x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') # x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') # x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') x = AveragePooling1D(classes, name='avg_pool')(x) x = Flatten()(x) x = Dense(classes, activation='softmax', name='fc'+str(classes))(x) self.model = Model(inputs=x_input, outputs=x, name='ResNet1D') self.model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) self.history = None
#Add zero padding zeropadding_layer = ZeroPadding1D(padding=1)(input_layer) # The first argument of Conv1D is the number of filters, which determine the number of features in the output. Second argument indicates length of the 1D convolution window. The third argument is strides and represent the number of places to shift the convolution window. Lastly, setting use_bias as True, add a bias value during computation of an output feature. Here, the 1D convolution can be thought of as generating local AR models over rolling window of three time units. # In[23]: #Add 1D convolution layer conv1D_layer = Conv1D(64, 3, strides=1, use_bias=True)(zeropadding_layer) # AveragePooling1D is added next to downsample the input by taking average over pool size of three with stride of one timesteps. The average pooling in this case can be thought of as taking moving averages over a rolling window of three time units. We have used average pooling instead of max pooling to generate the moving averages. # In[24]: #Add AveragePooling1D layer avgpooling_layer = AveragePooling1D(pool_size=3, strides=1)(conv1D_layer) # The preceeding pooling layer returns 3D output. Hence before passing to the output layer, a Flatten layer is added. The Flatten layer reshapes the input to (number of samples, number of timesteps*number of features per timestep), which is then fed to the output layer # In[25]: #Add Flatten layer flatten_layer = Flatten()(avgpooling_layer) # In[26]: dropout_layer = Dropout(0.2)(flatten_layer) # In[27]: #Finally the output layer gives prediction for the next day's air pressure.
#x_shuffled = x_train[shuffle_indices] #y_shuffled = score_train[shuffle_indices] embed_1 = Embedding(input_dim=len(word_indices) + 1, output_dim=EMBEDDING_DIM, weights=[word_embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=True) conv_1 = Conv1D(128, 3, activation='relu', name='conv1') conv_2 = Conv1D(128, 3, activation='relu', name='conv2') conv_3 = Conv1D(256, 3, activation='relu', name='conv3') conv_4 = Conv1D(256, 3, activation='relu', name='conv4') conv_5 = Conv1D(256, 3, activation='relu', name='conv5') conv_6 = Conv1D(1024, 3, activation='relu', name='conv6') conv_7 = Conv1D(1024, 3, activation='relu', name='conv7') conv_8 = Conv1D(1024, 3, activation='relu', name='conv8') pool_1 = AveragePooling1D(pool_length=3, name='pool1') pool_2 = AveragePooling1D(pool_length=3, name='pool2') pool_3 = MaxPooling1D(pool_length=3, name='pool3') pool_4 = MaxPooling1D(pool_length=3, name='pool4') lstm_1 = LSTM(256, name='lstm1', return_sequences=True) lstm_2 = LSTM(128, name='lstm2', return_sequences=True) lstm_3 = LSTM(64, name='lstm3') gru_1 = GRU(256, name='gru1', return_sequences=True) gru_2 = GRU(256, name='gru2', return_sequences=True) gru_3 = GRU(256, name='gru3') bi_lstm_1 = Bidirectional(lstm_1, name='bilstm1') bi_lstm_2 = Bidirectional(lstm_2, name='bilstm2') bi_lstm_3 = Bidirectional(lstm_3, name='bilstm3')
def model(self): """ Implementation of the popular ResNet https://www.kaggle.com/meownoid/tiny-resnet-with-keras-99-314 Returns: model -- a Model() instance in Keras """ from keras.layers.merge import add def block(n_output, upscale=False): # n_output: number of feature maps in the block # upscale: should we use the 1x1 conv2d mapping for shortcut or not # keras functional api: return the function of type # Tensor -> Tensor def f(x): # H_l(x): # first pre-activation h = BatchNormalization()(x) h = Activation('relu')(h) # first convolution h = Conv1D(kernel_size=3, filters=n_output, strides=1, padding='same')(h) # second pre-activation h = BatchNormalization()(h) h = Activation('relu')(h) # second convolution h = Conv1D(kernel_size=3, filters=n_output, strides=1, padding='same')(h) # f(x): if upscale: # 1x1 conv2d f = Conv1D(kernel_size=1, filters=n_output, strides=1, padding='same')(x) else: # identity f = x # F_l(x) = f(x) + H_l(x): return add([f, h]) return f x_input = Input(self.get_input_shape()) # first conv2d with post-activation to transform the input data to some reasonable form x = Conv1D(kernel_size=3, filters=4, strides=1, padding='same')(x_input) x = BatchNormalization()(x) x = Activation('relu')(x) # F_1 x = block(4)(x) # F_2 x = block(4)(x) # F_3 # H_3 is the function from the tensor of size 28x28x16 to the the tensor of size 28x28x32 # and we can't add together tensors of inconsistent sizes, so we use upscale=True x = block(8, upscale=True)( x) # !!! <------- Uncomment for local evaluation # F_4 x = block(8)(x) # !!! <------- Uncomment for local evaluation # F_5 x = block(8)(x) # !!! <------- Uncomment for local evaluation # F_6 x = block(16, upscale=True)( x) # !!! <------- Uncomment for local evaluation # F_7 x = block(16)(x) # !!! <------- Uncomment for local evaluation # last activation of the entire network's output x = BatchNormalization()(x) x = Activation('relu')(x) # average pooling across the channels # 28x28x48 -> 1x48 x = AveragePooling1D()(x) # dropout for more robust learning #x = Dropout(0.2)(x) # output layer x = Flatten()(x) x = Dense(len(self.targetname), activation=self.last_layer_activation, name='fc' + str(len(self.targetname)), kernel_initializer=glorot_uniform(seed=0))(x) # Create model model = Model(inputs=x_input, outputs=x, name='ResNet') return model
def model(self): x_input = Input(self.get_input_shape()) # Zero-Padding x = ZeroPadding1D(3)(x_input) # Stage 1 x = Conv1D(4, 7, strides=1, name='conv1', kernel_initializer=glorot_normal(seed=0))(x) x = BatchNormalization(name='bn_conv1')(x) x = Activation('relu')(x) # x = MaxPooling1D(3, strides=2)(x) # Stage 2 x = conv_block(x, kernel_size=3, filters=[4, 4, 16], stage=2, block='a', s=1) x = identity_block(x, 3, [4, 4, 16], stage=2, block='b') x = identity_block(x, 3, [4, 4, 16], stage=2, block='c') # Stage 3 x = conv_block(x, kernel_size=3, filters=[8, 8, 32], stage=3, block='a', s=2) x = identity_block(x, 3, [8, 8, 32], stage=3, block='b') x = identity_block(x, 3, [8, 8, 32], stage=3, block='c') x = identity_block(x, 3, [8, 8, 32], stage=3, block='d') # Stage 4 x = conv_block(x, kernel_size=3, filters=[16, 16, 64], stage=4, block='a', s=2) x = identity_block(x, 3, [16, 16, 64], stage=4, block='b') x = identity_block(x, 3, [16, 16, 64], stage=4, block='c') x = identity_block(x, 3, [16, 16, 64], stage=4, block='d') x = identity_block(x, 3, [16, 16, 64], stage=4, block='e') x = identity_block(x, 3, [16, 16, 64], stage=4, block='f') # Stage 5 x = conv_block(x, kernel_size=3, filters=[32, 32, 128], stage=5, block='a', s=2) x = identity_block(x, 3, [32, 32, 128], stage=5, block='b') x = identity_block(x, 3, [32, 32, 128], stage=5, block='c') # AVGPOOL x = AveragePooling1D(2, name="avg_pool")(x) # Output layer x = Flatten()(x) mu, sigma = GaussianLayer(len(self.targetname), name='main_output')(x) # Additional 'input' for the labels label_layer = Input((len(self.targetname), )) # Create model model = Model(inputs=[x_input, label_layer], outputs=[mu, sigma], name='ResNetDeepEnsemble') # Define the loss function (needs to be defined here because it uses an intermediate layer) # NOTE: do not include loss function when compiling model because of this div_result = Lambda(lambda y: y[0] / y[1])( [K.square(label_layer - mu), sigma]) loss = K.mean(0.5 * tf.log(sigma) + 0.5 * div_result) + 5 # Add loss to model model.add_loss(loss) return model
def model(self): x_input = Input(self.get_input_shape()) # Zero-Padding x = ZeroPadding1D(3)(x_input) # Stage 1 x = Conv1D(4, 7, strides=1, name='conv1', kernel_initializer=glorot_normal(seed=0))(x) x = BatchNormalization(name='bn_conv1')(x) x = Activation('relu')(x) # x = MaxPooling1D(3, strides=2)(x) # Stage 2 x = conv_block(x, kernel_size=3, filters=[4, 4, 16], stage=2, block='a', s=1) x = identity_block(x, 3, [4, 4, 16], stage=2, block='b') x = identity_block(x, 3, [4, 4, 16], stage=2, block='c') # Stage 3 x = conv_block(x, kernel_size=3, filters=[8, 8, 32], stage=3, block='a', s=2) x = identity_block(x, 3, [8, 8, 32], stage=3, block='b') x = identity_block(x, 3, [8, 8, 32], stage=3, block='c') x = identity_block(x, 3, [8, 8, 32], stage=3, block='d') # Stage 4 x = conv_block(x, kernel_size=3, filters=[16, 16, 64], stage=4, block='a', s=2) x = identity_block(x, 3, [16, 16, 64], stage=4, block='b') x = identity_block(x, 3, [16, 16, 64], stage=4, block='c') x = identity_block(x, 3, [16, 16, 64], stage=4, block='d') x = identity_block(x, 3, [16, 16, 64], stage=4, block='e') x = identity_block(x, 3, [16, 16, 64], stage=4, block='f') # Stage 5 x = conv_block(x, kernel_size=3, filters=[32, 32, 128], stage=5, block='a', s=2) x = identity_block(x, 3, [32, 32, 128], stage=5, block='b') x = identity_block(x, 3, [32, 32, 128], stage=5, block='c') # AVGPOOL x = AveragePooling1D(2, name="avg_pool")(x) # Output layer x = Flatten()(x) mu, sigma = GaussianLayer(len(self.targetname), name='main_output')(x) # Create model model = Model(inputs=x_input, outputs=mu, name='ResNetDeepEnsemble') return model, sigma
def model(self): """ Implementation of the popular ResNet the following architecture: CONV1D -> BATCHNORM -> RELU -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3 -> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER modified version retrieved from two sources: https://github.com/priya-dwivedi/Deep-Learning/blob/master/resnet_keras/ https://github.com/viig99/mkscancer/blob/master/medcan_evaluate_next.py Returns: model -- a Model() instance in Keras """ x_input = Input(self.get_input_shape()) # Zero-Padding x = ZeroPadding1D(3)(x_input) # Stage 1 x = Conv1D(4, 7, strides=1, name='conv1', kernel_initializer=glorot_uniform(seed=0))(x) x = BatchNormalization(name='bn_conv1')(x) x = Activation('relu')(x) # x = MaxPooling1D(3, strides=2)(x) # Stage 2 x = conv_block(x, kernel_size=3, filters=[4, 4, 16], stage=2, block='a', s=1) x = identity_block(x, 3, [4, 4, 16], stage=2, block='b') x = identity_block(x, 3, [4, 4, 16], stage=2, block='c') # Stage 3 x = conv_block(x, kernel_size=3, filters=[8, 8, 32], stage=3, block='a', s=2) x = identity_block(x, 3, [8, 8, 32], stage=3, block='b') x = identity_block(x, 3, [8, 8, 32], stage=3, block='c') x = identity_block(x, 3, [8, 8, 32], stage=3, block='d') # Stage 4 x = conv_block(x, kernel_size=3, filters=[16, 16, 64], stage=4, block='a', s=2) x = identity_block(x, 3, [16, 16, 64], stage=4, block='b') x = identity_block(x, 3, [16, 16, 64], stage=4, block='c') # AVGPOOL x = AveragePooling1D(2, name="avg_pool")(x) # Output layer x = Flatten()(x) x = Dense(len(self.targetname), activation=self.last_layer_activation, name='fc' + str(len(self.targetname)), kernel_initializer=glorot_uniform(seed=0))(x) # Create model model = Model(inputs=x_input, outputs=x, name='ResNet') return model
EMBEDDING_DIM, weights=[embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=False) main_input = Input(shape=(MAX_SENTENCE, MAX_SEQUENCE_LENGTH), dtype='float32', name="main_input") sequence_input = TimeDistributed(embedding_layer, name="sequence_input")(main_input) gru = GRU(HIDDEN_SIZE, return_sequences=True, kernel_initializer='glorot_uniform') bi_gru = TimeDistributed(Bidirectional(gru, merge_mode='concat', weights=None), name="bi_gru")(sequence_input) pooled_hidden = TimeDistributed(AveragePooling1D(pool_size=TIMESTAMP_1, strides=None, padding='valid'), name="pooled_hidden")(bi_gru) x = Reshape((TIMESTAMP_2, 2 * HIDDEN_SIZE), name="x")(pooled_hidden) h = Bidirectional(gru, merge_mode='concat', weights=None, name="h")(x) dd = AveragePooling1D(pool_size=TIMESTAMP_2, strides=None, padding='valid', name="dd")(h) d = Reshape((2 * HIDDEN_SIZE, ), name="d")(dd) wc = K.variable(value=0.25, name="wc") ws = K.variable(value=0.25, name="ws") wr = K.variable(value=0.25, name="wr") for j in range(MAX_SENTENCE):