def u_net_32_64(path_weight, iot=True): ''' relevent iot_weight: 'Models/Progressive/U_NET_32-64-IoT.hdf5' - total PRD - 13.3=5.92+7.41 or 0.5 + 0.7 10^-3 for MSE :param path_weight: :return: ''' input_shape = (2000, 1) inputs = Input(shape=input_shape, name='encoder_input_combined') # 1 encoder_32_output, encoder_64_output = Encoder_32_64_iot( True, name='adaptive_encoder')(inputs) Decoder_32_64_skip = decoder_skip(iot) recons_32CG = Decoder_32_64_skip( ZeroPadding1D(padding=(0, 32))(encoder_32_output)) recons_64CG = Decoder_32_64_skip( ZeroPadding1D(padding=(63, 0))(encoder_64_output)) # recons_16CG = Decoder_16_32(Concatenate(axis=1)([np.zeros([1,62,1]), encoder_16_output ])) model_u_unet = Model(inputs, [recons_32CG, recons_64CG], name='U-NET_32_64_iot') if path_weight != "": print("loading compression net weights") model_u_unet.load_weights(path_weight) return model_u_unet
def call(self, inputs, training=False): app_model, model_ecg_classification = self.app_net self.embeddings_flatten_first = [] self.embeddings_flatten_second = [] self.reconstrcuted_signal_full_first = [] self.reconstrcuted_signal_full_second = [] for i in range(4): offset_input = Lambda(lambda y: y[:, i * 2000:(i + 1) * 2000, ])( inputs) offset_embeddings_32, offset_embeddings_64 = self.compression_net.layers[ 1](offset_input) offset_recons_32 = self.compression_net.layers[4]( ZeroPadding1D(padding=(0, 32))(offset_embeddings_32)) self.reconstrcuted_signal_full_first.append(offset_recons_32) offset_recons_64 = self.compression_net.layers[4]( ZeroPadding1D(padding=(63, 0))(offset_embeddings_64)) self.reconstrcuted_signal_full_second.append(offset_recons_64) offset_embeddings_32 = self.policy_predictor_first( offset_embeddings_32) self.embeddings_flatten_first.append(offset_embeddings_32) offset_embeddings_64 = self.policy_predictor_second( offset_embeddings_64) self.embeddings_flatten_second.append(offset_embeddings_64) x_reconstructed_first = Concatenate(axis=1)( self.reconstrcuted_signal_full_first) x_reconstructed_second = Concatenate(axis=1)( self.reconstrcuted_signal_full_second) x_skip = Lambda(lambda y: y[:, 8000:, ])(inputs) out_reconstrcuted_first = Concatenate( axis=1, name='out_recons_first')([x_reconstructed_first, x_skip]) out_reconstrcuted_second = Concatenate( axis=1, name='out_recons_second')([x_reconstructed_second, x_skip]) x_downstream_loss_first = Concatenate(axis=1)( self.embeddings_flatten_first) x_downstream_loss_second = Concatenate(axis=1)( self.embeddings_flatten_second) x_downstream_loss_pred_first = self.dense_critic_1( x_downstream_loss_first) x_downstream_loss_pred_second = self.dense_critic_1( x_downstream_loss_second) # x_reconstructed_loss_pred_first = self.dense_critic_2(x_downstream_loss_first) # x_reconstructed_loss_pred_second = self.dense_critic_2(x_downstream_loss_second) classification_first = model_ecg_classification( out_reconstrcuted_first) classification_second = model_ecg_classification( out_reconstrcuted_second) RR_first = slide_window(app_model, out_reconstrcuted_first) RR_second = slide_window(app_model, out_reconstrcuted_second) return [ out_reconstrcuted_first, out_reconstrcuted_second, classification_first, classification_second, RR_first, RR_second, x_downstream_loss_pred_first, x_downstream_loss_pred_second ]
def build(self, input_shape): self.layers = [ Conv1D(filters=self.units, kernel_size=3, activation='relu'), ZeroPadding1D(padding=(1, 1)), Dropout(0.2), Conv1D(filters=self.units, kernel_size=3), ZeroPadding1D(padding=(1, 1)), ]
def real_values_1d_model(): input_layer = Input((7, )) first_conv_layer = Conv1D(16, kernel_size=3, strides=1) second_conv_layer = Conv1D(8, kernel_size=3, strides=1) x = ExpandInputDim()(input_layer) x = first_conv_layer(ZeroPadding1D(1)(x)) x = Activation('relu')(x) x = second_conv_layer(ZeroPadding1D(1)(x)) x = Activation('relu')(x) x = Conv1D(20, kernel_size=1)(x) x = Flatten()(x) first_dense_layer = Dense(7) second_dense_layer = Dense(1) x = first_dense_layer(x) x = Activation('relu')(x) x = second_dense_layer(x) return Model(input_layer, x)
def __init__(self, padding, conv_channels, kernel_size, dilation_rate, nonlinear_activation, nonlinear_activation_params): super().__init__([ ZeroPadding1D(padding), tfa.layers.WeightNormalization( Conv1D(conv_channels, kernel_size=kernel_size, padding='valid', dilation_rate=dilation_rate, kernel_initializer=tf.keras.initializers.HeNormal())), getattr(tf.keras.layers, nonlinear_activation)(**nonlinear_activation_params) ])
def __init__(self, output_channels: int, input_channels: Optional[int] = None, kernel_size: int = 3, pooling_size: int = 1, batch_normalization: bool = True, dropout_rate: float = 0.0, l2_regularization: float = 0.0): super().__init__() leaky_relu = LeakyReLU(alpha=0.01) dimension_decrease_factor = 4 if batch_normalization: self.batch_normalization = BatchNormalization(scale=False) self.batch_normalization1 = BatchNormalization(scale=False) self.batch_normalization2 = BatchNormalization(scale=False) else: self.batch_normalization = None if l2_regularization > 0: l2_regularizer = L2(l2_regularization) else: l2_regularizer = None self.dimension_decrease_layer = Convolution1D( output_channels // dimension_decrease_factor, kernel_size=1, activation=leaky_relu, kernel_regularizer=l2_regularizer) self.convolutional_layer = Convolution1D( output_channels // dimension_decrease_factor, kernel_size=kernel_size, activation=leaky_relu, padding='same', kernel_regularizer=l2_regularizer) self.dimension_increase_layer = Convolution1D( output_channels, kernel_size=1, activation=leaky_relu, kernel_regularizer=l2_regularizer) if pooling_size > 1: self.pooling_layer = MaxPooling1D(pool_size=pooling_size, padding='same') else: self.pooling_layer = None if input_channels is not None and output_channels != input_channels: if output_channels < input_channels: raise NotImplementedError( f'Residual blocks with less output channels than input channels is not' f'implemented. Output channels was {output_channels} and input was' f'{input_channels}') self.dimension_change_permute0 = Permute((2, 1)) self.dimension_change_layer = ZeroPadding1D( padding=(0, output_channels - input_channels)) self.dimension_change_permute1 = Permute((2, 1)) else: self.dimension_change_layer = None if dropout_rate > 0: self.dropout_layer = SpatialDropout1D(rate=dropout_rate) else: self.dropout_layer = None
def conv_unit(inp, n_gram, no_word=200, window=2): out = Conv1D(no_word, window, strides=1, padding="valid", activation='relu')(inp) out = TimeDistributed(Dense(5, input_shape=(n_gram, no_word)))(out) out = ZeroPadding1D(padding=(0, window - 1))(out) return out
def __call__(self, x): inputs = x in_channels = x.shape[-1] pointwise_conv_filters = int(self.filters * self.alpha) pointwise_filters = _make_divisible(pointwise_conv_filters, 8) prefix = 'block_{}_'.format(self.block_id) if self.block_id: # Expand x = Conv1D(self.expansion * in_channels, kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'expand')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'expand_BN')(x) x = ReLU(6., name=prefix + 'expand_relu')(x) else: prefix = 'expanded_conv_' # Depthwise if self.stride == 2: x = ZeroPadding1D(padding=1, name=prefix + 'pad')(x) x = SeparableConv1D(int(x.shape[-1]), kernel_size=3, strides=self.stride, activation=None, use_bias=False, padding='same' if self.stride == 1 else 'valid', name=prefix + 'depthwise')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x) x = ReLU(6., name=prefix + 'depthwise_relu')(x) # Project x = Conv1D(pointwise_filters, kernel_size=1, padding='same', use_bias=False, activation=None, name=prefix + 'project')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'project_BN')(x) if in_channels == pointwise_filters and self.stride == 1: return Add(name=prefix + 'add')([inputs, x]) return x
def music_sincnet( filter_size=2501, sr=22050, filter_num=256, amplifying_ratio=16, drop_rate=0.5, weight_decay=1e-4, num_classes=50, ): # Input&Reshape inp = Input(shape=(59049, 1)) # x = Reshape([-1, 1])(x) # MusicSinc x = ZeroPadding1D(padding=(filter_size - 1) // 2)(inp) x = MusicSinc1D(filter_num, filter_size, sr)(x) x = BatchNormalization()(x) x = Activation('relu')(x) # Strided Conv num_features = int(filter_num // 2) x = Conv1D( num_features, kernel_size=3, strides=3, padding="valid", use_bias=True, kernel_regularizer=l2(weight_decay), kernel_initializer="he_uniform", )(inp) x = BatchNormalization()(x) x = Activation("relu")(x) # rese-block layer_outputs = [] for i in range(9): num_features *= 2 if (i == 2 or i == 8) else 1 x = rese_block(x, num_features, weight_decay, amplifying_ratio) layer_outputs.append(x) x = Concatenate()( [GlobalMaxPool1D()(output) for output in layer_outputs[-3:]]) x = Dense(x.shape[-1], kernel_initializer="glorot_uniform")(x) x = BatchNormalization()(x) x = Activation("relu")(x) if drop_rate > 0.0: x = Dropout(drop_rate)(x) out = Dense(num_classes, activation="sigmoid", kernel_initializer="glorot_uniform")(x) model = Model(inputs=inp, outputs=out) model.summary() return model
def DenseNet(nb_dense_block, nb_layers, growth_rate, nb_filter, reduction, dropout_rate, classes=12): eps = 1.1e-5 # compute compression factor compression = 1.0 - reduction global concat_axis concat_axis = 2 img_input = Input(shape=(250, 6), name='data') # Initial convolution x = ZeroPadding1D(3)(img_input) x = Conv1D(nb_filter, 7, 2, use_bias=False)(x) x = BatchNormalization(epsilon=eps, axis=concat_axis)(x) x = Activation('relu')(x) x = ZeroPadding1D(1)(x) x = MaxPooling1D(3, strides=2)(x) # Add dense blocks for block_idx in range(nb_dense_block - 1): stage = block_idx+2 x, nb_filter = dense_block(x, nb_layers, nb_filter, growth_rate, dropout_rate=dropout_rate) # Add transition_block x = transition_block(x, nb_filter, compression=compression, dropout_rate=dropout_rate) nb_filter = int(nb_filter * compression) x, nb_filter = dense_block(x, nb_layers, nb_filter, growth_rate, dropout_rate=dropout_rate) x = BatchNormalization(epsilon=eps, axis=concat_axis)(x) x = Activation('relu')(x) x = GlobalAveragePooling1D()(x) x = Dropout(dropout_rate)(x) x = Dense(classes)(x) x = Activation('softmax')(x) model = Model(img_input, x) return model
def __init__(self, output_channels: int, input_channels: Optional[int] = None, kernel_size: int = 3, pooling_size: int = 1, dropout_rate: float = 0.0): super().__init__() dimension_decrease_factor = 4 kernel_initializer = LecunNormal() self.dimension_decrease_layer = Convolution1D( output_channels // dimension_decrease_factor, kernel_size=1, activation=selu, kernel_initializer=kernel_initializer) self.convolutional_layer = Convolution1D( output_channels // dimension_decrease_factor, kernel_size=kernel_size, activation=selu, padding='same', kernel_initializer=kernel_initializer) self.dimension_increase_layer = Convolution1D( output_channels, kernel_size=1, activation=selu, kernel_initializer=kernel_initializer) if pooling_size > 1: self.pooling_layer = AveragePooling1D(pool_size=pooling_size, padding='same') else: self.pooling_layer = None if input_channels is not None and output_channels != input_channels: if output_channels < input_channels: raise NotImplementedError( f'Residual blocks with less output channels than input channels is not' f'implemented. Output channels was {output_channels} and input was' f'{input_channels}') self.dimension_change_permute0 = Permute((2, 1)) self.dimension_change_layer = ZeroPadding1D( padding=(0, output_channels - input_channels)) self.dimension_change_permute1 = Permute((2, 1)) else: self.dimension_change_layer = None if dropout_rate > 0: self.dropout_layer = AlphaDropout(rate=dropout_rate, noise_shape=(50, 1, output_channels)) else: self.dropout_layer = None
def causal_conv_1d(x, filters, kernel_size, dilation_rate=1, activation=None, dtype=tensorflow.complex64): padding = kernel_size + (kernel_size - 1) * (dilation_rate - 1) - 1 if padding > 0: x = ZeroPadding1D(padding=(padding, 0))(x) x = ComplexConv1D(filters=filters, kernel_size=kernel_size, strides=1, dilation_rate=dilation_rate, dtype=dtype)(x) if activation is not None: x = Activation(activation)(x) return x
def __init__(self, norm_output, **kwargs): super(Decoder_shared_iot, self).__init__(**kwargs) self.norm_output = norm_output self.upsam1 = UpSampling1D(size=2) self.zeroPadding = ZeroPadding1D(padding=(1, 1)) self.conv3 = Conv1D(filters=64, kernel_size=7, activation='swish', padding='same') self.conv4 = Conv1D(filters=128, kernel_size=7, activation='swish', padding='same') # 20 self.upsam2 = UpSampling1D(size=2) self.conv5 = Conv1D(filters=16, kernel_size=3, activation='swish', padding='same') self.conv6 = Conv1D(filters=32, kernel_size=5, activation='swish', padding='same') self.upsam3 = UpSampling1D(size=2) self.conv7 = Conv1D(filters=32, kernel_size=3, activation='swish', padding='same') # 25 # self.upsam4 = UpSampling1D(size=2) self.conv8 = Conv1D(filters=8, kernel_size=3, activation='swish', padding='same') self.conv9 = Conv1D(filters=2, kernel_size=3, activation='swish', padding='same') self.flatten = Flatten() self.Dense = Dense(2000, name='outputs_decoder')
def conv_block(x, nb_filter, dropout_rate=None): eps = 1.1e-5 # 1 Convolution (Bottleneck layer) inter_channel = nb_filter * 4 x = BatchNormalization(epsilon=eps, axis=concat_axis)(x) x = Activation('relu')(x) x = Conv1D(inter_channel, 1, use_bias=False)(x) if dropout_rate: x = Dropout(dropout_rate)(x) # 3 Convolution x = BatchNormalization(epsilon=eps, axis=concat_axis)(x) x = Activation('relu')(x) x = ZeroPadding1D(1)(x) x = Conv1D(nb_filter, 3, use_bias=False)(x) if dropout_rate: x = Dropout(dropout_rate)(x) return x
def causal_conv_1d(x, filters, kernel_size, weights_normalization, dilation_rate=1, activation=None, skip_connection=None): padding = kernel_size + (kernel_size - 1) * (dilation_rate - 1) - 1 if padding > 0: x = ZeroPadding1D(padding=(padding, 0))(x) conv_layer = Conv1D(filters=filters, kernel_size=kernel_size, strides=1, dilation_rate=dilation_rate) if weights_normalization: conv_layer = WeightNormalization(conv_layer) x = conv_layer(x) if skip_connection is not None: x = Add()([x, skip_connection]) if activation is not None: x = Activation(activation)(x) return x
def backend(self, x): # x = UpSampling1D(size=16)(x) x = self.unpool(x) if self.supervised: x = self.backend_supervised(x, self.X1) # DECONVOLUTION x = ZeroPadding1D((32, 31), name='zero_padding_deconv')(x) # zero pad the time series before doing 1d convolution x = Permute((2, 1), input_shape=(1024, 128))(x) x = Lambda(lambda x: K.expand_dims(x, axis=3))(x) # add dimension to input for 1d conv kernel = Lambda(lambda x: K.transpose(x))(self.w1.kernel) # transpose kernel kernel = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(kernel) # switch kernel dimensions up kernel = Lambda(lambda x: K.expand_dims(x, axis=3))(kernel) # add dimension to the kernel x = Deconvolution1D(kernel)(x) # x = Lambda(lambda x: K.conv2d(x[0], x[1], padding='valid'))([x, kernel]) x = Lambda(lambda x: K.squeeze(x, axis=3))(x) # remove extra dimension x = Permute((2, 1), input_shape=(1, 1024))(x) # permute output dims return x
def __call__(self, x): if self.strides != 1: x = ZeroPadding1D((0, 1), name='conv_pad_%d' % self.block_id)(x) x = SeparableConv1D(int(x.shape[-1]), 3, padding='same' if self.strides == 1 else 'valid', depth_multiplier=self.depth_multipliter, strides=self.strides, use_bias=False, name='conv_dw_%d' % self.block_id)(x) x = BatchNormalization(name='conv_dw_%d_bn' % self.block_id)(x) x = ReLU(6., name='conv_dw_%d_relu' % self.block_id)(x) x = Conv1D(self.pointwise_conv_filter, 1, padding='same', use_bias=False, strides=1, name='conv_pw_%d' % self.block_id)(x) x = BatchNormalization(name='conv_pw_%d_bn' % self.block_id)(x) x = ReLU(6., name='conv_pw_%d_relu' % self.block_id)(x) return x
x_test = x_test.reshape(x_test.shape[0], 10, 1) # ----------------------------------------------------------------------------------------------------- from tensorflow.keras.models import Sequential, load_model, Model from tensorflow.keras.layers import Conv1D, Conv2D, Dense, Dropout, MaxPooling2D, MaxPooling1D, BatchNormalization, Flatten, Input from tensorflow.keras.layers import Activation, ZeroPadding2D, Concatenate, AveragePooling2D, ZeroPadding1D, AveragePooling1D from tensorflow.keras.regularizers import l2 input = Input(shape=(10, 1)) <<<<<<< HEAD conv1_7x7_s2 = Conv1D(64, 2, strides=1, padding='valid', activation='relu', name='conv1/7x7_s2', kernel_regularizer=l2(0.0002))(input) pool1_helper = BatchNormalization()(conv1_7x7_s2) ======= input_pad = ZeroPadding1D(padding=(3, 3))(input) conv1_7x7_s2 = Conv1D(64, 2, strides=1, padding='valid', activation='relu', name='conv1/7x7_s2', kernel_regularizer=l2(0.0002))(input_pad) conv1_zero_pad = ZeroPadding1D(padding=1)(conv1_7x7_s2) pool1_helper = BatchNormalization()(conv1_zero_pad) >>>>>>> 3ea26d3fe05deb1dad4d912ac88003632ad3c404 pool1_3x3_s2 = MaxPooling1D(pool_size=2, strides=1, padding='valid', name='pool1/3x3_s2')(pool1_helper) conv2_3x3_reduce = Conv1D(64, 1, padding='same', activation='relu', name='conv2/3x3_reduce', kernel_regularizer=l2(0.0002))(pool1_3x3_s2) conv2_3x3 = Conv1D(192, 2, padding='same', activation='relu', name='conv2/3x3', kernel_regularizer=l2(0.0002))(conv2_3x3_reduce) <<<<<<< HEAD pool2_helper = BatchNormalization()(conv2_3x3) ======= conv2_zero_pad = ZeroPadding1D(padding=1)(conv2_3x3) pool2_helper = BatchNormalization()(conv2_zero_pad) >>>>>>> 3ea26d3fe05deb1dad4d912ac88003632ad3c404 pool2_3x3_s2 = MaxPooling1D(pool_size=2, strides=1, padding='valid', name='pool2/3x3_s2')(pool2_helper)
def test_delete_channels_zeropadding1d(channel_index): layer = ZeroPadding1D(3) layer_test_helper_flatten_1d(layer, channel_index)
def build_model(data_handler, m_params): K.set_floatx('float64') levels = m_params.levels blocks = m_params.blocks dil_chan = m_params.dil_chan res_chan = m_params.res_chan skip_chan = m_params.skip_chan initial_kernel = m_params.init_kernel kernel_size = m_params.kernel l2_decay = m_params.l2_decay kernel_init = m_params.kernel_init input_chan = data_handler.get_data_channels() cond_chan = data_handler.get_label_channels() start_pad = m_params.start_pad # Inputs input_layer = Input(shape=( None, input_chan, )) label_input = Input(shape=( None, cond_chan, )) # Padding the data x = ZeroPadding1D((start_pad, 0), name='start_pad')(input_layer) x = Conv1D(res_chan, initial_kernel, use_bias=True, kernel_initializer=kernel_init, name='start-conv', kernel_regularizer=l2(l2_decay))(x) skip_connection = 0 for i in range(blocks): neural_levels = levels dilation_rate = 1 if i == (blocks - 1): neural_levels = levels - 1 for j in range(neural_levels): layer_name = "_Block_" + str(i) + "_Level_" + str(j) residual_input = x label_conv = Conv1D( dil_chan * 2, 1, use_bias=True, kernel_initializer=kernel_init, name=('label_conv' + layer_name), kernel_regularizer=l2(l2_decay))(label_input) x = Conv1D(dil_chan * 2, kernel_size, padding="causal", dilation_rate=dilation_rate, use_bias=True, kernel_initializer=kernel_init, name=('dilated_conv' + layer_name), kernel_regularizer=l2(l2_decay))(x) x = Add(name=('add_label_dil' + layer_name))([x, label_conv]) filter_layer, gate_layer = Lambda(split_layer, arguments={ 'parts': 2, 'axis': 2 }, name=('split' + layer_name))(x) filter_layer = Activation( 'tanh', name=('filter' + layer_name))(filter_layer) gate_layer = Activation('sigmoid', name=('gate' + layer_name))(gate_layer) x = Multiply(name=('multiply' + layer_name))([filter_layer, gate_layer]) skip_layer = x skip_layer = Conv1D( skip_chan, 1, use_bias=True, kernel_initializer=kernel_init, name=('skip' + layer_name), kernel_regularizer=l2(l2_decay))(skip_layer) try: skip_connection = Add(name=('add_skip' + layer_name))( [skip_layer, skip_connection]) except: skip_connection = skip_layer x = Conv1D(res_chan, 1, use_bias=True, kernel_initializer=kernel_init, name=('residual' + layer_name), kernel_regularizer=l2(l2_decay))(x) x = Add(name=('add_res' + layer_name))([x, residual_input]) dilation_rate = dilation_rate * 2 label_output = Conv1D(skip_chan, 1, use_bias=True, kernel_initializer=kernel_init, name='label_out', kernel_regularizer=l2(l2_decay))(label_input) x = Add(name='add_skip_out')([skip_connection, label_output]) x = Activation("tanh", name='output')(x) x = Conv1D(skip_chan, 1, use_bias=True, kernel_initializer=kernel_init, name='final-output', kernel_regularizer=l2(l2_decay))(x) network = Model([input_layer, label_input], x) return network
def get_test_model_exhaustive(): """Returns a exhaustive test model.""" input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10), (7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15), (14, 15), (16, ), (16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4), (1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4), (1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3), (4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3), (9, 16, 1), (1, 9, 16)] inputs = [Input(shape=s) for s in input_shapes] outputs = [] outputs.append(Conv1D(1, 3, padding='valid')(inputs[6])) outputs.append(Conv1D(2, 1, padding='same')(inputs[6])) outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6])) outputs.append(ZeroPadding1D(2)(inputs[6])) outputs.append(Cropping1D((2, 3))(inputs[6])) outputs.append(MaxPooling1D(2)(inputs[6])) outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append(MaxPooling1D(2, data_format="channels_first")(inputs[6])) outputs.append(AveragePooling1D(2)(inputs[6])) outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append( AveragePooling1D(2, data_format="channels_first")(inputs[6])) outputs.append(GlobalMaxPooling1D()(inputs[6])) outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6])) outputs.append(GlobalAveragePooling1D()(inputs[6])) outputs.append( GlobalAveragePooling1D(data_format="channels_first")(inputs[6])) outputs.append(Conv2D(4, (3, 3))(inputs[4])) outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4])) outputs.append( Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4])) outputs.append( Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4])) outputs.append(SeparableConv2D(3, (3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((1, 2))(inputs[4])) outputs.append(MaxPooling2D((2, 2))(inputs[4])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default MaxPoolingOp only supports NHWC on device type CPU outputs.append( MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(AveragePooling2D((2, 2))(inputs[4])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default AvgPoolingOp only supports NHWC on device type CPU outputs.append( AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(GlobalAveragePooling2D()(inputs[4])) outputs.append( GlobalAveragePooling2D(data_format="channels_first")(inputs[4])) outputs.append(GlobalMaxPooling2D()(inputs[4])) outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4])) outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0])) outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0])) outputs.append(Permute((3, 4, 1, 2))(inputs[2])) outputs.append(Permute((2, 1, 3))(inputs[4])) outputs.append(Permute((2, 1))(inputs[6])) outputs.append(Permute((1, ))(inputs[8])) outputs.append(Permute((3, 1, 2))(inputs[31])) outputs.append(Permute((3, 1, 2))(inputs[32])) outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31]))) outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32]))) outputs.append(BatchNormalization()(inputs[0])) outputs.append(BatchNormalization(axis=1)(inputs[0])) outputs.append(BatchNormalization(axis=2)(inputs[0])) outputs.append(BatchNormalization(axis=3)(inputs[0])) outputs.append(BatchNormalization(axis=4)(inputs[0])) outputs.append(BatchNormalization(axis=5)(inputs[0])) outputs.append(BatchNormalization()(inputs[2])) outputs.append(BatchNormalization(axis=1)(inputs[2])) outputs.append(BatchNormalization(axis=2)(inputs[2])) outputs.append(BatchNormalization(axis=3)(inputs[2])) outputs.append(BatchNormalization(axis=4)(inputs[2])) outputs.append(BatchNormalization()(inputs[4])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now. outputs.append(BatchNormalization(axis=2)(inputs[4])) outputs.append(BatchNormalization(axis=3)(inputs[4])) outputs.append(BatchNormalization()(inputs[6])) outputs.append(BatchNormalization(axis=1)(inputs[6])) outputs.append(BatchNormalization(axis=2)(inputs[6])) outputs.append(BatchNormalization()(inputs[8])) outputs.append(BatchNormalization(axis=1)(inputs[8])) outputs.append(BatchNormalization()(inputs[27])) outputs.append(BatchNormalization(axis=1)(inputs[27])) outputs.append(BatchNormalization()(inputs[14])) outputs.append(BatchNormalization(axis=1)(inputs[14])) outputs.append(BatchNormalization(axis=2)(inputs[14])) outputs.append(BatchNormalization()(inputs[16])) # todo: check if TensorFlow >= 2.1 supports this # outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now. outputs.append(BatchNormalization(axis=2)(inputs[16])) outputs.append(BatchNormalization(axis=3)(inputs[16])) outputs.append(BatchNormalization()(inputs[18])) outputs.append(BatchNormalization(axis=1)(inputs[18])) outputs.append(BatchNormalization(axis=2)(inputs[18])) outputs.append(BatchNormalization(axis=3)(inputs[18])) outputs.append(BatchNormalization(axis=4)(inputs[18])) outputs.append(BatchNormalization()(inputs[20])) outputs.append(BatchNormalization(axis=1)(inputs[20])) outputs.append(BatchNormalization(axis=2)(inputs[20])) outputs.append(BatchNormalization(axis=3)(inputs[20])) outputs.append(BatchNormalization(axis=4)(inputs[20])) outputs.append(BatchNormalization(axis=5)(inputs[20])) outputs.append(Dropout(0.5)(inputs[4])) outputs.append(ZeroPadding2D(2)(inputs[4])) outputs.append(ZeroPadding2D((2, 3))(inputs[4])) outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Cropping2D(2)(inputs[4])) outputs.append(Cropping2D((2, 3))(inputs[4])) outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Dense(3, use_bias=True)(inputs[13])) outputs.append(Dense(3, use_bias=True)(inputs[14])) outputs.append(Dense(4, use_bias=False)(inputs[16])) outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18])) outputs.append(Dense(4, use_bias=False)(inputs[20])) outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0])) outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0])) outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0])) outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0])) outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0])) outputs.append(Reshape((16, ))(inputs[8])) outputs.append(Reshape((2, 8))(inputs[8])) outputs.append(Reshape((2, 2, 4))(inputs[8])) outputs.append(Reshape((2, 2, 2, 2))(inputs[8])) outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8])) outputs.append(RepeatVector(3)(inputs[8])) outputs.append( UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4])) outputs.append(ReLU()(inputs[0])) for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]: outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]])) for axis in [-4, -3, -2, -1, 1, 2, 3, 4]: outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]])) for axis in [-3, -2, -1, 1, 2, 3]: outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]])) for axis in [-2, -1, 1, 2]: outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]])) for axis in [-1, 1]: outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]])) for axis in [-1, 2]: outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]])) for axis in [-1, 3]: outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]])) for axis in [-1, 4]: outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]])) for axis in [-1, 5]: outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]])) outputs.append(UpSampling1D(size=2)(inputs[6])) # outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16] outputs.append(Multiply()([inputs[10], inputs[11]])) outputs.append(Multiply()([inputs[11], inputs[10]])) outputs.append(Multiply()([inputs[11], inputs[13]])) outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]])) outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]])) shared_conv = Conv2D(1, (1, 1), padding='valid', name='shared_conv', activation='relu') up_scale_2 = UpSampling2D((2, 2)) x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8) x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8) x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8) x = Concatenate()([x1, x2, x3]) # (3, 8, 8) outputs.append(x) x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8) outputs.append(x) x = Dropout(0.5)(x) outputs.append(x) x = Concatenate()([MaxPooling2D((2, 2))(x), AveragePooling2D((2, 2))(x)]) # (6, 4, 4) outputs.append(x) x = Flatten()(x) # (1, 1, 96) x = Dense(4, use_bias=False)(x) outputs.append(x) x = Dense(3)(x) # (1, 1, 3) outputs.append(x) outputs.append(Add()([inputs[26], inputs[30], inputs[30]])) outputs.append(Subtract()([inputs[26], inputs[30]])) outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]])) outputs.append(Average()([inputs[26], inputs[30], inputs[30]])) outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]])) outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]])) intermediate_input_shape = (3, ) intermediate_in = Input(intermediate_input_shape) intermediate_x = intermediate_in intermediate_x = Dense(8)(intermediate_x) intermediate_x = Dense(5, name='duplicate_layer_name')(intermediate_x) intermediate_model = Model(inputs=[intermediate_in], outputs=[intermediate_x], name='intermediate_model') intermediate_model.compile(loss='mse', optimizer='nadam') x = intermediate_model(x) # (1, 1, 5) intermediate_model_2 = Sequential() intermediate_model_2.add(Dense(7, input_shape=(5, ))) intermediate_model_2.add(Dense(5, name='duplicate_layer_name')) intermediate_model_2.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = intermediate_model_2(x) # (1, 1, 5) intermediate_model_3_nested = Sequential() intermediate_model_3_nested.add(Dense(7, input_shape=(6, ))) intermediate_model_3_nested.compile(optimizer='rmsprop', loss='categorical_crossentropy') intermediate_model_3 = Sequential() intermediate_model_3.add(Dense(6, input_shape=(5, ))) intermediate_model_3.add(intermediate_model_3_nested) intermediate_model_3.add(Dense(8)) intermediate_model_3.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = intermediate_model_3(x) # (1, 1, 8) x = Dense(3)(x) # (1, 1, 3) shared_activation = Activation('tanh') outputs = outputs + [ Activation('tanh')(inputs[25]), Activation('hard_sigmoid')(inputs[25]), Activation('selu')(inputs[25]), Activation('sigmoid')(inputs[25]), Activation('softplus')(inputs[25]), Activation('softmax')(inputs[25]), Activation('relu')(inputs[25]), Activation('relu6')(inputs[25]), Activation('swish')(inputs[25]), Activation('exponential')(inputs[25]), Activation('gelu')(inputs[25]), Activation('softsign')(inputs[25]), LeakyReLU()(inputs[25]), ReLU()(inputs[25]), ReLU(max_value=0.4, negative_slope=1.1, threshold=0.3)(inputs[25]), ELU()(inputs[25]), PReLU()(inputs[24]), PReLU()(inputs[25]), PReLU()(inputs[26]), shared_activation(inputs[25]), Activation('linear')(inputs[26]), Activation('linear')(inputs[23]), x, shared_activation(x), ] model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive') model.compile(loss='mse', optimizer='nadam') # fit to dummy data training_data_size = 2 data_in = generate_input_data(training_data_size, input_shapes) initial_data_out = model.predict(data_in) data_out = generate_output_data(training_data_size, initial_data_out) model.fit(data_in, data_out, epochs=10) return model
plt.ylabel('Pressão Atmosférica') plt.xlabel('Indice') plt.show() """**Inciando o processo de previsão via CNN**""" from tensorflow.keras.layers import Flatten #camada flatten para transformar os dados em uma dimensão from tensorflow.keras.layers import ZeroPadding1D #completa os dados após a convolução from tensorflow.keras.layers import Conv1D #camada de convolução from tensorflow.keras.layers import AveragePooling1D #camada de redução (média dos dados encontrados) #define a camada de entrada camada_entrada = Input(shape=(10,1), dtype='float32') #adiciona a camada de padding camada_padding = ZeroPadding1D(padding=1)(camada_entrada) #matém a quantidade de dados #adiona a camada de convolução camada_convolucao_1D = Conv1D(64, 3, strides=1, use_bias=True)(camada_padding) #adiciona 64 filtros com uma janela de convolução=3 #camada de pooling camada_pooling = AveragePooling1D(pool_size=3, strides=1)(camada_convolucao_1D) #reduz através do valor médio encontrado para a convolução (pode ser também o valor máximo) #camada flatten camada_flatten = Flatten()(camada_pooling) #utilizada para realizar o "reshape" dos dados para um vetor #adicionando a camada de dropout camada_dropout_cnn = Dropout(0.2)(camada_flatten) #camada de saída camada_saida = Dense(1, activation='linear')(camada_dropout_cnn)
bias_initializer='ones'), Input((10, 10, 5)), 32), (Conv2D(8, kernel_size=3, strides=2, data_format='channels_last', bias_initializer='ones'), Input((10, 10, 5)), 32), (ZeroPadding2D(((1, 2), (5, 3))), Input((10, 10, 5)), 32), (ZeroPadding2D(((1, 0), (5, 0))), Input((10, 10, 5)), 32), (ZeroPadding2D(((0, 1), (0, 5))), Input((10, 10, 5)), 32), (ZeroPadding3D(((0, 1), (0, 5), (0, 3))), Input((10, 10, 8, 5)), 32), (ZeroPadding3D(((1, 0), (5, 0), (3, 0))), Input((10, 10, 8, 5)), 32), (ZeroPadding3D(((1, 2), (5, 4), (3, 6))), Input((10, 10, 8, 5)), 32), (Conv1D( 8, kernel_size=3, data_format='channels_last', bias_initializer='ones'), Input((6, 5)), 32), (ZeroPadding1D(padding=(1, 0)), Input((6, 5)), 32), (ZeroPadding1D(padding=(1, 1)), Input((6, 5)), 32), (ZeroPadding1D(padding=(1, 1)), Input((6, 5)), 32), (Conv3D(8, kernel_size=(3, 3, 3), data_format='channels_last', bias_initializer='ones'), Input((6, 6, 6, 5)), 32), (ComplexConv2D(8, kernel_size=3), Input( (6, 6, 5), dtype=tf.complex64), 32), (ComplexConv2D(8, kernel_size=3), Input( (7, 7, 5), dtype=tf.complex64), 32), (ComplexConv2D(8, kernel_size=3, dilation_rate=2), Input( (10, 10, 5), dtype=tf.complex64), 32), (ComplexConv2D(8, kernel_size=3, strides=2), Input((10, 10, 5), dtype=tf.complex64), 32),
def get_model( self, shape_inputs: Optional[List[Tuple]] = None, ): f"""keras model for {self.model_name} Args: n_output (int): number of neurons in the last layer output_layer (str): activation function of last layer shape_inputs (list of tuples): Eg: For two inputs [(4800,1),(1000,4)] """ if shape_inputs is None: shape_inputs = [(value.get("len_input"), len(value.get("leads"))) for value in self.inputs.values()] # Inputs inp = Input(shape=shape_inputs[0]) padd = ZeroPadding1D(3)(inp) conv1 = Conv1D( 64, 7, strides=2, padding='valid', name='conv1', )(padd) conv1 = BatchNormalization(name='batch2')(conv1) conv1 = Activation('relu')(conv1) conv1 = ZeroPadding1D(1)(conv1) conv1 = MaxPool1D(3, 2)(conv1) conv2 = self.convolutional_block(conv1, [64, 64, 256], 3, '2', '1', strides=1) conv2 = self.identity_block(conv2, [64, 64, 256], 3, '2', '2') conv2 = self.identity_block(conv2, [64, 64, 256], 3, '2', '3') conv3 = self.convolutional_block(conv2, [128, 128, 512], 3, '3', '1') conv3 = self.identity_block(conv3, [128, 128, 512], 3, '3', '2') conv3 = self.identity_block(conv3, [128, 128, 512], 3, '3', '3') conv3 = self.identity_block(conv3, [128, 128, 512], 3, '3', '4') conv4 = self.convolutional_block(conv3, [256, 256, 1024], 3, '4', '1') conv4 = self.identity_block(conv4, [256, 256, 1024], 3, '4', '2') conv4 = self.identity_block(conv4, [256, 256, 1024], 3, '4', '3') conv4 = self.identity_block(conv4, [256, 256, 1024], 3, '4', '4') conv4 = self.identity_block(conv4, [256, 256, 1024], 3, '4', '5') conv4 = self.identity_block(conv4, [256, 256, 1024], 3, '4', '6') conv5 = self.convolutional_block(conv4, [512, 512, 2048], 3, '5', '1') conv5 = self.identity_block(conv5, [512, 512, 2048], 3, '5', '2') conv5 = self.identity_block(conv5, [512, 512, 2048], 3, '5', '3') avg_pool = GlobalAveragePooling1D()(conv5) output = Dense(self.n_output, activation=self.output_layer, dtype=tf.float32)(avg_pool) model = Model(inp, output) return model
def CNN1d(features_training, labels_training, features_testing, labels_testing, features_validation, labels_validation, modelname): """convert the 2D numpy arrary to 3D arrary""" features_training = features_training.reshape(features_training.shape + (1, )) features_testing = features_testing.reshape(features_testing.shape + (1, )) features_validation = features_validation.reshape( features_validation.shape + (1, )) nfeature = features_testing.shape[1] # """Empty the results file whenever we run TF""" # with open('CNN_accuracy_results.csv', 'w', newline='') as writeFile: # writer = csv.writer(writeFile) # writer.writerow(['Epoch', 'Testing Accuracy']) """Check testing accuracy after each epoch, does this by looking at accuracy of first 20000 shuffled testing features and testing labels""" class MyCallBack(Callback): def on_epoch_end(self, epoch, logs=None): acc = accuracy_score( labels_testing[:20000], model.predict_classes(features_testing[:20000])) print("Testing accuracy:", acc) # with open('CNN_accuracy_results.csv', 'a', newline='') as writeFile: # writer = csv.writer(writeFile) # try: # writer.writerow([self.model.history.epoch[-1]+2,acc]) # except: # writer.writerow([1, acc]) # writeFile.close() cbk = MyCallBack() """CNN model code""" print("Training data...") model = tf.keras.models.Sequential() # model.add(ZeroPadding1D(1,input_shape=(nfeature,1))) # model.add(Conv1D(filters=64, kernel_size=3, activation='relu',data_format='channels_last')) # model.add(ZeroPadding1D(1)) # model.add(Conv1D(64, 3, activation='relu',data_format='channels_last')) # model.add(MaxPooling1D(pool_size=2, strides=2)) # model.add(ZeroPadding1D(1)) # model.add(Conv1D(128, 3, activation='relu',data_format='channels_last')) # model.add(ZeroPadding1D(1)) # model.add(Conv1D(128,3, activation='relu',data_format='channels_last')) # model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(ZeroPadding1D(1, input_shape=(nfeature, 1))) model.add(Conv1D(512, 3, activation='relu', data_format='channels_last')) model.add(ZeroPadding1D(1)) model.add(Conv1D(512, 3, activation='relu', data_format='channels_last')) model.add(ZeroPadding1D(1)) model.add(Conv1D(512, 3, activation='relu', data_format='channels_last')) # model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Flatten()) model.add(Dense(512, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(8, activation='softmax')) start = time.time() model.compile( optimizer=Adam(lr=1.5e-4, ), # Good default optimizer to start with loss= 'sparse_categorical_crossentropy', # how will we calculate our "error." Neural network aims to minimize loss. metrics=['accuracy']) # what to track # simple early stopping es = EarlyStopping(monitor='val_loss', mode='min', verbose=1) model.fit( features_training, labels_training, epochs=20, batch_size=3200, # validation_split=0.2, validation_data=(features_validation, labels_validation), callbacks=[es], shuffle=True) # train the model # # summarize history for accuracy # plt.plot(history.history['accuracy']) # plt.plot(history.history['val_accuracy']) # plt.title('model accuracy') # plt.ylabel('accuracy') # plt.xlabel('epoch') # plt.legend(['train', 'validation'], loc='upper left') # plt.show() # # summarize history for loss # plt.plot(history.history['loss']) # plt.plot(history.history['val_loss']) # plt.title('model loss') # plt.ylabel('loss') # plt.xlabel('epoch') # plt.legend(['train', 'validation'], loc='upper left') # plt.show() OA = accuracy_score(labels_testing, model.predict_classes(features_testing)) Kappa = cohen_kappa_score(labels_testing, model.predict_classes(features_testing)) array = confusion_matrix(labels_testing, model.predict_classes(features_testing)) print("Test Accuracy ", OA) print("Confusion matrix ", array) model.save('CNN1d_model_' + modelname + '.h5') end = time.time() print(end - start) t = end - start with open('CNN1d_accuracy_results_' + modelname + '.csv', 'a', newline='') as writeFile: writer = csv.writer(writeFile) writer.writerow([OA, Kappa, t]) writeFile.close() """Visualization of results ~ Confusion matrix ~ Labels_validation X Features_validation""" # array=normalize(confusion_matrix(model.history.validation_data[1], model.predict_classes(model.history.validation_data[0]))) # df_cm = pd.DataFrame(array, range(1,8), # range(1,8)) # sn.set(font_scale=1.4) #for label size # sn.heatmap(df_cm, annot=True,annot_kws={"size": 10}, cmap='Blues')# font size # plt.title('Validation accuracy') # plt.xlabel('Predicted label', fontsize=16) # plt.ylabel('True label', fontsize=16) # plt.show() """Visualization of results ~ Confusion matrix ~ Labels_testing X Features_testing""" df_cm = pd.DataFrame(array, range(1, 8), range(1, 8)) df_cm.to_csv('CNN1d_ConfusionMatrix_' + modelname + '.csv') with open('CNN1d_accuracy_results_' + modelname + '.csv', 'a', newline='') as targetcsv: writer = csv.writer(targetcsv) with open('CNN1d_ConfusionMatrix_' + modelname + '.csv', 'r') as sourcecsv: reader = csv.reader(sourcecsv) for row in reader: writer.writerow(row) targetcsv.close()
def ResNet1D(self, save_img=False): """ Builds the residual network architecture by calling identity_block_1D and convolutional_block_1D. Puts together structure and depth of model here by stacking identity and convolutional layers. More stages/blocks may be added at will. """ # X_input = X_input X = ZeroPadding1D(3)(self.X_input) # stage 1 X = Conv1D(filters=32, kernel_size=7, strides=2, kernel_initializer='glorot_uniform', name='Conv1D_Stage_1')(X) X = BatchNormalization(name='BN1D_Stage_1')(X) X = relu(X) X = MaxPooling1D(pool_size=3, strides=2)(X) # stage 2 X = self.convolutional_block_1D(X=X, filters=[32, 32, 126], kernel_size=3, stride=1, stage=2, block='A') X = self.identity_block_1D(X=X, filters=[32, 32, 126], kernel_size=3, stage=2, block='B') X = self.identity_block_1D(X=X, filters=[32, 32, 126], kernel_size=3, stage=2, block='C') # stage 3 X = self.convolutional_block_1D(X=X, filters=[64, 64, 256], kernel_size=3, stride=2, stage=3, block='A') X = self.identity_block_1D(X=X, filters=[64, 64, 256], kernel_size=3, stage=3, block='B') X = self.identity_block_1D(X=X, filters=[64, 64, 256], kernel_size=3, stage=3, block='C') X = self.identity_block_1D(X=X, filters=[64, 64, 256], kernel_size=3, stage=3, block='D') # stage 4 X = self.convolutional_block_1D(X=X, filters=[128, 128, 512], kernel_size=3, stride=2, stage=4, block='A') X = self.identity_block_1D(X=X, filters=[128, 128, 512], kernel_size=3, stage=4, block='B') X = self.identity_block_1D(X=X, filters=[128, 128, 512], kernel_size=3, stage=4, block='C') X = self.identity_block_1D(X=X, filters=[128, 128, 512], kernel_size=3, stage=4, block='D') X = self.identity_block_1D(X=X, filters=[128, 128, 512], kernel_size=3, stage=4, block='E') X = self.identity_block_1D(X=X, filters=[128, 128, 512], kernel_size=3, stage=4, block='F') if self.trim_end: # return feature map and do NOT perform sigmoid activation. return X else: # get output layers and perform sigmoid activation. X = Flatten()(X) X = Dense(units=self.num_classes, activation='sigmoid', name='final_dense')(X) # create model model = Model(inputs=self.X_input, outputs=X, name='myResNet') if save_img: keras.utils.plot_model(model, 'ResNet1D.png') return model
def check_and_add_padding(x, y): if y.shape[1] % 2 != 0: return ZeroPadding1D(padding=(1, 0))(x) return x
def get_test_model_exhaustive(): """Returns a exhaustive test model.""" input_shapes = [ (2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10), (7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15), (14, 15), (16, ), (16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4), (1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4), (1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3), (4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3), ] inputs = [Input(shape=s) for s in input_shapes] outputs = [] outputs.append(Conv1D(1, 3, padding='valid')(inputs[6])) outputs.append(Conv1D(2, 1, padding='same')(inputs[6])) outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6])) outputs.append(ZeroPadding1D(2)(inputs[6])) outputs.append(Cropping1D((2, 3))(inputs[6])) outputs.append(MaxPooling1D(2)(inputs[6])) outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append(AveragePooling1D(2)(inputs[6])) outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6])) outputs.append(GlobalMaxPooling1D()(inputs[6])) outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6])) outputs.append(GlobalAveragePooling1D()(inputs[6])) outputs.append( GlobalAveragePooling1D(data_format="channels_first")(inputs[6])) outputs.append(Conv2D(4, (3, 3))(inputs[4])) outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4])) outputs.append( Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4])) outputs.append( Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4])) outputs.append(SeparableConv2D(3, (3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((1, 2))(inputs[4])) outputs.append(MaxPooling2D((2, 2))(inputs[4])) outputs.append( MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(AveragePooling2D((2, 2))(inputs[4])) outputs.append( AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4])) outputs.append(GlobalAveragePooling2D()(inputs[4])) outputs.append( GlobalAveragePooling2D(data_format="channels_first")(inputs[4])) outputs.append(GlobalMaxPooling2D()(inputs[4])) outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4])) outputs.append(BatchNormalization()(inputs[4])) outputs.append(Dropout(0.5)(inputs[4])) outputs.append(ZeroPadding2D(2)(inputs[4])) outputs.append(ZeroPadding2D((2, 3))(inputs[4])) outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Cropping2D(2)(inputs[4])) outputs.append(Cropping2D((2, 3))(inputs[4])) outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4])) outputs.append(Dense(3, use_bias=True)(inputs[13])) outputs.append(Dense(3, use_bias=True)(inputs[14])) outputs.append(Dense(4, use_bias=False)(inputs[16])) outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18])) outputs.append(Dense(4, use_bias=False)(inputs[20])) outputs.append( UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4])) outputs.append( UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4])) outputs.append( UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4])) for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]: outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]])) for axis in [-4, -3, -2, -1, 1, 2, 3, 4]: outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]])) for axis in [-3, -2, -1, 1, 2, 3]: outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]])) for axis in [-2, -1, 1, 2]: outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]])) for axis in [-1, 1]: outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]])) for axis in [-1, 2]: outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]])) for axis in [-1, 3]: outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]])) for axis in [-1, 4]: outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]])) for axis in [-1, 5]: outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]])) outputs.append(UpSampling1D(size=2)(inputs[6])) outputs.append(Multiply()([inputs[10], inputs[11]])) outputs.append(Multiply()([inputs[11], inputs[10]])) outputs.append(Multiply()([inputs[11], inputs[13]])) outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]])) outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]])) shared_conv = Conv2D(1, (1, 1), padding='valid', name='shared_conv', activation='relu') up_scale_2 = UpSampling2D((2, 2)) x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8) x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8) x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8) x = Concatenate()([x1, x2, x3]) # (3, 8, 8) outputs.append(x) x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8) outputs.append(x) x = Dropout(0.5)(x) outputs.append(x) x = Concatenate()([MaxPooling2D((2, 2))(x), AveragePooling2D((2, 2))(x)]) # (6, 4, 4) outputs.append(x) x = Flatten()(x) # (1, 1, 96) x = Dense(4, use_bias=False)(x) outputs.append(x) x = Dense(3)(x) # (1, 1, 3) outputs.append(x) outputs.append(Add()([inputs[26], inputs[30], inputs[30]])) outputs.append(Subtract()([inputs[26], inputs[30]])) outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]])) outputs.append(Average()([inputs[26], inputs[30], inputs[30]])) outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]])) outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]])) intermediate_input_shape = (3, ) intermediate_in = Input(intermediate_input_shape) intermediate_x = intermediate_in intermediate_x = Dense(8)(intermediate_x) intermediate_x = Dense(5)(intermediate_x) intermediate_model = Model(inputs=[intermediate_in], outputs=[intermediate_x], name='intermediate_model') intermediate_model.compile(loss='mse', optimizer='nadam') x = intermediate_model(x) # (1, 1, 5) intermediate_model_2 = Sequential() intermediate_model_2.add(Dense(7, input_shape=(5, ))) intermediate_model_2.add(Dense(5)) intermediate_model_2.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = intermediate_model_2(x) # (1, 1, 5) x = Dense(3)(x) # (1, 1, 3) shared_activation = Activation('tanh') outputs = outputs + [ Activation('tanh')(inputs[25]), Activation('hard_sigmoid')(inputs[25]), Activation('selu')(inputs[25]), Activation('sigmoid')(inputs[25]), Activation('softplus')(inputs[25]), Activation('softmax')(inputs[25]), Activation('relu')(inputs[25]), LeakyReLU()(inputs[25]), ELU()(inputs[25]), PReLU()(inputs[24]), PReLU()(inputs[25]), PReLU()(inputs[26]), shared_activation(inputs[25]), Activation('linear')(inputs[26]), Activation('linear')(inputs[23]), x, shared_activation(x), ] model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive') model.compile(loss='mse', optimizer='nadam') # fit to dummy data training_data_size = 1 data_in = generate_input_data(training_data_size, input_shapes) initial_data_out = model.predict(data_in) data_out = generate_output_data(training_data_size, initial_data_out) model.fit(data_in, data_out, epochs=10) return model
def FCN(input_shape, max_seqlen, num_classes=2, norm_max=1.0): """ Generate a fully convolutional neural network (FCN) model. Parameters ---------- input_shape : tuple defining shape of the input dataset: (num_timesteps, num_channels) num_classes : integer defining number of classes for classification task norm_max : maximum norm for constraint Returns ------- model : Keras model """ outputdim = num_classes # number of classes inputs = Input(shape = input_shape) # Zero padding pad_wd = (max_seqlen - input_shape[0])//2 x = ZeroPadding1D((pad_wd,pad_wd))(inputs) # Stage 1 x = Conv1D(filters=32, kernel_size=7, strides=2, padding='valid', use_bias=False, kernel_constraint=MaxNorm(norm_max, axis=[0,1,2]), name = 'conv1', kernel_initializer=glorot_uniform(seed=0))(x) x = LeakyReLU(alpha=0.1)(x) x = BatchNormalization(axis=-1, momentum=0.9, name='bn_conv1', gamma_constraint=MaxNorm(norm_max,axis=0), beta_constraint=MaxNorm(norm_max,axis=0))(x) # Stage 2 x = conv_block(x, ksz=3, filters=[16,16,32], stage=2, block='a', s=2, norm_max=norm_max) x = identity_block(x, ksz=3, filters=[16,16,32], stage=2, block='b', norm_max=norm_max) x = identity_block(x, ksz=3, filters=[16,16,32], stage=2, block='c', norm_max=norm_max) # # Stage 3 # x = conv_block(x, ksz=3, filters=[64,64,128], stage=3, block='a', s=2) # x = identity_block(x, ksz=3, filters=[64,64,128], stage=3, block='b') # x = identity_block(x, ksz=3, filters=[64,64,128], stage=3, block='c') # x = identity_block(x, ksz=3, filters=[64,64,128], stage=3, block='d') # # Stage 4 # x = conv_block(x, ksz=3, filters=[128,128,256], stage=4, block='a', s=2) # x = identity_block(x, ksz=3, filters=[128,128,256], stage=4, block='b') # x = identity_block(x, ksz=3, filters=[128,128,256], stage=4, block='c') # x = identity_block(x, ksz=3, filters=[128,128,256], stage=4, block='d') # x = identity_block(x, ksz=3, filters=[128,128,256], stage=4, block='e') # # Stage 5 # x = conv_block(x, ksz=3, filters=[256,256,512], stage=5, block='a', s=2) # x = identity_block(x, ksz=3, filters=[256,256,512], stage=5, block='b') # x = identity_block(x, ksz=3, filters=[256,256,512], stage=5, block='c') # x = identity_block(x, ksz=3, filters=[256,256,512], stage=5, block='d') # x = identity_block(x, ksz=3, filters=[256,256,512], stage=5, block='e') # x = identity_block(x, ksz=3, filters=[256,256,512], stage=5, block='f') # Output stage x = Conv1DTranspose(x, filters=64, ksz=5, s=4, norm_max=norm_max) x = GlobalAveragePooling1D()(x) outputs = Dense(num_classes, activation='softmax', name='Dense', kernel_constraint=MaxNorm(norm_max,axis=[0,1]), bias_constraint=MaxNorm(norm_max,axis=0), kernel_initializer=glorot_uniform(seed=0))(x) model = Model(inputs=inputs, outputs=outputs) return model
def resnet18_1d(input_shape=(750, 1), classes=1, as_model=False): # Define the input as a tensor with shape input_shape X_input = Input(input_shape) # Zero-Padding X = ZeroPadding1D(3)(X_input) # Stage 1 X = Conv1D(64, 7, strides=2, name='conv1', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=2, name='bn_conv1')(X) X = Activation('relu')(X) X = MaxPooling1D(3, strides=2)(X) # Stage 2 X = identity_block_18_1d(X, 3, [64, 64], stage=2, block='a') X = identity_block_18_1d(X, 3, [64, 64], stage=2, block='b') # Stage 3 X = convolutional_block_18_1d(X, f=3, filters=[128, 128], stage=3, block='a', s=2) X = identity_block_18_1d(X, 3, [128, 128], stage=3, block='b') # Stage 4 X = convolutional_block_18_1d(X, f=3, filters=[256, 256], stage=4, block='a', s=2) X = identity_block_18_1d(X, 3, [256, 256], stage=4, block='b') # Stage 5 X = convolutional_block_18_1d(X, f=3, filters=[512, 512], stage=5, block='a', s=2) X = identity_block_18_1d(X, 3, [512, 512], stage=5, block='b') # AVGPOOL X = AveragePooling1D(2, name="avg_pool")(X) # output layer X = Flatten()(X) if as_model: X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer=glorot_uniform(seed=0))(X) # Create modelzoo model = Model(inputs=X_input, outputs=X, name='ResNet18_1d') return model