def build_decoder(self, latent_dim, block_size, k_size): model = Sequential() model.add(Input(shape=(latent_dim, ))) model.add(Dense(units=latent_dim * 2, activation='relu')) model.add(Reshape(target_shape=(latent_dim, latent_dim))) current_len = model.layers[-1].output_shape[1] filters = 256 while current_len < block_size: model.add( Conv1DTranspose(filters, strides=1, kernel_size=k_size, padding='same')) model.add(UpSampling1D(size=2)) model.add(Activation('relu')) filters = filters // 2 current_len = model.layers[-1].output_shape[1] model.add( Conv1DTranspose(1, kernel_size=k_size, padding='same', activation=None)) return model
def define_generator(latent_dim, n_classes=91): # weight initialization init = RandomNormal(stddev=0.02) # label input in_label = Input(shape=(1,)) #print("in_label.shape===",in_label.shape) # embedding for categorical input li = Embedding(n_classes, 50)(in_label) #print("li.shape====",li.shape) # linear multiplication n_nodes = 1250*128 #196*128 #print("n_nodes.shape==",n_nodes.shape) li = Dense(n_nodes, kernel_initializer=init)(li) # reshape to additional channel li = Reshape((1250, 128))(li) #196,128 # image generator input in_lat = Input(shape=(latent_dim,)) # foundation for 7x7 image n_nodes = int(128 *(5000/4) * 1) gen = Dense(n_nodes, kernel_initializer=init)(in_lat) gen = Activation('relu')(gen) gen = Reshape((math.ceil(5000/4)* 1, 128))(gen) # merge image gen and label input merge = Concatenate()([gen, li]) # upsample to 14x14 gen = Conv1DTranspose(128, 4*1, strides=2*1, padding='same', kernel_initializer=init)(merge) gen = BatchNormalization()(gen) gen = Activation('relu')(gen) # upsample to 28x28 gen = Conv1DTranspose(1, 4*1, strides=2*1, padding='same', kernel_initializer=init)(gen) out_layer = Activation('tanh')(gen) # define model model = Model([in_lat, in_label], out_layer) return model
def build_generator(z_dim): model = Sequential() model.add(Reshape((z_dim, 1), input_dim=z_dim)) model.add(Conv1DTranspose(8, kernel_size=4, strides=2)) model.add(LayerNormalization()) model.add(LeakyReLU()) model.add(Conv1DTranspose(8, kernel_size=6, strides=2)) model.add(LayerNormalization()) model.add(LeakyReLU()) model.add(Conv1DTranspose(8, kernel_size=6, strides=1)) model.add(LayerNormalization()) model.add(LeakyReLU()) model.add(Conv1DTranspose(1, kernel_size=7, strides=1, activation='tanh')) # inputs = tf.keras.Input(shape=(z_dim,)) # x1 = Dense(16)(inputs) # x1 = LayerNormalization()(x1) # x1 = LeakyReLU(alpha=0.01)(x1) # x2 = Reshape((16, 1))(x1) # x3 = Conv1DTranspose(1,kernel_size=10, strides=2)(x2) # x3 = LayerNormalization()(x3) # x3 = LeakyReLU(alpha=0.01)(x3) # x4 = Conv1DTranspose(1,kernel_size=10, strides=2)(x3) # x4 = LayerNormalization()(x4) # x4 = LeakyReLU(alpha=0.01)(x4) # x5 = Conv1DTranspose(1,kernel_size=12, strides=1, activation='tanh')(x4) # x5 = Reshape([99])(x5) # model = tf.keras.Model(inputs=inputs, # outputs=x5) return model
def create_fcdae(input_shape, activation_function, kernel_initializer, lr): autoencoder_model = Sequential() kernel_size = 3 n_layers = 5 # encoder part autoencoder_model.add(Conv1D(32, kernel_size = kernel_size ,input_shape = input_shape, strides = 1, padding = 'same', activation = activation_function, kernel_initializer = kernel_initializer)) for i in range(1, n_layers): autoencoder_model.add(Conv1D(2**(i + 5), kernel_size = kernel_size, strides = 2, padding = 'same', activation = activation_function, kernel_initializer = kernel_initializer)) # decoder part for i in range(n_layers - 2, -1, -1): autoencoder_model.add(Conv1DTranspose(2**(i + 5) , kernel_size = kernel_size, strides = 2, padding = 'same', activation = activation_function, kernel_initializer = kernel_initializer )) autoencoder_model.add(Conv1DTranspose(4 , kernel_size = kernel_size, strides = 1, padding = 'same', activation = 'linear', kernel_initializer = kernel_initializer )) opt = tf.keras.optimizers.Adam(learning_rate = lr) autoencoder_model.compile(optimizer=opt, loss= 'mse') return autoencoder_model
def __design_generator(self): inputs = Input(shape=self.latent_features) x = Dense(units=self.latent_features * 32, kernel_initializer=RandomNormal(stddev=0.02))(inputs) x = LeakyReLU(alpha=0.2)(x) x = Reshape(target_shape=(self.latent_features, 32))(x) x = Conv1DTranspose(filters=32, kernel_size=4, strides=2, padding='same', kernel_initializer=RandomNormal(stddev=0.02))(x) x = LeakyReLU(alpha=0.2)(x) x = Conv1DTranspose(filters=32, kernel_size=4, strides=2, padding='same', kernel_initializer=RandomNormal(stddev=0.02))(x) x = LeakyReLU(alpha=0.2)(x) x = Conv1DTranspose(filters=32, kernel_size=4, strides=2, padding='same', kernel_initializer=RandomNormal(stddev=0.02))(x) x = LeakyReLU(alpha=0.2)(x) x = Flatten()(x) outputs = Dense(units=self.features, activation='tanh', kernel_initializer=RandomNormal(stddev=0.02))(x) model = Model(inputs=inputs, outputs=outputs) return model
def __init__(self, input_shape): super(Autoencoder, self).__init__() self.encoder = tf.keras.Sequential([ Input(shape=input_shape), Conv1D(128, 3, activation='relu', padding='same', strides=1), Conv1D(64, 3, activation='relu', padding='same', strides=1), Conv1D(32, 3, activation='relu', padding='same', strides=1), ]) self.decoder = tf.keras.Sequential([ Conv1DTranspose(32, kernel_size=3, strides=1, activation='relu', padding='same'), Conv1DTranspose(64, kernel_size=3, strides=1, activation='relu', padding='same'), Conv1DTranspose(128, kernel_size=3, strides=1, activation='relu', padding='same'), Conv1D(1, kernel_size=3, activation='sigmoid', padding='same') ])
def define_generator_tf2(latent_dim, n_outputs=2): ''' questo modello funziona solo con TF2 modello generatore, coda del modello, impara a creare un dato che possa essere confuso con un dato reale. latent_dim: (int) dimensione dello spazio latente, che sono numeri random n_outputs: (int) dimensione del dato reale, questo e' il dato generato che vuole imitare il dato reale return: model ''' model = Sequential() # foundation for 7x7 image n_nodes = 128 * int(n_outputs / 4) model.add(Dense(n_nodes, input_dim=latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(Reshape((int(n_outputs / 4), 128))) # upsample to 14x14 model.add(Conv1DTranspose(128, 4, strides=2, padding='same')) model.add(LeakyReLU(alpha=0.2)) # upsample to 28x28 model.add(Conv1DTranspose(128, 4, strides=2, padding='same')) model.add(LeakyReLU(alpha=0.2)) model.add( Conv1D(1, int(n_outputs / 4), activation='sigmoid', padding='same')) model.add(Flatten()) return model
def __init__(self, kernel_size, block_size): super(Autoencoder, self).__init__() stride = 2 self.encoder = tf.keras.Sequential([ Input(shape=(block_size, 1)), Conv1D(64, kernel_size, strides=stride, padding='same'), # AveragePooling1D(pool_size=2), Conv1D(64, kernel_size, strides=stride, padding='same'), # AveragePooling1D(pool_size=2), Conv1D(16, kernel_size, strides=stride, padding='same'), # AveragePooling1D(pool_size=2), Conv1D(16, kernel_size, strides=stride, padding='same'), # AveragePooling1D(pool_size=2), Conv1D(4, kernel_size, strides=stride, padding='same'), # Conv1D(1, kernel_size, activation='tanh', padding='same', # activity_regularizer=tf.keras.regularizers.l1(10e-5))]) Conv1D(1, kernel_size, activation=None, padding='same') ]) self.decoder = tf.keras.Sequential([ Conv1DTranspose(4, kernel_size, strides=stride, activation='tanh', padding='same'), # UpSampling1D(size=2), Conv1DTranspose(16, kernel_size, strides=stride, activation='tanh', padding='same'), # UpSampling1D(size=2), Conv1DTranspose(16, kernel_size, strides=stride, activation='tanh', padding='same'), # UpSampling1D(size=2), Conv1DTranspose(64, kernel_size, strides=stride, activation='tanh', padding='same'), # UpSampling1D(size=2), Conv1DTranspose(64, kernel_size, strides=stride, activation='tanh', padding='same'), Conv1D(1, kernel_size, padding='same', activation='tanh') ]) self.encoded_shape = self.encoder.layers[-1].output_shape print(f'ENCODED SHAPE {self.encoded_shape}')
def build_model(self, input_shape, latent_dim=4): encoder_input = Input(shape=input_shape) encoder = Conv1D(320, kernel_size=9, strides=4, activation='relu', padding='same')(encoder_input) encoder = Conv1D(160, kernel_size=9, strides=4, activation='relu', padding='same')(encoder) encoder = Conv1D(8, kernel_size=9, strides=4, activation='relu', padding='same')(encoder) encoder = Conv1D(1, kernel_size=9, activation=None, padding='same')(encoder) encoder_output = Dense(latent_dim)(encoder) print(encoder_output.shape) decoder_input = Input( (encoder_output.shape[1], encoder_output.shape[2])) decoder = Conv1DTranspose(8, kernel_size=9, strides=4, activation='relu', padding='same')(decoder_input) decoder = Conv1DTranspose(160, kernel_size=9, strides=4, activation='relu', padding='same')(decoder) decoder = Conv1DTranspose(320, kernel_size=9, strides=4, activation='relu', padding='same')(decoder) decoder_output = Conv1D(1, kernel_size=9, strides=1, activation='tanh', padding='same')(decoder) return Model(inputs=[encoder_input], outputs=[encoder_output]), Model(inputs=[decoder_input], outputs=[decoder_output])
def up_down(config, name,down_sample, l1, l2, gms, filters_up, filters_down, vocab): projection = Dense(filters_down[0], activation = None, use_bias = False, kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2), name=name+"_in_proj") conv_down = [Conv1D(filters_down[i], 9, strides=2, padding="same", activation = "relu", kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2), name = name + "_down_{}".format(i)) for i in range(down_sample)] conv_up = [Conv1DTranspose(filters_up[i], 9, strides=2, padding="same", activation = "relu", kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2), name = name + "up_{}".format(i)) for i in range(down_sample)] outconv1 = Conv1D(filters_up[-1], 9, padding = 'same', activation = "relu", name = name + "_output1") outconv2 = SpectralNormalization(Conv1D(vocab, 9, padding = 'same', activation = gms, name = name + "_output2"), name = name + "_output2_sn") return projection, conv_down, conv_up, outconv1, outconv2
def __init__(self): super(SSRN, self).__init__() self.model = tf.keras.Sequential([ Conv1D(filters=c, kernel_size=1, padding='same', kernel_initializer='he_normal'), highway(filters=c, kernel_size=3, padding='same', dilation_rate=1), highway(filters=c, kernel_size=3, padding='same', dilation_rate=3) ]) for _ in range(2): self.model.add(Conv1DTranspose(filters=c, kernel_size=2, strides=2, padding='same', kernel_initializer='he_normal')) self.model.add(highway(filters=c, kernel_size=3, padding='same', dilation_rate=1)) self.model.add(highway(filters=c, kernel_size=3, padding='same', dilation_rate=3)) self.model.add(Conv1D(filters=2*c, kernel_size=1, padding='same', kernel_initializer='he_normal')) for _ in range(2): self.model.add(highway(filters=2*c, kernel_size=3, padding='same', dilation_rate=1)) self.model.add(Conv1D(filters=f, kernel_size=1, padding='same', kernel_initializer='he_normal')) for _ in range(2): self.model.add(Conv1D(filters=f, kernel_size=1, padding='same', activation='relu', kernel_initializer='he_normal')) self.model.add(Conv1D(filters=f, kernel_size=1, padding='same', activation='sigmoid', kernel_initializer='he_normal'))
def __init__(self, latent_dim, inputShape, temperature=1): """ Parameters ---------- inputShape : tuple The shape of the data for which the model is trained on. The input shape is the shape of one element in your dataset, not the shape of the dataset itself. latent_dim : int Dimension of the encoded matrix. Default is 2. tau : float Temperature of the Gumbel-Softmax activation. Using tau=0 discretizes the output of the decoder and the higher the temperature, the less discrete the output will be. """ super(VAE, self).__init__() self.latent_dim = latent_dim self.inputShape = inputShape self.tau = temperature #self.encoder = Encoder(latent_dim, encoder_shape) self.encoder = tf.keras.Sequential( [ Input(self.inputShape), Conv1D(8, 3, activation="relu", strides=2, padding="same"), Conv1D(16, 3, activation="relu", strides=2, padding="same"), Conv1D(48, 3, activation="relu", strides=3, padding="same"), Conv1D(144, 3, activation="relu", strides=3, padding="same"), Flatten(), Dense(latent_dim + latent_dim) ] ) #self.decoder = Decoder() self.decoder = tf.keras.Sequential( [ Input((latent_dim,)), Dense((14 * 144), activation="relu"), Reshape((14, 144)), Conv1DTranspose(48, 3, activation="relu", strides=3, padding="same"), Conv1DTranspose(16, 3, activation="relu", strides=3, padding="same"), Conv1DTranspose(8, 3, activation="relu", strides=2, padding="same"), Conv1DTranspose(8, 3, activation="relu", strides=2, padding="same"), Conv1DTranspose(5, 3, activation="relu", padding="same"), Activation(Gumbel_Softmax(temperature=self.tau)) ] )
def __init__(self, frame_length, **kwargs): super(Decoder, self).__init__(**kwargs) self.frame_length = frame_length self.upsampling = UpSampling1D(size=frame_length // 64) self.dense1 = Dense(128, activation='softplus') self.dense2 = Dense(64, activation='softplus') self.dense3 = Dense(64, activation='softplus') self.dense4 = Dense(128, activation='relu') self.convtranspose1 = Conv1DTranspose(1, 128, padding='same')
def create_conv_transpose_autoencoder( input_dimension, optimizer='adam', loss='binary_crossentropy', kernel_size=7, dropout_rate=0.2, strides=2, activation='relu', padding='same', number_of_features=1, ) -> Model: model = Sequential([ Input(shape=(input_dimension, number_of_features)), Conv1D(filters=32, kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), Dropout(rate=dropout_rate), Conv1D(filters=16, kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), Conv1DTranspose(filters=16, kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), Dropout(rate=dropout_rate), Conv1DTranspose(filters=32, kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), Conv1DTranspose(filters=1, kernel_size=kernel_size, padding=padding), ]) model.compile(optimizer=optimizer, loss=loss) return model
def __init__(self, latent_dim): super(CVAE, self).__init__() self.latent_dim = latent_dim self.encoder = Sequential( [InputLayer(input_shape=(1,90001)), Conv1D(64,1,2), Conv1D(128,1,2), Conv1D(128,1,2), Conv1D(256,1,2), Flatten(), Dense(latent_dim+latent_dim) ]) self.decoder = tf.keras.Sequential( [InputLayer(input_shape=(latent_dim,)), Reshape(target_shape=(1,latent_dim)), Conv1DTranspose(512,1,1), Conv1DTranspose(256,1,1), Conv1DTranspose(128,1,1), Conv1DTranspose(64,1,1), Conv1DTranspose(90001,1,1), ] )
def test_conv1dtranspose(): ignore = True major, minor, _ = tf.version.VERSION.split(".") if int(major) >= 2 and int(minor) >= 3: ignore = False if ignore: return from tensorflow.keras.layers import Conv1DTranspose in_w = 32 in_ch = 3 kernel = 32 ker_w = 3 model = Sequential( Conv1DTranspose(kernel, (ker_w,), padding="same", input_shape=(in_w, in_ch)) ) flops = get_flops(model, batch_size=1) assert flops == ((2 * ker_w * in_ch) + 1) * in_w * kernel + 1
def __init__(self, _latent_dim=10, _input_shape=(100, 5, 1), _encoder_filters=[32, 64], _encoder_kernel_sizes=[5, 5], _encoder_strides=[1, 1], _decoder_filters=[64, 32], _decoder_kernel_sizes=[5, 5], _decoder_strides=[1, 1], _encoder_use_bias=True, _decoder_use_bias=True): super(VAE, self).__init__() # Shared Hyper Parameters self.latent_dim = _latent_dim self.leaky_relu = LeakyReLU(alpha=0.5) self.weight_initializer = RandomNormal(mean=0.0, stddev=0.02) self.lamb = 0.00005 # Encoder Hyper Parameters self.encoder_filters = _encoder_filters self.encoder_kernel_sizes = _encoder_kernel_sizes self.encoder_strides = _encoder_strides self.encoder_use_bias = _encoder_use_bias # Decoder Hyper Parameters self.decoder_filters = _decoder_filters self.decoder_kernel_sizes = _decoder_kernel_sizes self.decoder_strides = _decoder_strides self.decoder_use_bias = _decoder_use_bias self.optimizer = tf.keras.optimizers.Adam(1e-4) # Encoder self.encoder = tf.keras.Sequential([ Conv1D(filters=self.encoder_filters[0], kernel_size=self.encoder_kernel_sizes[0], strides=self.encoder_strides[0], padding='SAME', use_bias=self.encoder_use_bias, kernel_initializer=self.weight_initializer, input_shape=_input_shape), self.leaky_relu, Conv1D(filters=self.encoder_filters[1], kernel_size=self.encoder_kernel_sizes[1], strides=self.encoder_strides[1], padding='SAME', use_bias=self.encoder_use_bias, kernel_initializer=self.weight_initializer), self.leaky_relu, Flatten(), Dense(self.latent_dim + self.latent_dim), ]) # Decoder self.decoder = tf.keras.Sequential([ InputLayer(input_shape=(self.latent_dim, )), Dense(units=100 * 64), self.leaky_relu, Reshape(target_shape=(100, 64)), Conv1DTranspose(filters=self.decoder_filters[0], kernel_size=self.decoder_kernel_sizes[0], strides=self.decoder_strides[0], padding='SAME', use_bias=self.decoder_use_bias, kernel_initializer=self.weight_initializer), self.leaky_relu, Conv1DTranspose(filters=self.decoder_filters[1], kernel_size=self.decoder_kernel_sizes[1], strides=self.decoder_strides[1], padding='SAME', use_bias=self.decoder_use_bias, kernel_initializer=self.weight_initializer), self.leaky_relu, Conv1DTranspose(filters=5, kernel_size=1, strides=1, padding='SAME', activation=tf.nn.tanh) ])
def __init__( self, sequence_length=30, number_of_vars=10, fil1_conv_enc=16, fil2_conv_enc=32, unit_den_enc=16, fil1_conv_dec=32, fil2_conv_dec=16, kernel=3, strs=2, drop_rate=0.2, batch_size=100, latent_dim=1, epochs=100, learning_rate=1e-2, decay_rate=0.96, decay_step=1000, ): # reparameterization trick # instead of sampling from Q(z|X), sample epsilon = N(0,I) # z = z_mean + sqrt(var) * epsilon def sampling(args): """Reparameterization trick by sampling from an isotropic unit Gaussian. # Arguments args (tensor): mean and log of variance of Q(z|X) # Returns z (tensor): sampled latent vector """ z_mean, z_log_var = args batch = K.shape(z_mean)[0] dim = K.int_shape(z_mean)[1] # by default, random_normal has mean = 0 and std = 1.0 epsilon = K.random_normal(shape=(batch, dim)) return z_mean + K.exp(0.5 * z_log_var) * epsilon # network parameters input_shape = (sequence_length, number_of_vars) self.batch_size = batch_size self.epochs = epochs # VAE model = encoder + decoder # build encoder model inputs = Input(shape=input_shape, name='enc_input') h_enc = Conv1D(fil1_conv_enc, kernel, activation='tanh', strides=strs, padding="same", name='enc_conv1d_1')(inputs) h_enc = Dropout(drop_rate)(h_enc) h_enc = Conv1D(fil2_conv_enc, kernel, activation='tanh', strides=strs, padding="same", name='enc_conv1d_2')(h_enc) h_enc = Flatten(name='enc_flatten')(h_enc) h_enc = Dense(unit_den_enc, activation='tanh', name='enc_output')(h_enc) # reparameterization trick z_mean = Dense(latent_dim, name='z_mean')(h_enc) z_log_var = Dense(latent_dim, name='z_log_var')(h_enc) # use reparameterization trick to push the sampling out as input # note that "output_shape" isn't necessary with the TensorFlow backend z = Lambda(sampling, output_shape=(latent_dim, ), name='z')([z_mean, z_log_var]) # instantiate encoder model encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder') encoder.summary() # build decoder model reduce = sequence_length // (strs * 2) latent_inputs = Input(shape=(latent_dim, ), name='z_sampling') h_dec = Dense(reduce * fil1_conv_dec, activation='tanh', name='dec_input')(latent_inputs) h_dec = Reshape((reduce, fil1_conv_dec), name='dec_reshape')(h_dec) h_dec = Conv1DTranspose(fil1_conv_dec, kernel, activation='tanh', strides=strs, padding="same", name='dec_conv1d_1')(h_dec) h_dec = Dropout(drop_rate)(h_dec) h_dec = Conv1DTranspose(fil2_conv_dec, kernel, activation='tanh', strides=strs, padding="same", name='dec_conv1d_2')(h_dec) outputs = Conv1DTranspose(number_of_vars, kernel, activation=None, padding="same", name='dec_conv1d_output')(h_dec) # instantiate decoder model decoder = Model(latent_inputs, outputs, name='decoder') decoder.summary() # instantiate VAE model outputs = decoder(encoder(inputs)[2]) self.vae = Model(inputs, outputs, name='vae_mlp') reconstruction_loss = tf.reduce_mean(mse(inputs, outputs)) reconstruction_loss *= (sequence_length * number_of_vars) kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var) kl_loss = K.sum(kl_loss, axis=-1) kl_loss *= -0.5 vae_loss = K.mean(reconstruction_loss + kl_loss) self.vae.add_loss(vae_loss) lr_schedule = optimizers.schedules.ExponentialDecay( learning_rate, decay_steps=decay_step, decay_rate=decay_rate, staircase=True, ) opt = optimizers.Adam(learning_rate=lr_schedule) self.vae.compile(optimizer=opt)
def get_generator(config, vocab): # Input layer model_input = tf.keras.layers.Input(shape=(512,21)) # Parameters n_layers = config['n_layers'] down_sample = config['down_sample'] l1 = config['l1'] l2 = config['l2'] atte_loc = config['attention_loc'] use_atte = config['use_attention'] filters = config['filters'] kernels = config['kernels'] dilation = config['dilations'] use_spectral_norm = config['use_spectral_norm'] norm=config['norm'] use_gumbel = config['use_gumbel'] filters_down = [128, 256] filters_up = [256, 128] print(filters) print(atte_loc) # Sampling if use_gumbel: gms = GumbelSoftmax(temperature = 1) else: gms = Softmax() if use_spectral_norm: projection = SpectralNormalization(Conv1D(64,9, padding='same', activation = 'relu', use_bias = True ,kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2))) att = SelfAttentionSN(filters[atte_loc]) down = [SpectralNormalization(Conv1D(filters_down[i], 9, strides=2, padding="same", activation = "relu", kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2))) for i in range(down_sample)] up = [SpectralNormalization(Conv1DTranspose(filters_up[i], 9, strides=2, padding="same", activation = "relu", kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2))) for i in range(down_sample)] outconv1 = SpectralNormalization(Conv1D(filters_up[-1],9,padding = 'same', activation = "relu")) outconv2 = SpectralNormalization(Conv1D(vocab, 9, padding = 'same', activation = gms)) else: projection = Conv1D(64,9, padding='same', activation = 'relu', use_bias = True ,kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2)) att = SelfAttention(filters[atte_loc]) down = [Conv1D(filters_down[i], 9, strides=2, padding="same", activation = "relu", kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2)) for i in range(down_sample)] up = [Conv1DTranspose(filters_up[i], 9, strides=2, padding="same", activation = "relu", kernel_regularizer = tf.keras.regularizers.L1L2(l1=l1, l2=l2)) for i in range(down_sample)] outconv1 = Conv1D(filters_up[-1],9,padding = 'same', activation = "relu") outconv2 = Conv1D(vocab, 9, padding = 'same', activation = None) res = [residual_mod(filters[i], kernels[i], dilation=dilation[i], l1=l1, l2=l2, use_dout = False, use_bias=True, norm=norm, sn = use_spectral_norm, act="ReLU") for i in range(n_layers)] # Normalisations if norm == "Layer": norm_up_down = [LayerNormalization(axis = -1, epsilon = 1e-6) for i in range(down_sample*2+1)] elif norm == "Batch": norm_up_down = [BatchNormalization() for i in range(down_sample*2+1)] elif norm == "Instance": norm_up_down = [tfa.layers.InstanceNormalization() for i in range(down_sample*2+1)] else: norm_up_down = [linear for i in range(down_sample*2+1)] skip = [] x = model_input x = projection(x) x = norm_up_down[0](x) x = down[0](x) x = norm_up_down[1](x) skip.append(x) x = down[1](x) x = norm_up_down[2](x) skip.append(x) for i in range(n_layers): x_out = res[i](x) x = tf.keras.layers.Add()([x_out, x]) if i == atte_loc and use_atte: x = att(x)[0] x = tf.keras.layers.Concatenate()([skip[1], x]) x = up[0](x) x = norm_up_down[3](x) x = tf.keras.layers.Concatenate()([skip[0], x]) x = up[1](x) x = norm_up_down[4](x) x = outconv1(x) x = outconv2(x) x = gms(x) model = tf.keras.Model(inputs=model_input, outputs=x) return model
def my_net(k = 1, k_size = 3, num_classes = 6, input_shape= (30, 300)): img_input = Input(input_shape) #1 x = Conv1D(64 * k , k_size, padding='same')(img_input) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(64 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling1D()(x) #2 x = Conv1D(128 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(128 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling1D()(x) #3 x = Conv1D(256 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(256 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(256 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling1D()(x) #4 x = Conv1D(512 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(512 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(512 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) # UP 2 x = Conv1DTranspose(256, kernel_size=2, strides=2, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(256 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(256 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) # UP 3 x = Conv1DTranspose(128, kernel_size=2, strides=2, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(128 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(128 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) # UP 4 x = Conv1DTranspose(64, kernel_size=2, strides=2, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(64 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(64 * k , k_size, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(num_classes, k_size, activation='sigmoid', padding='same')(x) model = Model(img_input, x) model = Model(img_input, x) model.compile(optimizer=Adam(0.0025), loss='categorical_crossentropy', metrics=[dice_coef]) return model
def build_1d_unet(signal_length, channels): inputs = Input((signal_length, channels)) c0 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(inputs) c0 = BatchNormalization()(c0) c0 = Dropout(0.1)(c0) c0 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c0) c0 = BatchNormalization()(c0) p0 = MaxPooling1D(2)(c0) c1 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(p0) c1 = BatchNormalization()(c1) c1 = Dropout(0.1)(c1) c1 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c1) c1 = BatchNormalization()(c1) p1 = MaxPooling1D(2)(c1) c2 = Conv1D(64, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(p1) c2 = BatchNormalization()(c2) c2 = Dropout(0.1)(c2) c2 = Conv1D(64, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c2) c2 = BatchNormalization()(c2) p2 = MaxPooling1D(2)(c2) c3 = Conv1D(128, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(p2) c3 = BatchNormalization()(c3) c3 = Dropout(0.2)(c3) c3 = Conv1D(128, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c3) c3 = BatchNormalization()(c3) p3 = MaxPooling1D(2)(c3) c4 = Conv1D(256, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(p3) c4 = BatchNormalization()(c4) c4 = Dropout(0.2)(c4) c4 = Conv1D(256, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c4) c4 = BatchNormalization()(c4) p4 = MaxPooling1D(2)(c4) c5 = Conv1D(512, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(p4) c5 = BatchNormalization()(c5) c5 = Dropout(0.3)(c5) c5 = Conv1D(512, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c5) c5 = BatchNormalization()(c5) u6 = Conv1DTranspose(128, (2,), strides=(2,), padding='same')(c5) u6 = check_and_add_padding(u6, c4) u6 = concatenate([u6, c4]) c6 = Conv1D(256, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(u6) c6 = BatchNormalization()(c6) c6 = Dropout(0.2)(c6) c6 = Conv1D(256, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c6) c6 = BatchNormalization()(c6) u7 = Conv1DTranspose(64, (2,), strides=(2,), padding='same')(c6) u7 = check_and_add_padding(u7, c3) u7 = concatenate([u7, c3]) c7 = Conv1D(128, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(u7) c7 = BatchNormalization()(c7) c7 = Dropout(0.2)(c7) c7 = Conv1D(128, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c7) c7 = BatchNormalization()(c7) u8 = Conv1DTranspose(32, (2,), strides=(2,), padding='same')(c7) u8 = check_and_add_padding(u8, c2) u8 = concatenate([u8, c2]) c8 = Conv1D(64, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(u8) c8 = BatchNormalization()(c8) c8 = Dropout(0.1)(c8) c8 = Conv1D(64, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c8) c8 = BatchNormalization()(c8) u9 = Conv1DTranspose(16, (2,), strides=(2,), padding='same')(c8) u9 = check_and_add_padding(u9, c1) u9 = concatenate([u9, c1]) c9 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(u9) c9 = BatchNormalization()(c9) c9 = Dropout(0.1)(c9) c9 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c9) c9 = BatchNormalization()(c9) u10 = Conv1DTranspose(16, (2,), strides=(2,), padding='same')(c9) u10 = check_and_add_padding(u10, c0) u10 = concatenate([u10, c0]) c10 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(u10) c10 = BatchNormalization()(c10) c10 = Dropout(0.1)(c10) c10 = Conv1D(32, (3,), activation='relu', kernel_initializer='he_normal', padding='same')(c10) c10 = BatchNormalization()(c10) outputs = Conv1D(1, (1,), activation='sigmoid')(c10) return Model(inputs=[inputs], outputs=[outputs])
def __init__(self, sequence_length=30, number_of_vars=1, hidden_units_e=100, hidden_units_d=100, batch_size=100, latent_dim=1, epochs=100, learning_rate=1e-2, decay_rate=0.96, decay_step=1000): # reparameterization trick # instead of sampling from Q(z|X), sample epsilon = N(0,I) # z = z_mean + sqrt(var) * epsilon def sampling(args): """Reparameterization trick by sampling from an isotropic unit Gaussian. # Arguments args (tensor): mean and log of variance of Q(z|X) # Returns z (tensor): sampled latent vector """ z_mean, z_log_var = args batch = K.shape(z_mean)[0] dim = K.int_shape(z_mean)[1] # by default, random_normal has mean = 0 and std = 1.0 epsilon = K.random_normal(shape=(batch, dim)) return z_mean + K.exp(0.5 * z_log_var) * epsilon # network parameters self.sequence_length = sequence_length self.number_of_vars = number_of_vars self.learning_rate = learning_rate self.decay_rate = decay_rate self.decay_step = decay_step self.input_shape = (sequence_length, number_of_vars) self.hidden_units_e = hidden_units_e self.hidden_units_d = hidden_units_d self.batch_size = batch_size self.latent_dim = latent_dim self.epochs = epochs # VAE model = encoder + decoder # build encoder model self.__inputs = Input(shape=self.input_shape, name='enc_input') self.__inputs_h_enc = Conv1D(16, 3, activation='tanh', strides=2, padding="same", name='enc_conv1d_1')(self.__inputs) self.__inputs_h_enc = Dropout(0.2)(self.__inputs_h_enc) self.__inputs_h_enc = Conv1D(32, 3, activation='tanh', strides=2, padding="same", name='enc_conv1d_2')(self.__inputs_h_enc) self.__inputs_flatten = Flatten(name='enc_flatten')( self.__inputs_h_enc) self.__h_enc = Dense(self.hidden_units_e, activation='tanh', name='enc_output')(self.__inputs_flatten) # reparameterization trick self.__z_mean = Dense(self.latent_dim, name='z_mean')(self.__h_enc) self.__z_log_var = Dense(self.latent_dim, name='z_log_var')(self.__h_enc) # use reparameterization trick to push the sampling out as input # note that "output_shape" isn't necessary with the TensorFlow backend self.__z = Lambda(sampling, output_shape=(self.latent_dim, ), name='z')([self.__z_mean, self.__z_log_var]) # instantiate encoder model encoder = Model(self.__inputs, [self.__z_mean, self.__z_log_var, self.__z], name='encoder') logger.info('Encoder. \n %s', encoder.summary()) # build decoder model self.__latent_inputs = Input(shape=(self.latent_dim, ), name='z_sampling') self.__h_z = Dense(60 * 32, activation='tanh', name='dec_input')(self.__latent_inputs) self.__h_z = Reshape((60, 32), name='dec_reshape')(self.__h_z) self.__h_z = Conv1DTranspose(32, 3, activation='tanh', strides=2, padding="same", name='dec_conv1d_1')(self.__h_z) self.__inputs_h_enc = Dropout(0.2)(self.__h_z) self.__h_z = Conv1DTranspose(16, 3, activation='tanh', strides=2, padding="same", name='dec_conv1d_2')(self.__h_z) self.__outputs = Conv1DTranspose(self.number_of_vars, 3, activation=None, padding="same", name='dec_conv1d_output')(self.__h_z) #self.__x_recons_flatten = Dense(sequence_length * number_of_vars)(self.__h_z) #self.__outputs = Reshape((sequence_length, number_of_vars))(self.__x_recons_flatten) # instantiate decoder model decoder = Model(self.__latent_inputs, self.__outputs, name='decoder') logger.info('Decoder. \n %s', decoder.summary()) # instantiate VAE model self.__outputs = decoder(encoder(self.__inputs)[2]) self.vae = Model(self.__inputs, self.__outputs, name='vae_mlp')