def build_decoder(self, latent_dim, X_shape): """ Build the decoder """ latent_inputs = keras.Input(shape=(latent_dim, )) x = latent_inputs x = layers.Dense(16)(x) x = layers.Reshape((8, 2))(x) x = layers.Conv1DTranspose(filters=4, kernel_size=7, padding="same", strides=4, activation="relu")(x) x = layers.Conv1DTranspose(filters=8, kernel_size=7, padding="same", strides=4, activation="relu")(x) x = layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same", strides=2)(x) decoder_outputs = x decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder") decoder.summary() return decoder
def make_generator(latent_dim = 100, audio_input_dim = 16384, dim = 64, kernel_len = 25, shape_before_flatten = (None, 16, 1024)): generator_input = layers.Input(shape = (latent_dim,)) x = layers.Dense(audio_input_dim)(generator_input) x = layers.Reshape((shape_before_flatten[1], shape_before_flatten[2]))(x) x = layers.ReLU()(x) x = layers.Conv1DTranspose(dim * 8, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv4')(x) x = layers.ReLU()(x) x = layers.Conv1DTranspose(dim * 4, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv3')(x) x = layers.ReLU()(x) x = layers.Conv1DTranspose(dim * 2, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv2')(x) x = layers.ReLU()(x) x = layers.Conv1DTranspose(dim, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv1')(x) x = layers.ReLU()(x) x = layers.Conv1DTranspose(1, kernel_size = kernel_len, strides=4, padding = 'same', activation = 'tanh', name = 'deconv0')(x) generator_output = x generator = keras.Model(generator_input, generator_output, name = 'generator') return generator
def get_model(): global x_train model = keras.Sequential([ layers.Input(shape=(x_train.shape[1], x_train.shape[2])), layers.Conv1D(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1D(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1DTranspose(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same"), ]) model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss="mse") model.summary() return model
def construct_decoder(latent_dim, num_features): latent_inputs = keras.Input(shape=(latent_dim,)) x = layers.Dense(32 * 64, activation="relu")(latent_inputs) x = layers.Reshape((32, 64))(x) x = layers.Conv1DTranspose(64, 3, activation="relu", strides=2, padding='same')(x) x = layers.Conv1DTranspose(32, 3, activation="relu", strides=2, padding='same')(x) x = layers.Conv1DTranspose(1, 3, activation='relu', padding='same')(x) x = layers.Flatten()(x) x = layers.Dense(num_features)(x) decoder_outputs = layers.Reshape((num_features, 1))(x) decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder") return decoder
def _define_model(self, ): input_series = keras.Input(shape=(self.sequence_length, self.num_features)) x = layers.Conv1D(filters=32, kernel_size=7, padding="same", strides=2, activation=self.activation)(input_series) x = layers.Dropout(rate=0.2)(x) encoded = layers.Conv1D(filters=16, kernel_size=7, padding="same", strides=2, activation=self.activation)(x) encoder = keras.Model(input_series, encoded) x = layers.Conv1DTranspose(filters=16, kernel_size=7, padding="same", strides=2, activation=self.activation)(encoded) x = layers.Dropout(rate=0.2)(x) x = layers.Conv1DTranspose(filters=32, kernel_size=7, padding="same", strides=2, activation=self.activation)(x) decoded = layers.Conv1DTranspose(filters=self.num_features, kernel_size=7, padding="same")(x) autoencoder = keras.Model(input_series, decoded) optimizer = keras.optimizers.Adam(learning_rate=self.learning_rate, clipnorm=1.0, clipvalue=0.5) autoencoder.compile(optimizer=optimizer, loss=self.loss) self.model = autoencoder self.encoder = encoder self.model.summary()
def buildAutoencoderModel(self, train, filters=[32, 16], kernel_size=7, padding='same', strides=2, activation='relu', rate=0.2, learning_rate=0.001, loss='mse'): model = keras.Sequential([ layers.Input(shape=(train.shape[1], train.shape[2])), layers.Conv1D(filters=filters[0], kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), layers.Dropout(rate=rate), layers.Conv1D(filters=filters[1], kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), layers.Conv1DTranspose(filters=filters[1], kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), layers.Dropout(rate=0.2), layers.Conv1DTranspose(filters=filters[0], kernel_size=kernel_size, padding=padding, strides=strides, activation=activation), layers.Conv1DTranspose(filters=1, kernel_size=kernel_size, padding=padding), ]) model.compile( optimizer=keras.optimizers.Adam(learning_rate=learning_rate), loss=loss) return model
def conv_block(input, conv_dim, upsampling_factor): # Dilated convolutional block with weight normalizaton. conv_t = addon_layers.WeightNormalization( layers.Conv1DTranspose(conv_dim, 16, upsampling_factor, padding="same"), data_init=False, )(input) lrelu1 = layers.LeakyReLU()(conv_t) res_stack = residual_stack(lrelu1, conv_dim) lrelu2 = layers.LeakyReLU()(res_stack) return lrelu2
def build_decoder(self): latent_inputs = layers.Input(shape=(self.latent_dim,), name="decoder_input") decoder_dense_layer1 = layers.Dense(units=75 * 32, name="decoder_dense_1")( latent_inputs) decoder_reshape = layers.Reshape(target_shape=(75, 32))(decoder_dense_layer1) decoder_conv_tran_layer1 = layers.Conv1DTranspose(filters=32, kernel_size=8, padding="same", strides=1, name="decoder_conv_tran_1")(decoder_reshape) decoder_norm_layer1 = layers.BatchNormalization(name="decoder_norm_1")(decoder_conv_tran_layer1) decoder_activ_layer1 = layers.LeakyReLU(name="decoder_leakyrelu_1")(decoder_norm_layer1) decoder_conv_tran_layer2 = layers.Conv1DTranspose(filters=64, kernel_size=8, padding="same", strides=2, name="decoder_conv_tran_2")(decoder_activ_layer1) decoder_norm_layer2 = layers.BatchNormalization(name="decoder_norm_2")(decoder_conv_tran_layer2) decoder_activ_layer2 = layers.LeakyReLU(name="decoder_leakyrelu_2")(decoder_norm_layer2) decoder_conv_tran_layer3 = layers.Conv1DTranspose(filters=128, kernel_size=8, padding="same", strides=1, name="decoder_conv_tran_3")(decoder_activ_layer2) decoder_norm_layer3 = keras.layers.BatchNormalization(name="decoder_norm_3")(decoder_conv_tran_layer3) decoder_activ_layer3 = keras.layers.LeakyReLU(name="decoder_leakyrelu_3")(decoder_norm_layer3) decoder_conv_tran_layer4 = keras.layers.Conv1DTranspose(filters=1, kernel_size=16, padding="same", strides=2, name="decoder_conv_tran_4")(decoder_activ_layer3) decoder_output = keras.layers.LeakyReLU(name="decoder_output")(decoder_conv_tran_layer4) decoder = keras.models.Model(latent_inputs, decoder_output, name="decoder_model") return decoder
def conv_block(input, conv_dim, upsampling_factor): """Dilated Convolutional Block with weight normalization. Args: conv_dim: int, determines filter size for the block. upsampling_factor: int, scale for upsampling. Returns: Dilated convolution block. """ conv_t = addon_layers.WeightNormalization( layers.Conv1DTranspose(conv_dim, 16, upsampling_factor, padding="same"), data_init=False, )(input) lrelu1 = layers.LeakyReLU()(conv_t) res_stack = residual_stack(lrelu1, conv_dim) lrelu2 = layers.LeakyReLU()(res_stack) return lrelu2
model = keras.Sequential([ layers.Input(shape=(x_train.shape[1], x_train.shape[2])), layers.Conv1D(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1D(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1DTranspose(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same"), ]) model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss="mse") model.summary() """ ## Train the model
for param in PARAMS_TO_INFER: pre_pred = layers.Dense(fily, activation="relu")(z_mean) y_predictions.append( layers.Dense(1, name='{}_pred'.format(param))(pre_pred)) encoder = keras.Model([encoder_inputs] + param_inputs, [z_mean, z_log_var, z] + y_predictions, name='encoder') latent_inputs = keras.Input(shape=(latent_dim, ), name='z_sampling') x = layers.Dense(np.prod(intermediate_shape), activation="relu")(latent_inputs) x = layers.Reshape(intermediate_shape)(x) x = layers.Conv1DTranspose(fil2, 10, activation="relu", strides=1, padding="same")(x) x = layers.UpSampling1D(size=2)(x) x = layers.Conv1DTranspose(fil1, 3, activation="relu", strides=1, padding="same")(x) x = layers.UpSampling1D(size=2)(x) decoder_outputs = layers.Conv1DTranspose(2, 3, activation="sigmoid", padding="same")(x) decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder")
def _generator(self): model = tf.keras.Sequential() dims = [self.gen_input_dim] + self.hidden_dim if self.use_convolutions: model.add( layers.Dense( 24 * 256, use_bias=True, input_shape=(self.gen_input_dim, ), kernel_initializer=tf.keras.initializers.GlorotNormal( seed=self.seed))) model.add(layers.BatchNormalization()) model.add(layers.ReLU()) model.add(layers.Reshape((24, 256))) model.add( layers.Conv1DTranspose( 128, (13, ), (2, ), padding='same', use_bias=False, kernel_initializer=tf.keras.initializers.GlorotNormal( seed=self.seed))) model.add(layers.BatchNormalization()) model.add(layers.ReLU()) model.add( layers.Conv1DTranspose( 64, (13, ), (2, ), padding='same', use_bias=False, kernel_initializer=tf.keras.initializers.GlorotNormal( seed=self.seed))) model.add(layers.BatchNormalization()) model.add(layers.ReLU()) # model.add(layers.Conv1DTranspose(64, # (13,), # (2,), # padding='same', # use_bias=False, # kernel_initializer=tf.keras.initializers.GlorotNormal( # seed=self.seed))) # model.add(layers.BatchNormalization()) # model.add(layers.ReLU()) model.add( layers.Conv1DTranspose(1, (13, ), (1, ), use_bias=False, activation='sigmoid')) assert model.output_shape == (None, self.input_dim, 1), print(model.output_shape) else: for i in range(len(dims) - 1): model.add( layers.Dense( dims[i + 1], use_bias=True, input_shape=(dims[i], ), kernel_initializer=tf.keras.initializers.GlorotNormal( seed=self.seed))) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU()) model.add( layers.Dense( self.input_dim, use_bias=True, input_shape=(dims[-1], ), kernel_initializer=tf.keras.initializers.GlorotNormal( seed=self.seed))) model.add(layers.Activation(tf.nn.sigmoid)) # model.add(layers.Activation(tf.nn.tanh)) return model
class Demucs(keras.Model): """ Demucs speech enhancement model. Args: - chin (int): number of input channels. - chout (int): number of output channels. - hidden (int): number of initial hidden channels. - depth (int): number of layers. - kernel_size (int): kernel size for each layer. - stride (int): stride for each layer. - causal (bool): if false, uses BiLSTM instead of LSTM. - resample (int): amount of resampling to apply to the input/output. Can be one of 1, 2 or 4. - growth (float): number of channels is multiplied by this for every layer. - max_hidden (int): maximum number of channels. Can be useful to control the size/speed of the model. - normalize (bool): if true, normalize the input. - glu (bool): if true uses GLU instead of ReLU in 1x1 convolutions. - rescale (float): controls custom weight initialization. See https://arxiv.org/abs/1911.13254. - floor (float): stability flooring when normalizing. """ @capture_init def __init__(self, chin=1, chout=1, hidden=48, depth=5, kernel_size=8, stride=4, causal=True, resample=4, growth=2, max_hidden=10_000, normalize=True, glu=True, rescale=0.1, floor=1e-3, name='demucs', **kwargs): super().__init__(name=name, **kwargs) if resample not in [1, 2, 4]: raise ValueError("Resample should be 1, 2 or 4.") self.chin = chin self.chout = chout self.hidden = hidden self.depth = depth self.kernel_size = kernel_size self.stride = stride self.causal = causal self.resample = resample self.length_calc = LengthCalc(depth, kernel_size, stride) if resample == 2: self.upsample = Upsample2() self.downsample = Downsample2() elif resample == 4: self.upsample = keras.Sequential([Upsample2(), Upsample2()], name="upsample4") self.downsample = keras.Sequential([Downsample2(), Downsample2()], name="downsample4") else: self.upsample = None self.downsample = None if normalize: self.normalize = Normalize(floor) self.encoder = [] self.decoder = [] if rescale: initializer = initializers.RandomNormal(stddev=rescale) else: initializer = initializers.GlorotNormal() for index in range(depth): # Encode layer: chin -> hidden encode = keras.Sequential(name=f"Encode_{index+1}") encode.add(layers.Conv1D(hidden, kernel_size, strides=stride, activation='relu', input_shape=(None,chin), kernel_initializer=initializer)) if glu: encode.add(layers.Conv1D(hidden*2, 1, kernel_initializer=initializer)) encode.add(GLU()) else: encode.add(layers.Conv1D(hidden, 1, activation='relu', kernel_initializer=initializer)) self.encoder.append(encode) # Decode layer: hidden -> chout decode = keras.Sequential(name=f"Decode_{index+1}") if glu: decode.add(layers.Conv1D(hidden*2, 1, kernel_initializer=initializer)) decode.add(GLU()) else: decode.add(layers.Conv1D(hidden, 1, activation='relu', kernel_initializer=initializer)) decode.add(layers.Conv1DTranspose(chout, kernel_size, strides=stride, kernel_initializer=initializer)) self.decoder.insert(0, decode) # Update chin, chout, hidden for next depth, growing hidden by growth chout = hidden chin = hidden hidden = min(int(growth * hidden), max_hidden) self.lstm = BLSTM(bi=not causal)
def main(): # Load the data. Use the Numenta Anomaly Benchmark (NAB) dataset. # It provides artificial timeseries data containing labeled # anomalous periods of behavior. Data rae ordered, timestamped, # single-valued metrics. Use the art_daily_small_noise.csv file for # training and the art_daily_jumpsup.csv file for testing. The # simplicity of this dataset allows us to demonstrate anomaly # detection effectively. master_url_root = "https://raw.githubusercontent.com/numenta/NAB/master/data/" df_small_noise_url_suffix = "artificialNoAnomaly/art_daily_small_noise.csv" df_small_noise_url = master_url_root + df_small_noise_url_suffix df_small_noise = pd.read_csv(df_small_noise_url, parse_dates=True, index_col="timestamp") df_daily_jumpsup_url_suffix = "artificialWithAnomaly/art_daily_jumpsup.csv" df_daily_jumpsup_url = master_url_root + df_daily_jumpsup_url_suffix df_daily_jumpsup = pd.read_csv(df_daily_jumpsup_url, parse_dates=True, index_col="timestamp") # Quick look at the data. print(df_small_noise.head()) print(df_daily_jumpsup.head()) # Visualize the data. Timeseries data without anomalies will be # used for training. ''' fig, ax = plt.subplots() df_small_noise.plot(legend=False, ax=ax) plt.show() ''' # Timeseries data with anomalies will be used for testing and see # if the sudden jump up in the data is detected as an anomaly. ''' fig, ax = plt.subplots() df_daily_jumpsup.plot(legend=False, ax=ax) plt.show() ''' # Prepare training data. Get data values from the training # timeseries data file and normalize the value data. There is a # value for every 5 mins for 14 days. # 24 * 60 / 5 = 288timesteps per day # 288 * 14 = 4032 data points in total # Normalize and save the mean and std. training_mean = df_small_noise.mean() training_std = df_small_noise.std() df_training_value = (df_small_noise - training_mean) / training_std print("Number of training samples:", len(df_training_value)) # Create sequences. Create sequences combining TIME_STEPS # contiguous data values from the training data. TIME_STEPS = 288 x_train = create_sequences(df_training_value.values, TIME_STEPS) print("Training input shape: ", x_train.shape) # Build a model. Build a convolutional reconstruction autoencoder # model. The model will take input of shape (batch_size, # sequence_length, num_features) and return output of the same # shape. In this case, sequence_length is 288 and num_features is # 1. model = keras.Sequential([ layers.Input(shape=(x_train.shape[1], x_train.shape[2])), layers.Conv1D(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1D(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1DTranspose(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same"), ]) model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss="mse") model.summary() # Train the model. Note that using x_train as both the input and # the target since this is a reconstruction model. history = model.fit( x_train, x_train, epochs=50, batch_size=128, validation_split=0.1, callbacks=[ keras.callbacks.EarlyStopping(monitor="val_loss", patience=5, mode="min") ], ) # Plot training and validation loss to see how training went. ''' plt.plot(history.history["loss"], label="Training Loss") plt.plot(history.history["val_loss"], label="Validation Loss") plt.legend() plt.show() ''' # Detecting anomalies. Detect anomalies by determining how well the # model can reconstruct the input data. # 1) Find MAE loss on training samples. # 2) Find max MAE loss value. This is the worst the model has # performed trying to reconstruct a sample. This will be the # threshold for anomaly detection. # 3) If the reconstruction loss for a sample is greater than this # threshold value then it can be infered that the model is seeing # a patter that it isn't familiar with. This sample will be # labeled an anomaly. # Get train MAE loss. x_train_pred = model.predict(x_train) train_mae_loss = np.mean(np.abs(x_train_pred - x_train), axis=1) ''' plt.hist(train_mae_loss, bins=50) plt.xlabel("Train MAE loss") plt.ylabel("No of samples") plt.show() ''' # Get reconstruction of loss threshold. threshold = np.max(train_mae_loss) print("Reconstruction error threshold: ", threshold) # Compare reconstruction. Just for fun, see how the model has # reconstructed the first sample. This is the 288 timesteps from # day 1 of the training dataset. # Checking how the first sequence is learned. ''' plt.plot(x_train[0]) plt.plot(x_train_pred[0]) plt.show() ''' # Prepare test data. def_test_value = (df_daily_jumpsup - training_mean) / training_std ''' fig, ax = plt.subplots() def_test_value.plot(legend=False, ax=ax) plt.show() ''' # Create sequences from test values. x_test = create_sequences(def_test_value.values, TIME_STEPS) print("Test input shape: ", x_test.shape) # Get test MAE loss. x_test_pred = model.predict(x_test) test_mae_loss = np.mean(np.abs(x_test_pred - x_test), axis=1) test_mae_loss = test_mae_loss.reshape((-1)) ''' plt.hist(test_mae_loss, bins=50) plt.xlabel("test MAE loss") plt.ylabel("No of samples") plt.show() ''' # Detect all the samples which are anomalies. anomalies = test_mae_loss > threshold print("Number of anomaly samples: ", np.sum(anomalies)) print("Indices of anomaly samples: ", np.where(anomalies)) # Plot anomalies. Now that it is known which samples of the data # are anomalies, find the corresponding timestamps from the # original test data. anomalous_data_indices = [] for data_idx in range(TIME_STEPS - 1, len(def_test_value) - TIME_STEPS + 1): if np.all(anomalies[data_idx - TIME_STEPS + 1:data_idx]): anomalous_data_indices.append(data_idx) # Overlay the anomalies on the original test data plot. df_subset = df_daily_jumpsup.iloc[anomalous_data_indices] ''' fig, ax = plt.subplots() df_daily_jumpsup.plot(legend=False, ax=ax) df_subset.plot(legend=False, ax=ax, color="r") plt.show() ''' # Exit the program. exit(0)