def build_decoder(self, latent_dim, X_shape):
"""
Build the decoder
"""
latent_inputs = keras.Input(shape=(latent_dim, ))
x = latent_inputs
x = layers.Dense(16)(x)
x = layers.Reshape((8, 2))(x)
x = layers.Conv1DTranspose(filters=4,
kernel_size=7,
padding="same",
strides=4,
activation="relu")(x)
x = layers.Conv1DTranspose(filters=8,
kernel_size=7,
padding="same",
strides=4,
activation="relu")(x)
x = layers.Conv1DTranspose(filters=1,
kernel_size=7,
padding="same",
strides=2)(x)
decoder_outputs = x
decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder")
decoder.summary()
return decoder
def make_generator(latent_dim = 100,
audio_input_dim = 16384,
dim = 64,
kernel_len = 25,
shape_before_flatten = (None, 16, 1024)):
generator_input = layers.Input(shape = (latent_dim,))
x = layers.Dense(audio_input_dim)(generator_input)
x = layers.Reshape((shape_before_flatten[1], shape_before_flatten[2]))(x)
x = layers.ReLU()(x)
x = layers.Conv1DTranspose(dim * 8, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv4')(x)
x = layers.ReLU()(x)
x = layers.Conv1DTranspose(dim * 4, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv3')(x)
x = layers.ReLU()(x)
x = layers.Conv1DTranspose(dim * 2, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv2')(x)
x = layers.ReLU()(x)
x = layers.Conv1DTranspose(dim, kernel_size = kernel_len, strides=4, padding = 'same', name = 'deconv1')(x)
x = layers.ReLU()(x)
x = layers.Conv1DTranspose(1, kernel_size = kernel_len, strides=4, padding = 'same', activation = 'tanh', name = 'deconv0')(x)
generator_output = x
generator = keras.Model(generator_input, generator_output, name = 'generator')
return generator
def get_model():
global x_train
model = keras.Sequential([
layers.Input(shape=(x_train.shape[1], x_train.shape[2])),
layers.Conv1D(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1D(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1DTranspose(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same"),
])
model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
loss="mse")
model.summary()
return model
def construct_decoder(latent_dim, num_features):
latent_inputs = keras.Input(shape=(latent_dim,))
x = layers.Dense(32 * 64, activation="relu")(latent_inputs)
x = layers.Reshape((32, 64))(x)
x = layers.Conv1DTranspose(64, 3, activation="relu", strides=2, padding='same')(x)
x = layers.Conv1DTranspose(32, 3, activation="relu", strides=2, padding='same')(x)
x = layers.Conv1DTranspose(1, 3, activation='relu', padding='same')(x)
x = layers.Flatten()(x)
x = layers.Dense(num_features)(x)
decoder_outputs = layers.Reshape((num_features, 1))(x)
decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder")
return decoder
def _define_model(self, ):
input_series = keras.Input(shape=(self.sequence_length,
self.num_features))
x = layers.Conv1D(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation=self.activation)(input_series)
x = layers.Dropout(rate=0.2)(x)
encoded = layers.Conv1D(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation=self.activation)(x)
encoder = keras.Model(input_series, encoded)
x = layers.Conv1DTranspose(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation=self.activation)(encoded)
x = layers.Dropout(rate=0.2)(x)
x = layers.Conv1DTranspose(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation=self.activation)(x)
decoded = layers.Conv1DTranspose(filters=self.num_features,
kernel_size=7,
padding="same")(x)
autoencoder = keras.Model(input_series, decoded)
optimizer = keras.optimizers.Adam(learning_rate=self.learning_rate,
clipnorm=1.0,
clipvalue=0.5)
autoencoder.compile(optimizer=optimizer, loss=self.loss)
self.model = autoencoder
self.encoder = encoder
self.model.summary()
def buildAutoencoderModel(self,
train,
filters=[32, 16],
kernel_size=7,
padding='same',
strides=2,
activation='relu',
rate=0.2,
learning_rate=0.001,
loss='mse'):
model = keras.Sequential([
layers.Input(shape=(train.shape[1], train.shape[2])),
layers.Conv1D(filters=filters[0],
kernel_size=kernel_size,
padding=padding,
strides=strides,
activation=activation),
layers.Dropout(rate=rate),
layers.Conv1D(filters=filters[1],
kernel_size=kernel_size,
padding=padding,
strides=strides,
activation=activation),
layers.Conv1DTranspose(filters=filters[1],
kernel_size=kernel_size,
padding=padding,
strides=strides,
activation=activation),
layers.Dropout(rate=0.2),
layers.Conv1DTranspose(filters=filters[0],
kernel_size=kernel_size,
padding=padding,
strides=strides,
activation=activation),
layers.Conv1DTranspose(filters=1,
kernel_size=kernel_size,
padding=padding),
])
model.compile(
optimizer=keras.optimizers.Adam(learning_rate=learning_rate),
loss=loss)
return model
def conv_block(input, conv_dim, upsampling_factor):
# Dilated convolutional block with weight normalizaton.
conv_t = addon_layers.WeightNormalization(
layers.Conv1DTranspose(conv_dim,
16,
upsampling_factor,
padding="same"),
data_init=False,
)(input)
lrelu1 = layers.LeakyReLU()(conv_t)
res_stack = residual_stack(lrelu1, conv_dim)
lrelu2 = layers.LeakyReLU()(res_stack)
return lrelu2
def build_decoder(self):
latent_inputs = layers.Input(shape=(self.latent_dim,), name="decoder_input")
decoder_dense_layer1 = layers.Dense(units=75 * 32, name="decoder_dense_1")(
latent_inputs)
decoder_reshape = layers.Reshape(target_shape=(75, 32))(decoder_dense_layer1)
decoder_conv_tran_layer1 = layers.Conv1DTranspose(filters=32, kernel_size=8, padding="same", strides=1,
name="decoder_conv_tran_1")(decoder_reshape)
decoder_norm_layer1 = layers.BatchNormalization(name="decoder_norm_1")(decoder_conv_tran_layer1)
decoder_activ_layer1 = layers.LeakyReLU(name="decoder_leakyrelu_1")(decoder_norm_layer1)
decoder_conv_tran_layer2 = layers.Conv1DTranspose(filters=64, kernel_size=8, padding="same", strides=2,
name="decoder_conv_tran_2")(decoder_activ_layer1)
decoder_norm_layer2 = layers.BatchNormalization(name="decoder_norm_2")(decoder_conv_tran_layer2)
decoder_activ_layer2 = layers.LeakyReLU(name="decoder_leakyrelu_2")(decoder_norm_layer2)
decoder_conv_tran_layer3 = layers.Conv1DTranspose(filters=128, kernel_size=8, padding="same", strides=1,
name="decoder_conv_tran_3")(decoder_activ_layer2)
decoder_norm_layer3 = keras.layers.BatchNormalization(name="decoder_norm_3")(decoder_conv_tran_layer3)
decoder_activ_layer3 = keras.layers.LeakyReLU(name="decoder_leakyrelu_3")(decoder_norm_layer3)
decoder_conv_tran_layer4 = keras.layers.Conv1DTranspose(filters=1, kernel_size=16, padding="same", strides=2,
name="decoder_conv_tran_4")(decoder_activ_layer3)
decoder_output = keras.layers.LeakyReLU(name="decoder_output")(decoder_conv_tran_layer4)
decoder = keras.models.Model(latent_inputs, decoder_output, name="decoder_model")
return decoder
def conv_block(input, conv_dim, upsampling_factor):
"""Dilated Convolutional Block with weight normalization.
Args:
conv_dim: int, determines filter size for the block.
upsampling_factor: int, scale for upsampling.
Returns:
Dilated convolution block.
"""
conv_t = addon_layers.WeightNormalization(
layers.Conv1DTranspose(conv_dim, 16, upsampling_factor,
padding="same"),
data_init=False,
)(input)
lrelu1 = layers.LeakyReLU()(conv_t)
res_stack = residual_stack(lrelu1, conv_dim)
lrelu2 = layers.LeakyReLU()(res_stack)
return lrelu2
model = keras.Sequential([
layers.Input(shape=(x_train.shape[1], x_train.shape[2])),
layers.Conv1D(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1D(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1DTranspose(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same"),
])
model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss="mse")
model.summary()
"""
## Train the model
for param in PARAMS_TO_INFER:
pre_pred = layers.Dense(fily, activation="relu")(z_mean)
y_predictions.append(
layers.Dense(1, name='{}_pred'.format(param))(pre_pred))
encoder = keras.Model([encoder_inputs] + param_inputs,
[z_mean, z_log_var, z] + y_predictions,
name='encoder')
latent_inputs = keras.Input(shape=(latent_dim, ), name='z_sampling')
x = layers.Dense(np.prod(intermediate_shape),
activation="relu")(latent_inputs)
x = layers.Reshape(intermediate_shape)(x)
x = layers.Conv1DTranspose(fil2,
10,
activation="relu",
strides=1,
padding="same")(x)
x = layers.UpSampling1D(size=2)(x)
x = layers.Conv1DTranspose(fil1,
3,
activation="relu",
strides=1,
padding="same")(x)
x = layers.UpSampling1D(size=2)(x)
decoder_outputs = layers.Conv1DTranspose(2,
3,
activation="sigmoid",
padding="same")(x)
decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder")
def _generator(self):
model = tf.keras.Sequential()
dims = [self.gen_input_dim] + self.hidden_dim
if self.use_convolutions:
model.add(
layers.Dense(
24 * 256,
use_bias=True,
input_shape=(self.gen_input_dim, ),
kernel_initializer=tf.keras.initializers.GlorotNormal(
seed=self.seed)))
model.add(layers.BatchNormalization())
model.add(layers.ReLU())
model.add(layers.Reshape((24, 256)))
model.add(
layers.Conv1DTranspose(
128, (13, ), (2, ),
padding='same',
use_bias=False,
kernel_initializer=tf.keras.initializers.GlorotNormal(
seed=self.seed)))
model.add(layers.BatchNormalization())
model.add(layers.ReLU())
model.add(
layers.Conv1DTranspose(
64, (13, ), (2, ),
padding='same',
use_bias=False,
kernel_initializer=tf.keras.initializers.GlorotNormal(
seed=self.seed)))
model.add(layers.BatchNormalization())
model.add(layers.ReLU())
# model.add(layers.Conv1DTranspose(64,
# (13,),
# (2,),
# padding='same',
# use_bias=False,
# kernel_initializer=tf.keras.initializers.GlorotNormal(
# seed=self.seed)))
# model.add(layers.BatchNormalization())
# model.add(layers.ReLU())
model.add(
layers.Conv1DTranspose(1, (13, ), (1, ),
use_bias=False,
activation='sigmoid'))
assert model.output_shape == (None, self.input_dim,
1), print(model.output_shape)
else:
for i in range(len(dims) - 1):
model.add(
layers.Dense(
dims[i + 1],
use_bias=True,
input_shape=(dims[i], ),
kernel_initializer=tf.keras.initializers.GlorotNormal(
seed=self.seed)))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU())
model.add(
layers.Dense(
self.input_dim,
use_bias=True,
input_shape=(dims[-1], ),
kernel_initializer=tf.keras.initializers.GlorotNormal(
seed=self.seed)))
model.add(layers.Activation(tf.nn.sigmoid))
# model.add(layers.Activation(tf.nn.tanh))
return model
class Demucs(keras.Model):
"""
Demucs speech enhancement model.
Args:
- chin (int): number of input channels.
- chout (int): number of output channels.
- hidden (int): number of initial hidden channels.
- depth (int): number of layers.
- kernel_size (int): kernel size for each layer.
- stride (int): stride for each layer.
- causal (bool): if false, uses BiLSTM instead of LSTM.
- resample (int): amount of resampling to apply to the input/output.
Can be one of 1, 2 or 4.
- growth (float): number of channels is multiplied by this for every layer.
- max_hidden (int): maximum number of channels. Can be useful to
control the size/speed of the model.
- normalize (bool): if true, normalize the input.
- glu (bool): if true uses GLU instead of ReLU in 1x1 convolutions.
- rescale (float): controls custom weight initialization.
See https://arxiv.org/abs/1911.13254.
- floor (float): stability flooring when normalizing.
"""
@capture_init
def __init__(self,
chin=1,
chout=1,
hidden=48,
depth=5,
kernel_size=8,
stride=4,
causal=True,
resample=4,
growth=2,
max_hidden=10_000,
normalize=True,
glu=True,
rescale=0.1,
floor=1e-3,
name='demucs',
**kwargs):
super().__init__(name=name, **kwargs)
if resample not in [1, 2, 4]:
raise ValueError("Resample should be 1, 2 or 4.")
self.chin = chin
self.chout = chout
self.hidden = hidden
self.depth = depth
self.kernel_size = kernel_size
self.stride = stride
self.causal = causal
self.resample = resample
self.length_calc = LengthCalc(depth, kernel_size, stride)
if resample == 2:
self.upsample = Upsample2()
self.downsample = Downsample2()
elif resample == 4:
self.upsample = keras.Sequential([Upsample2(), Upsample2()], name="upsample4")
self.downsample = keras.Sequential([Downsample2(), Downsample2()], name="downsample4")
else:
self.upsample = None
self.downsample = None
if normalize:
self.normalize = Normalize(floor)
self.encoder = []
self.decoder = []
if rescale:
initializer = initializers.RandomNormal(stddev=rescale)
else:
initializer = initializers.GlorotNormal()
for index in range(depth):
# Encode layer: chin -> hidden
encode = keras.Sequential(name=f"Encode_{index+1}")
encode.add(layers.Conv1D(hidden, kernel_size, strides=stride, activation='relu', input_shape=(None,chin), kernel_initializer=initializer))
if glu:
encode.add(layers.Conv1D(hidden*2, 1, kernel_initializer=initializer))
encode.add(GLU())
else:
encode.add(layers.Conv1D(hidden, 1, activation='relu', kernel_initializer=initializer))
self.encoder.append(encode)
# Decode layer: hidden -> chout
decode = keras.Sequential(name=f"Decode_{index+1}")
if glu:
decode.add(layers.Conv1D(hidden*2, 1, kernel_initializer=initializer))
decode.add(GLU())
else:
decode.add(layers.Conv1D(hidden, 1, activation='relu', kernel_initializer=initializer))
decode.add(layers.Conv1DTranspose(chout, kernel_size, strides=stride, kernel_initializer=initializer))
self.decoder.insert(0, decode)
# Update chin, chout, hidden for next depth, growing hidden by growth
chout = hidden
chin = hidden
hidden = min(int(growth * hidden), max_hidden)
self.lstm = BLSTM(bi=not causal)
def main():
# Load the data. Use the Numenta Anomaly Benchmark (NAB) dataset.
# It provides artificial timeseries data containing labeled
# anomalous periods of behavior. Data rae ordered, timestamped,
# single-valued metrics. Use the art_daily_small_noise.csv file for
# training and the art_daily_jumpsup.csv file for testing. The
# simplicity of this dataset allows us to demonstrate anomaly
# detection effectively.
master_url_root = "https://raw.githubusercontent.com/numenta/NAB/master/data/"
df_small_noise_url_suffix = "artificialNoAnomaly/art_daily_small_noise.csv"
df_small_noise_url = master_url_root + df_small_noise_url_suffix
df_small_noise = pd.read_csv(df_small_noise_url,
parse_dates=True,
index_col="timestamp")
df_daily_jumpsup_url_suffix = "artificialWithAnomaly/art_daily_jumpsup.csv"
df_daily_jumpsup_url = master_url_root + df_daily_jumpsup_url_suffix
df_daily_jumpsup = pd.read_csv(df_daily_jumpsup_url,
parse_dates=True,
index_col="timestamp")
# Quick look at the data.
print(df_small_noise.head())
print(df_daily_jumpsup.head())
# Visualize the data. Timeseries data without anomalies will be
# used for training.
'''
fig, ax = plt.subplots()
df_small_noise.plot(legend=False, ax=ax)
plt.show()
'''
# Timeseries data with anomalies will be used for testing and see
# if the sudden jump up in the data is detected as an anomaly.
'''
fig, ax = plt.subplots()
df_daily_jumpsup.plot(legend=False, ax=ax)
plt.show()
'''
# Prepare training data. Get data values from the training
# timeseries data file and normalize the value data. There is a
# value for every 5 mins for 14 days.
# 24 * 60 / 5 = 288timesteps per day
# 288 * 14 = 4032 data points in total
# Normalize and save the mean and std.
training_mean = df_small_noise.mean()
training_std = df_small_noise.std()
df_training_value = (df_small_noise - training_mean) / training_std
print("Number of training samples:", len(df_training_value))
# Create sequences. Create sequences combining TIME_STEPS
# contiguous data values from the training data.
TIME_STEPS = 288
x_train = create_sequences(df_training_value.values, TIME_STEPS)
print("Training input shape: ", x_train.shape)
# Build a model. Build a convolutional reconstruction autoencoder
# model. The model will take input of shape (batch_size,
# sequence_length, num_features) and return output of the same
# shape. In this case, sequence_length is 288 and num_features is
# 1.
model = keras.Sequential([
layers.Input(shape=(x_train.shape[1], x_train.shape[2])),
layers.Conv1D(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1D(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=16,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Dropout(rate=0.2),
layers.Conv1DTranspose(filters=32,
kernel_size=7,
padding="same",
strides=2,
activation="relu"),
layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same"),
])
model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
loss="mse")
model.summary()
# Train the model. Note that using x_train as both the input and
# the target since this is a reconstruction model.
history = model.fit(
x_train,
x_train,
epochs=50,
batch_size=128,
validation_split=0.1,
callbacks=[
keras.callbacks.EarlyStopping(monitor="val_loss",
patience=5,
mode="min")
],
)
# Plot training and validation loss to see how training went.
'''
plt.plot(history.history["loss"], label="Training Loss")
plt.plot(history.history["val_loss"], label="Validation Loss")
plt.legend()
plt.show()
'''
# Detecting anomalies. Detect anomalies by determining how well the
# model can reconstruct the input data.
# 1) Find MAE loss on training samples.
# 2) Find max MAE loss value. This is the worst the model has
# performed trying to reconstruct a sample. This will be the
# threshold for anomaly detection.
# 3) If the reconstruction loss for a sample is greater than this
# threshold value then it can be infered that the model is seeing
# a patter that it isn't familiar with. This sample will be
# labeled an anomaly.
# Get train MAE loss.
x_train_pred = model.predict(x_train)
train_mae_loss = np.mean(np.abs(x_train_pred - x_train), axis=1)
'''
plt.hist(train_mae_loss, bins=50)
plt.xlabel("Train MAE loss")
plt.ylabel("No of samples")
plt.show()
'''
# Get reconstruction of loss threshold.
threshold = np.max(train_mae_loss)
print("Reconstruction error threshold: ", threshold)
# Compare reconstruction. Just for fun, see how the model has
# reconstructed the first sample. This is the 288 timesteps from
# day 1 of the training dataset.
# Checking how the first sequence is learned.
'''
plt.plot(x_train[0])
plt.plot(x_train_pred[0])
plt.show()
'''
# Prepare test data.
def_test_value = (df_daily_jumpsup - training_mean) / training_std
'''
fig, ax = plt.subplots()
def_test_value.plot(legend=False, ax=ax)
plt.show()
'''
# Create sequences from test values.
x_test = create_sequences(def_test_value.values, TIME_STEPS)
print("Test input shape: ", x_test.shape)
# Get test MAE loss.
x_test_pred = model.predict(x_test)
test_mae_loss = np.mean(np.abs(x_test_pred - x_test), axis=1)
test_mae_loss = test_mae_loss.reshape((-1))
'''
plt.hist(test_mae_loss, bins=50)
plt.xlabel("test MAE loss")
plt.ylabel("No of samples")
plt.show()
'''
# Detect all the samples which are anomalies.
anomalies = test_mae_loss > threshold
print("Number of anomaly samples: ", np.sum(anomalies))
print("Indices of anomaly samples: ", np.where(anomalies))
# Plot anomalies. Now that it is known which samples of the data
# are anomalies, find the corresponding timestamps from the
# original test data.
anomalous_data_indices = []
for data_idx in range(TIME_STEPS - 1,
len(def_test_value) - TIME_STEPS + 1):
if np.all(anomalies[data_idx - TIME_STEPS + 1:data_idx]):
anomalous_data_indices.append(data_idx)
# Overlay the anomalies on the original test data plot.
df_subset = df_daily_jumpsup.iloc[anomalous_data_indices]
'''
fig, ax = plt.subplots()
df_daily_jumpsup.plot(legend=False, ax=ax)
df_subset.plot(legend=False, ax=ax, color="r")
plt.show()
'''
# Exit the program.
exit(0)
