def train_outlier_detector(data: Cifar10, artifacts_path: str):
tf.keras.backend.clear_session()
# define encoder and decoder networks
latent_dim = 1024
encoder_net = tf.keras.Sequential([
InputLayer(input_shape=(32, 32, 3)),
Conv2D(64, 4, strides=2, padding="same", activation=tf.nn.relu),
Conv2D(128, 4, strides=2, padding="same", activation=tf.nn.relu),
Conv2D(512, 4, strides=2, padding="same", activation=tf.nn.relu),
])
decoder_net = tf.keras.Sequential([
InputLayer(input_shape=(latent_dim, )),
Dense(4 * 4 * 128),
Reshape(target_shape=(4, 4, 128)),
Conv2DTranspose(256,
4,
strides=2,
padding="same",
activation=tf.nn.relu),
Conv2DTranspose(64,
4,
strides=2,
padding="same",
activation=tf.nn.relu),
Conv2DTranspose(3, 4, strides=2, padding="same", activation="sigmoid"),
])
# initialize outlier detector
od = OutlierVAE(
threshold=0.015, # threshold for outlier score
encoder_net=encoder_net, # can also pass VAE model instead
decoder_net=decoder_net, # of separate encoder and decoder
latent_dim=latent_dim,
)
# train
od.fit(data.X_train, epochs=50, verbose=False)
# save the trained outlier detector
save_detector(od, f"{artifacts_path}/outlier")
def test_vae(vae_params):
# OutlierVAE parameters
threshold, score_type, samples, loss_fn, threshold_perc, return_instance_score, \
return_feature_score, outlier_perc, outlier_type = vae_params
# define encoder and decoder
encoder_net = tf.keras.Sequential([
InputLayer(input_shape=(input_dim, )),
Dense(5, activation=tf.nn.relu),
Dense(latent_dim, activation=None)
])
decoder_net = tf.keras.Sequential([
InputLayer(input_shape=(latent_dim, )),
Dense(5, activation=tf.nn.relu),
Dense(input_dim, activation=tf.nn.sigmoid)
])
# init OutlierVAE
vae = OutlierVAE(threshold=threshold,
score_type=score_type,
encoder_net=encoder_net,
decoder_net=decoder_net,
latent_dim=latent_dim,
samples=samples)
assert vae.threshold == threshold
assert vae.meta == {
'name': 'OutlierVAE',
'detector_type': 'offline',
'data_type': None,
'version': __version__
}
# fit OutlierVAE, infer threshold and compute scores
vae.fit(X, loss_fn=loss_fn, epochs=5, verbose=False)
vae.infer_threshold(X, threshold_perc=threshold_perc)
fscore, iscore = vae.score(X)
perc_score = 100 * (iscore <
vae.threshold).astype(int).sum() / iscore.shape[0]
assert threshold_perc + 5 > perc_score > threshold_perc - 5
# make and check predictions
od_preds = vae.predict(X,
return_instance_score=return_instance_score,
return_feature_score=return_feature_score,
outlier_type=outlier_type,
outlier_perc=outlier_perc)
assert od_preds['meta'] == vae.meta
if outlier_type == 'instance':
assert od_preds['data']['is_outlier'].shape == (X.shape[0], )
if return_instance_score:
assert od_preds['data']['is_outlier'].sum() == (
od_preds['data']['instance_score'] >
vae.threshold).astype(int).sum()
elif outlier_type == 'feature':
assert od_preds['data']['is_outlier'].shape == X.shape
if return_feature_score:
assert od_preds['data']['is_outlier'].sum() == (
od_preds['data']['feature_score'] >
vae.threshold).astype(int).sum()
if return_feature_score:
assert od_preds['data']['feature_score'].shape == X.shape
else:
assert od_preds['data']['feature_score'] is None
if return_instance_score:
assert od_preds['data']['instance_score'].shape == (X.shape[0], )
else:
assert od_preds['data']['instance_score'] is None
# define encoder and decoder networks
latent_dim = 1024
encoder_net = tf.keras.Sequential([
InputLayer(input_shape=(32, 32, 3)),
Conv2D(64, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2D(128, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2D(512, 4, strides=2, padding='same', activation=tf.nn.relu)
])
decoder_net = tf.keras.Sequential([
InputLayer(input_shape=(latent_dim, )),
Dense(4 * 4 * 128),
Reshape(target_shape=(4, 4, 128)),
Conv2DTranspose(256, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2DTranspose(64, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2DTranspose(3, 4, strides=2, padding='same', activation='sigmoid')
])
# initialize outlier detector
od = OutlierVAE(
threshold=.015, # threshold for outlier score
encoder_net=encoder_net, # can also pass VAE model instead
decoder_net=decoder_net, # of separate encoder and decoder
latent_dim=latent_dim)
# train
od.fit(X_train, epochs=50, verbose=True)
# save the trained outlier detector
save_detector(od, './outlier-detector')
