def train_outlier_detector(data: Cifar10, artifacts_path: str): tf.keras.backend.clear_session() # define encoder and decoder networks latent_dim = 1024 encoder_net = tf.keras.Sequential([ InputLayer(input_shape=(32, 32, 3)), Conv2D(64, 4, strides=2, padding="same", activation=tf.nn.relu), Conv2D(128, 4, strides=2, padding="same", activation=tf.nn.relu), Conv2D(512, 4, strides=2, padding="same", activation=tf.nn.relu), ]) decoder_net = tf.keras.Sequential([ InputLayer(input_shape=(latent_dim, )), Dense(4 * 4 * 128), Reshape(target_shape=(4, 4, 128)), Conv2DTranspose(256, 4, strides=2, padding="same", activation=tf.nn.relu), Conv2DTranspose(64, 4, strides=2, padding="same", activation=tf.nn.relu), Conv2DTranspose(3, 4, strides=2, padding="same", activation="sigmoid"), ]) # initialize outlier detector od = OutlierVAE( threshold=0.015, # threshold for outlier score encoder_net=encoder_net, # can also pass VAE model instead decoder_net=decoder_net, # of separate encoder and decoder latent_dim=latent_dim, ) # train od.fit(data.X_train, epochs=50, verbose=False) # save the trained outlier detector save_detector(od, f"{artifacts_path}/outlier")
def init_od_vae(state_dict: Dict, vae: tf.keras.Model) -> OutlierVAE: """ Initialize OutlierVAE. Parameters ---------- state_dict Dictionary containing the parameter values. vae Loaded VAE. Returns ------- Initialized OutlierVAE instance. """ od = OutlierVAE(threshold=state_dict['threshold'], score_type=state_dict['score_type'], vae=vae, samples=state_dict['samples']) return od
# define model inputs = tf.keras.Input(shape=(input_dim, )) outputs = tf.keras.layers.Dense(2, activation=tf.nn.softmax)(inputs) model = tf.keras.Model(inputs=inputs, outputs=outputs) detector = [ AdversarialAE(threshold=threshold, model=model, **kwargs), IForest(threshold=threshold), Mahalanobis(threshold=threshold), OutlierAEGMM(threshold=threshold, gmm_density_net=gmm_density_net, n_gmm=n_gmm, **kwargs), OutlierVAE(threshold=threshold, latent_dim=latent_dim, samples=samples, **kwargs), OutlierAE(threshold=threshold, **kwargs), OutlierVAEGMM(threshold=threshold, gmm_density_net=gmm_density_net, n_gmm=n_gmm, latent_dim=latent_dim, samples=samples, **kwargs), OutlierProphet(threshold=.7, growth='logistic'), SpectralResidual(threshold=threshold, window_amp=10, window_local=10), OutlierSeq2Seq(input_dim, seq_len, threshold=threshold, threshold_net=threshold_net, latent_dim=latent_dim)
def test_vae(vae_params): # OutlierVAE parameters threshold, score_type, samples, loss_fn, threshold_perc, return_instance_score, \ return_feature_score, outlier_perc, outlier_type = vae_params # define encoder and decoder encoder_net = tf.keras.Sequential([ InputLayer(input_shape=(input_dim, )), Dense(5, activation=tf.nn.relu), Dense(latent_dim, activation=None) ]) decoder_net = tf.keras.Sequential([ InputLayer(input_shape=(latent_dim, )), Dense(5, activation=tf.nn.relu), Dense(input_dim, activation=tf.nn.sigmoid) ]) # init OutlierVAE vae = OutlierVAE(threshold=threshold, score_type=score_type, encoder_net=encoder_net, decoder_net=decoder_net, latent_dim=latent_dim, samples=samples) assert vae.threshold == threshold assert vae.meta == { 'name': 'OutlierVAE', 'detector_type': 'offline', 'data_type': None, 'version': __version__ } # fit OutlierVAE, infer threshold and compute scores vae.fit(X, loss_fn=loss_fn, epochs=5, verbose=False) vae.infer_threshold(X, threshold_perc=threshold_perc) fscore, iscore = vae.score(X) perc_score = 100 * (iscore < vae.threshold).astype(int).sum() / iscore.shape[0] assert threshold_perc + 5 > perc_score > threshold_perc - 5 # make and check predictions od_preds = vae.predict(X, return_instance_score=return_instance_score, return_feature_score=return_feature_score, outlier_type=outlier_type, outlier_perc=outlier_perc) assert od_preds['meta'] == vae.meta if outlier_type == 'instance': assert od_preds['data']['is_outlier'].shape == (X.shape[0], ) if return_instance_score: assert od_preds['data']['is_outlier'].sum() == ( od_preds['data']['instance_score'] > vae.threshold).astype(int).sum() elif outlier_type == 'feature': assert od_preds['data']['is_outlier'].shape == X.shape if return_feature_score: assert od_preds['data']['is_outlier'].sum() == ( od_preds['data']['feature_score'] > vae.threshold).astype(int).sum() if return_feature_score: assert od_preds['data']['feature_score'].shape == X.shape else: assert od_preds['data']['feature_score'] is None if return_instance_score: assert od_preds['data']['instance_score'].shape == (X.shape[0], ) else: assert od_preds['data']['instance_score'] is None
# define encoder and decoder networks latent_dim = 1024 encoder_net = tf.keras.Sequential([ InputLayer(input_shape=(32, 32, 3)), Conv2D(64, 4, strides=2, padding='same', activation=tf.nn.relu), Conv2D(128, 4, strides=2, padding='same', activation=tf.nn.relu), Conv2D(512, 4, strides=2, padding='same', activation=tf.nn.relu) ]) decoder_net = tf.keras.Sequential([ InputLayer(input_shape=(latent_dim, )), Dense(4 * 4 * 128), Reshape(target_shape=(4, 4, 128)), Conv2DTranspose(256, 4, strides=2, padding='same', activation=tf.nn.relu), Conv2DTranspose(64, 4, strides=2, padding='same', activation=tf.nn.relu), Conv2DTranspose(3, 4, strides=2, padding='same', activation='sigmoid') ]) # initialize outlier detector od = OutlierVAE( threshold=.015, # threshold for outlier score encoder_net=encoder_net, # can also pass VAE model instead decoder_net=decoder_net, # of separate encoder and decoder latent_dim=latent_dim) # train od.fit(X_train, epochs=50, verbose=True) # save the trained outlier detector save_detector(od, './outlier-detector')