def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
if not type(det_load) == OutlierProphet:
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [AdversarialVAE, OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialVAE:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialVAE, OutlierVAE]:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
elif type(det_load) == OutlierProphet:
assert det_load.model.interval_width == .7
assert det_load.model.growth == 'logistic'
assert det_load.meta['data_type'] == 'time-series'
elif type(det_load) == SpectralResidual:
assert det_load.window_amp == 10
assert det_load.window_local == 10
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [AdversarialVAE, OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialVAE:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialVAE, OutlierVAE]:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
def train_outlier_detector(data: Cifar10, artifacts_path: str):
tf.keras.backend.clear_session()
# define encoder and decoder networks
latent_dim = 1024
encoder_net = tf.keras.Sequential([
InputLayer(input_shape=(32, 32, 3)),
Conv2D(64, 4, strides=2, padding="same", activation=tf.nn.relu),
Conv2D(128, 4, strides=2, padding="same", activation=tf.nn.relu),
Conv2D(512, 4, strides=2, padding="same", activation=tf.nn.relu),
])
decoder_net = tf.keras.Sequential([
InputLayer(input_shape=(latent_dim, )),
Dense(4 * 4 * 128),
Reshape(target_shape=(4, 4, 128)),
Conv2DTranspose(256,
4,
strides=2,
padding="same",
activation=tf.nn.relu),
Conv2DTranspose(64,
4,
strides=2,
padding="same",
activation=tf.nn.relu),
Conv2DTranspose(3, 4, strides=2, padding="same", activation="sigmoid"),
])
# initialize outlier detector
od = OutlierVAE(
threshold=0.015, # threshold for outlier score
encoder_net=encoder_net, # can also pass VAE model instead
decoder_net=decoder_net, # of separate encoder and decoder
latent_dim=latent_dim,
)
# train
od.fit(data.X_train, epochs=50, verbose=False)
# save the trained outlier detector
save_detector(od, f"{artifacts_path}/outlier")
# define encoder and decoder networks
latent_dim = 1024
encoder_net = tf.keras.Sequential([
InputLayer(input_shape=(32, 32, 3)),
Conv2D(64, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2D(128, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2D(512, 4, strides=2, padding='same', activation=tf.nn.relu)
])
decoder_net = tf.keras.Sequential([
InputLayer(input_shape=(latent_dim, )),
Dense(4 * 4 * 128),
Reshape(target_shape=(4, 4, 128)),
Conv2DTranspose(256, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2DTranspose(64, 4, strides=2, padding='same', activation=tf.nn.relu),
Conv2DTranspose(3, 4, strides=2, padding='same', activation='sigmoid')
])
# initialize outlier detector
od = OutlierVAE(
threshold=.015, # threshold for outlier score
encoder_net=encoder_net, # can also pass VAE model instead
decoder_net=decoder_net, # of separate encoder and decoder
latent_dim=latent_dim)
# train
od.fit(X_train, epochs=50, verbose=True)
# save the trained outlier detector
save_detector(od, './outlier-detector')
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
# save and load functionality does not work for OutlierProphet and Python 3.6.
# https://github.com/facebook/prophet/issues/1361
if sys.version_info.minor == 6 and isinstance(det, OutlierProphet):
return
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
if not type(det_load) in [OutlierProphet, KSDrift, MMDDrift]:
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialAE:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == MMDDrift:
assert det_load.infer_sigma
assert isinstance(det_load.permutation_test, Callable)
if type(det_load) == KSDrift:
assert det_load.n_features == latent_dim
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialAE, OutlierAE]:
assert isinstance(det_load.ae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae, tf.keras.Model)
elif type(det_load) == OutlierVAE:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
elif type(det_load) == OutlierProphet:
assert det_load.model.interval_width == .7
assert det_load.model.growth == 'logistic'
assert det_load.meta['data_type'] == 'time-series'
elif type(det_load) == SpectralResidual:
assert det_load.window_amp == 10
assert det_load.window_local == 10
elif type(det_load) == OutlierSeq2Seq:
assert isinstance(det_load.seq2seq, tf.keras.Model)
assert isinstance(det_load.seq2seq.threshold_net,
tf.keras.Sequential)
assert isinstance(det_load.seq2seq.encoder, EncoderLSTM)
assert isinstance(det_load.seq2seq.decoder, DecoderLSTM)
assert det_load.latent_dim == latent_dim
assert det_load.threshold == threshold
assert det_load.shape == (-1, seq_len, input_dim)
elif type(det_load) in [KSDrift, MMDDrift]:
assert isinstance(det_load.preprocess_fn, Callable)
assert det_load.preprocess_fn.__name__ == 'uae'
assert isinstance(det_load.preprocess_kwargs['encoder_net'],
tf.keras.Sequential)
assert det_load.p_val == p_val
assert (det_load.X_ref == X_ref).all()
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
# save and load functionality does not work for OutlierProphet and Python 3.6.
# https://github.com/facebook/prophet/issues/1361
if sys.version_info.minor == 6 and isinstance(det, OutlierProphet):
return
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
if not type(det_load) in [
OutlierProphet, ChiSquareDrift, ClassifierDrift, KSDrift,
MMDDrift, TabularDrift
]:
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialAE or type(
det_load) == ModelDistillation:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialAE, OutlierAE]:
assert isinstance(det_load.ae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae, tf.keras.Model)
elif type(det_load) == ModelDistillation:
assert isinstance(det_load.model,
tf.keras.Sequential) or isinstance(
det_load.model, tf.keras.Model)
assert (isinstance(det_load.distilled_model, tf.keras.Sequential)
or isinstance(det_load.distilled_model, tf.keras.Model))
elif type(det_load) == OutlierVAE:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
elif type(det_load) == OutlierProphet:
assert det_load.model.interval_width == .7
assert det_load.model.growth == 'logistic'
assert det_load.meta['data_type'] == 'time-series'
elif type(det_load) == SpectralResidual:
assert det_load.window_amp == 10
assert det_load.window_local == 10
elif type(det_load) == OutlierSeq2Seq:
assert isinstance(det_load.seq2seq, tf.keras.Model)
assert isinstance(det_load.seq2seq.threshold_net,
tf.keras.Sequential)
assert isinstance(det_load.seq2seq.encoder, EncoderLSTM)
assert isinstance(det_load.seq2seq.decoder, DecoderLSTM)
assert det_load.latent_dim == latent_dim
assert det_load.threshold == threshold
assert det_load.shape == (-1, seq_len, input_dim)
elif type(det_load) == KSDrift:
assert det_load.n_features == latent_dim
assert det_load.p_val == p_val
assert (det_load.x_ref == X_ref).all()
assert isinstance(det_load.preprocess_fn, Callable)
assert det_load.preprocess_fn.func.__name__ == 'preprocess_drift'
elif type(det_load) in [ChiSquareDrift, TabularDrift]:
assert isinstance(det_load.x_ref_categories, dict)
assert det_load.p_val == p_val
x = X_ref_cat.copy() if isinstance(
det_load, ChiSquareDrift) else X_ref_mix.copy()
assert (det_load.x_ref == x).all()
elif type(det_load) == MMDDrift:
assert not det_load._detector.infer_sigma
assert det_load._detector.n_permutations == n_permutations
assert det_load._detector.p_val == p_val
assert (det_load._detector.x_ref == X_ref).all()
assert isinstance(det_load._detector.preprocess_fn, Callable)
assert det_load._detector.preprocess_fn.func.__name__ == 'preprocess_drift'
elif type(det_load) == ClassifierDrift:
assert det_load._detector.p_val == p_val
assert (det_load._detector.x_ref == X_ref).all()
assert isinstance(det_load._detector.skf, StratifiedKFold)
assert isinstance(det_load._detector.compile_kwargs, dict)
assert isinstance(det_load._detector.train_kwargs, dict)
assert isinstance(det_load._detector.model, tf.keras.Model)
elif type(det_load) == LLR:
assert isinstance(det_load.dist_s, tf.keras.Model)
assert isinstance(det_load.dist_b, tf.keras.Model)
assert not det_load.sequential
assert not det_load.has_log_prob
