def __init__(self):
from alibi_detect.utils.saving import load_detector
model = self.get_tempo()
models_folder = model.details.local_folder
print(f"Loading from {models_folder}")
self.od = load_detector(os.path.join(models_folder, "cifar10"))
async def load(self) -> bool:
model_uri = await get_model_uri(self._settings)
if not self.alibi_detect_settings.init_detector:
try:
self._model = load_detector(model_uri)
except (
ValueError,
FileNotFoundError,
EOFError,
pickle.UnpicklingError,
) as e:
raise InvalidAlibiDetector(self._settings.name) from e
else:
try:
detector_data = pickle.load(open(model_uri, "rb"))
drift_detector = self.alibi_detect_settings.detector_type
except (FileNotFoundError, EOFError, pickle.UnpicklingError) as e:
raise InvalidAlibiDetector(self._settings.name) from e
parameters = self.alibi_detect_settings.init_parameters
self._model = drift_detector(**detector_data, **parameters)
self.ready = True
return self.ready
def fetch_detector(filepath: str,
detector_type: str,
dataset: str,
detector_name: str,
model: str = None) -> Data:
"""
Fetch an outlier or adversarial detector from a google bucket, save it locally and return
the initialised detector.
Parameters
----------
filepath
Local directory to save detector to.
detector_type
`outlier` or `adversarial`.
dataset
Dataset of pre-trained detector. E.g. `kddcup`, `cifar10` or `ecg`.
detector_name
Name of the detector in the bucket.
model
Classification model used for adversarial detection.
Returns
-------
Initialised pre-trained detector.
"""
# create url of detector
filepath = os.path.join(filepath, detector_name)
if not os.path.isdir(filepath):
logger.warning(
'Directory {} does not exist and is now created.'.format(filepath))
os.mkdir(filepath)
url = 'https://storage.googleapis.com/seldon-models/alibi-detect/'
if detector_type == 'adversarial':
url = os.path.join(url, 'ad', dataset, model, detector_name)
elif detector_type == 'outlier':
url = os.path.join(url, 'od', detector_name, dataset)
# fetch the metadata and state dict
meta, state_dict = fetch_state_dict(url, filepath, save_state_dict=True)
# load detector
name = meta['name']
kwargs = {}
if name == 'OutlierAE':
fetch_ae(url, filepath)
elif name == 'OutlierAEGMM':
fetch_aegmm(url, filepath)
elif name == 'OutlierVAE':
fetch_vae(url, filepath)
elif name == 'OutlierVAEGMM':
fetch_vaegmm(url, filepath)
elif name == 'OutlierSeq2Seq':
fetch_seq2seq(url, filepath)
elif name == 'AdversarialAE':
fetch_ad_ae(url, filepath, state_dict)
if model == 'resnet56':
kwargs = {'custom_objects': {'backend': backend}}
detector = load_detector(filepath, **kwargs)
return detector
def load(self):
"""
Load the model from storage
"""
model_folder = kfserving.Storage.download(self.storage_uri)
self.model = load_detector(model_folder)
self.ready = True
def load(self):
"""
Load the model from storage
"""
model_folder = download_model(self.storage_uri)
self.model: Data = load_detector(model_folder)
self.ready = True
def __init__(self) -> None:
import tensorflow as tf
for gpu in tf.config.experimental.list_physical_devices("GPU"):
tf.config.experimental.set_memory_growth(gpu, True)
from alibi_detect.od.vae import OutlierVAE
from alibi_detect.utils.saving import load_detector
self.model = cast(OutlierVAE, load_detector(paths.vae_model))
self.model.threshold = self.THRESHOLD
self.model.predict(np.random.random((1, *self.IMG_SIZE, 3)))
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
if not type(det_load) == OutlierProphet:
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [AdversarialVAE, OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialVAE:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialVAE, OutlierVAE]:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
elif type(det_load) == OutlierProphet:
assert det_load.model.interval_width == .7
assert det_load.model.growth == 'logistic'
assert det_load.meta['data_type'] == 'time-series'
elif type(det_load) == SpectralResidual:
assert det_load.window_amp == 10
assert det_load.window_local == 10
def evaluate_outlier(
pedestrian_results_path: Path,
model_path: Path,
batch_size=1000,
) -> None:
# NOTE: Speed up cli.
# TODO: Try lazy loading sub commands instead.
from alibi_detect.od.ae import OutlierAE
from alibi_detect.utils.saving import load_detector
save_dir = paths.temp_dir_path / "ae_box" / f"{int(time())}_eval_outlier"
save_dir.mkdir(parents=True)
model = cast(OutlierAE, load_detector(model_path))
df: pd.DataFrame = pd.read_pickle(pedestrian_results_path)
df["has_pred"] = df.predictions.str.len() > 0
df["has_label"] = df.labels.str.len() > 0
chunks = np.array_split(df, len(df) // batch_size)
for chunk in tqdm(chunks):
with concurrent.futures.ThreadPoolExecutor(
max_workers=None) as executor:
boxes = list(
executor.map(extract_box, chunk[chunk.has_pred].itertuples()))
chunk_res = model.predict(np.stack(boxes, 0),
return_feature_score=False)
chunk.loc[chunk.has_pred, "instance_score"] = chunk_res["data"][
"instance_score"] # type: ignore
df = cast(pd.DataFrame, pd.concat(chunks, ignore_index=True))
df.to_pickle(save_dir / "ae-box-aug-res.pkl")
outlier_threshold = estimate_threshold(df)
df.boxplot(column="instance_score",
by="object_type").figure.savefig("box.png")
df["object_color"] = color_series(df.object_type)
for distance in range(0, 40, 10):
plot_with_distance_threshold(
df,
distance,
outlier_threshold,
save_dir / f"scatter_{distance}.png",
)
df[(df.current_distance >= distance)
& (df.instance_score > outlier_threshold)].object_type.value_counts(
).to_csv(f"filtered_objects_{distance}.csv")
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [AdversarialVAE, OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialVAE:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialVAE, OutlierVAE]:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
# save and load functionality does not work for OutlierProphet and Python 3.6.
# https://github.com/facebook/prophet/issues/1361
if sys.version_info.minor == 6 and isinstance(det, OutlierProphet):
return
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
if not type(det_load) in [OutlierProphet, KSDrift, MMDDrift]:
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialAE:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == MMDDrift:
assert det_load.infer_sigma
assert isinstance(det_load.permutation_test, Callable)
if type(det_load) == KSDrift:
assert det_load.n_features == latent_dim
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialAE, OutlierAE]:
assert isinstance(det_load.ae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae, tf.keras.Model)
elif type(det_load) == OutlierVAE:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
elif type(det_load) == OutlierProphet:
assert det_load.model.interval_width == .7
assert det_load.model.growth == 'logistic'
assert det_load.meta['data_type'] == 'time-series'
elif type(det_load) == SpectralResidual:
assert det_load.window_amp == 10
assert det_load.window_local == 10
elif type(det_load) == OutlierSeq2Seq:
assert isinstance(det_load.seq2seq, tf.keras.Model)
assert isinstance(det_load.seq2seq.threshold_net,
tf.keras.Sequential)
assert isinstance(det_load.seq2seq.encoder, EncoderLSTM)
assert isinstance(det_load.seq2seq.decoder, DecoderLSTM)
assert det_load.latent_dim == latent_dim
assert det_load.threshold == threshold
assert det_load.shape == (-1, seq_len, input_dim)
elif type(det_load) in [KSDrift, MMDDrift]:
assert isinstance(det_load.preprocess_fn, Callable)
assert det_load.preprocess_fn.__name__ == 'uae'
assert isinstance(det_load.preprocess_kwargs['encoder_net'],
tf.keras.Sequential)
assert det_load.p_val == p_val
assert (det_load.X_ref == X_ref).all()
def test_save_load(select_detector):
det = select_detector
det_name = det.meta['name']
# save and load functionality does not work for OutlierProphet and Python 3.6.
# https://github.com/facebook/prophet/issues/1361
if sys.version_info.minor == 6 and isinstance(det, OutlierProphet):
return
with TemporaryDirectory() as temp_dir:
temp_dir += '/'
save_detector(det, temp_dir)
det_load = load_detector(temp_dir)
det_load_name = det_load.meta['name']
assert det_load_name == det_name
if not type(det_load) in [
OutlierProphet, ChiSquareDrift, ClassifierDrift, KSDrift,
MMDDrift, TabularDrift
]:
assert det_load.threshold == det.threshold == threshold
if type(det_load) in [OutlierVAE, OutlierVAEGMM]:
assert det_load.samples == det.samples == samples
if type(det_load) == AdversarialAE or type(
det_load) == ModelDistillation:
for layer in det_load.model.layers:
assert not layer.trainable
if type(det_load) == OutlierAEGMM:
assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.aegmm, tf.keras.Model)
assert det_load.aegmm.n_gmm == n_gmm
elif type(det_load) == OutlierVAEGMM:
assert isinstance(det_load.vaegmm.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential)
assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential)
assert isinstance(det_load.vaegmm, tf.keras.Model)
assert det_load.vaegmm.latent_dim == latent_dim
assert det_load.vaegmm.n_gmm == n_gmm
elif type(det_load) in [AdversarialAE, OutlierAE]:
assert isinstance(det_load.ae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.ae, tf.keras.Model)
elif type(det_load) == ModelDistillation:
assert isinstance(det_load.model,
tf.keras.Sequential) or isinstance(
det_load.model, tf.keras.Model)
assert (isinstance(det_load.distilled_model, tf.keras.Sequential)
or isinstance(det_load.distilled_model, tf.keras.Model))
elif type(det_load) == OutlierVAE:
assert isinstance(det_load.vae.encoder.encoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae.decoder.decoder_net,
tf.keras.Sequential)
assert isinstance(det_load.vae, tf.keras.Model)
assert det_load.vae.latent_dim == latent_dim
elif type(det_load) == Mahalanobis:
assert det_load.clip is None
assert det_load.mean == det_load.C == det_load.n == 0
assert det_load.meta['detector_type'] == 'online'
elif type(det_load) == OutlierProphet:
assert det_load.model.interval_width == .7
assert det_load.model.growth == 'logistic'
assert det_load.meta['data_type'] == 'time-series'
elif type(det_load) == SpectralResidual:
assert det_load.window_amp == 10
assert det_load.window_local == 10
elif type(det_load) == OutlierSeq2Seq:
assert isinstance(det_load.seq2seq, tf.keras.Model)
assert isinstance(det_load.seq2seq.threshold_net,
tf.keras.Sequential)
assert isinstance(det_load.seq2seq.encoder, EncoderLSTM)
assert isinstance(det_load.seq2seq.decoder, DecoderLSTM)
assert det_load.latent_dim == latent_dim
assert det_load.threshold == threshold
assert det_load.shape == (-1, seq_len, input_dim)
elif type(det_load) == KSDrift:
assert det_load.n_features == latent_dim
assert det_load.p_val == p_val
assert (det_load.x_ref == X_ref).all()
assert isinstance(det_load.preprocess_fn, Callable)
assert det_load.preprocess_fn.func.__name__ == 'preprocess_drift'
elif type(det_load) in [ChiSquareDrift, TabularDrift]:
assert isinstance(det_load.x_ref_categories, dict)
assert det_load.p_val == p_val
x = X_ref_cat.copy() if isinstance(
det_load, ChiSquareDrift) else X_ref_mix.copy()
assert (det_load.x_ref == x).all()
elif type(det_load) == MMDDrift:
assert not det_load._detector.infer_sigma
assert det_load._detector.n_permutations == n_permutations
assert det_load._detector.p_val == p_val
assert (det_load._detector.x_ref == X_ref).all()
assert isinstance(det_load._detector.preprocess_fn, Callable)
assert det_load._detector.preprocess_fn.func.__name__ == 'preprocess_drift'
elif type(det_load) == ClassifierDrift:
assert det_load._detector.p_val == p_val
assert (det_load._detector.x_ref == X_ref).all()
assert isinstance(det_load._detector.skf, StratifiedKFold)
assert isinstance(det_load._detector.compile_kwargs, dict)
assert isinstance(det_load._detector.train_kwargs, dict)
assert isinstance(det_load._detector.model, tf.keras.Model)
elif type(det_load) == LLR:
assert isinstance(det_load.dist_s, tf.keras.Model)
assert isinstance(det_load.dist_b, tf.keras.Model)
assert not det_load.sequential
assert not det_load.has_log_prob
def fetch_detector(filepath: Union[str, os.PathLike],
detector_type: str,
dataset: str,
detector_name: str,
model: str = None) -> Data:
"""
Fetch an outlier or adversarial detector from a google bucket, save it locally and return
the initialised detector.
Parameters
----------
filepath
Local directory to save detector to.
detector_type
`outlier` or `adversarial`.
dataset
Dataset of pre-trained detector. E.g. `kddcup`, `cifar10` or `ecg`.
detector_name
Name of the detector in the bucket.
model
Classification model used for adversarial detection.
Returns
-------
Initialised pre-trained detector.
"""
# create path (if needed)
filepath = Path(filepath)
if not filepath.is_dir():
filepath.mkdir(parents=True, exist_ok=True)
logger.warning(
'Directory {} does not exist and is now created.'.format(filepath))
# create url of detector
url = 'https://storage.googleapis.com/seldon-models/alibi-detect/'
if detector_type == 'adversarial':
url = _join_url(url, ['ad', dataset, model, detector_name])
elif detector_type == 'outlier':
url = _join_url(url, ['od', detector_name, dataset])
# fetch the metadata and state dict
meta, state_dict = fetch_state_dict(url, filepath, save_state_dict=True)
# load detector
name = meta['name']
kwargs = {} # type: dict
if name == 'OutlierAE':
fetch_ae(url, filepath)
elif name == 'OutlierAEGMM':
fetch_aegmm(url, filepath)
elif name == 'OutlierVAE':
fetch_vae(url, filepath)
elif name == 'OutlierVAEGMM':
fetch_vaegmm(url, filepath)
elif name == 'OutlierSeq2Seq':
fetch_seq2seq(url, filepath)
elif name == 'AdversarialAE':
fetch_ad_ae(url, filepath, state_dict)
if model == 'resnet56':
kwargs = {'custom_objects': {'backend': backend}}
elif name == 'ModelDistillation':
fetch_ad_md(url, filepath)
if model == 'resnet56':
kwargs = {'custom_objects': {'backend': backend}}
elif name == 'LLR':
model_type = fetch_llr(url, filepath)
if model_type == 'weights':
kwargs = get_pixelcnn_default_kwargs()
detector = load_detector(filepath, **kwargs)
return detector
def load(self):
self.model = load_detector(self.model_path)
self.ready = True
