def __init__(self): from alibi_detect.utils.saving import load_detector model = self.get_tempo() models_folder = model.details.local_folder print(f"Loading from {models_folder}") self.od = load_detector(os.path.join(models_folder, "cifar10"))
async def load(self) -> bool: model_uri = await get_model_uri(self._settings) if not self.alibi_detect_settings.init_detector: try: self._model = load_detector(model_uri) except ( ValueError, FileNotFoundError, EOFError, pickle.UnpicklingError, ) as e: raise InvalidAlibiDetector(self._settings.name) from e else: try: detector_data = pickle.load(open(model_uri, "rb")) drift_detector = self.alibi_detect_settings.detector_type except (FileNotFoundError, EOFError, pickle.UnpicklingError) as e: raise InvalidAlibiDetector(self._settings.name) from e parameters = self.alibi_detect_settings.init_parameters self._model = drift_detector(**detector_data, **parameters) self.ready = True return self.ready
def fetch_detector(filepath: str, detector_type: str, dataset: str, detector_name: str, model: str = None) -> Data: """ Fetch an outlier or adversarial detector from a google bucket, save it locally and return the initialised detector. Parameters ---------- filepath Local directory to save detector to. detector_type `outlier` or `adversarial`. dataset Dataset of pre-trained detector. E.g. `kddcup`, `cifar10` or `ecg`. detector_name Name of the detector in the bucket. model Classification model used for adversarial detection. Returns ------- Initialised pre-trained detector. """ # create url of detector filepath = os.path.join(filepath, detector_name) if not os.path.isdir(filepath): logger.warning( 'Directory {} does not exist and is now created.'.format(filepath)) os.mkdir(filepath) url = 'https://storage.googleapis.com/seldon-models/alibi-detect/' if detector_type == 'adversarial': url = os.path.join(url, 'ad', dataset, model, detector_name) elif detector_type == 'outlier': url = os.path.join(url, 'od', detector_name, dataset) # fetch the metadata and state dict meta, state_dict = fetch_state_dict(url, filepath, save_state_dict=True) # load detector name = meta['name'] kwargs = {} if name == 'OutlierAE': fetch_ae(url, filepath) elif name == 'OutlierAEGMM': fetch_aegmm(url, filepath) elif name == 'OutlierVAE': fetch_vae(url, filepath) elif name == 'OutlierVAEGMM': fetch_vaegmm(url, filepath) elif name == 'OutlierSeq2Seq': fetch_seq2seq(url, filepath) elif name == 'AdversarialAE': fetch_ad_ae(url, filepath, state_dict) if model == 'resnet56': kwargs = {'custom_objects': {'backend': backend}} detector = load_detector(filepath, **kwargs) return detector
def load(self): """ Load the model from storage """ model_folder = kfserving.Storage.download(self.storage_uri) self.model = load_detector(model_folder) self.ready = True
def load(self): """ Load the model from storage """ model_folder = download_model(self.storage_uri) self.model: Data = load_detector(model_folder) self.ready = True
def __init__(self) -> None: import tensorflow as tf for gpu in tf.config.experimental.list_physical_devices("GPU"): tf.config.experimental.set_memory_growth(gpu, True) from alibi_detect.od.vae import OutlierVAE from alibi_detect.utils.saving import load_detector self.model = cast(OutlierVAE, load_detector(paths.vae_model)) self.model.threshold = self.THRESHOLD self.model.predict(np.random.random((1, *self.IMG_SIZE, 3)))
def test_save_load(select_detector): det = select_detector det_name = det.meta['name'] with TemporaryDirectory() as temp_dir: temp_dir += '/' save_detector(det, temp_dir) det_load = load_detector(temp_dir) det_load_name = det_load.meta['name'] assert det_load_name == det_name if not type(det_load) == OutlierProphet: assert det_load.threshold == det.threshold == threshold if type(det_load) in [AdversarialVAE, OutlierVAE, OutlierVAEGMM]: assert det_load.samples == det.samples == samples if type(det_load) == AdversarialVAE: for layer in det_load.model.layers: assert not layer.trainable if type(det_load) == OutlierAEGMM: assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.aegmm, tf.keras.Model) assert det_load.aegmm.n_gmm == n_gmm elif type(det_load) == OutlierVAEGMM: assert isinstance(det_load.vaegmm.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.vaegmm, tf.keras.Model) assert det_load.vaegmm.latent_dim == latent_dim assert det_load.vaegmm.n_gmm == n_gmm elif type(det_load) in [AdversarialVAE, OutlierVAE]: assert isinstance(det_load.vae.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vae.decoder.decoder_net, tf.keras.Sequential) assert isinstance(det_load.vae, tf.keras.Model) assert det_load.vae.latent_dim == latent_dim elif type(det_load) == Mahalanobis: assert det_load.clip is None assert det_load.mean == det_load.C == det_load.n == 0 assert det_load.meta['detector_type'] == 'online' elif type(det_load) == OutlierProphet: assert det_load.model.interval_width == .7 assert det_load.model.growth == 'logistic' assert det_load.meta['data_type'] == 'time-series' elif type(det_load) == SpectralResidual: assert det_load.window_amp == 10 assert det_load.window_local == 10
def evaluate_outlier( pedestrian_results_path: Path, model_path: Path, batch_size=1000, ) -> None: # NOTE: Speed up cli. # TODO: Try lazy loading sub commands instead. from alibi_detect.od.ae import OutlierAE from alibi_detect.utils.saving import load_detector save_dir = paths.temp_dir_path / "ae_box" / f"{int(time())}_eval_outlier" save_dir.mkdir(parents=True) model = cast(OutlierAE, load_detector(model_path)) df: pd.DataFrame = pd.read_pickle(pedestrian_results_path) df["has_pred"] = df.predictions.str.len() > 0 df["has_label"] = df.labels.str.len() > 0 chunks = np.array_split(df, len(df) // batch_size) for chunk in tqdm(chunks): with concurrent.futures.ThreadPoolExecutor( max_workers=None) as executor: boxes = list( executor.map(extract_box, chunk[chunk.has_pred].itertuples())) chunk_res = model.predict(np.stack(boxes, 0), return_feature_score=False) chunk.loc[chunk.has_pred, "instance_score"] = chunk_res["data"][ "instance_score"] # type: ignore df = cast(pd.DataFrame, pd.concat(chunks, ignore_index=True)) df.to_pickle(save_dir / "ae-box-aug-res.pkl") outlier_threshold = estimate_threshold(df) df.boxplot(column="instance_score", by="object_type").figure.savefig("box.png") df["object_color"] = color_series(df.object_type) for distance in range(0, 40, 10): plot_with_distance_threshold( df, distance, outlier_threshold, save_dir / f"scatter_{distance}.png", ) df[(df.current_distance >= distance) & (df.instance_score > outlier_threshold)].object_type.value_counts( ).to_csv(f"filtered_objects_{distance}.csv")
def test_save_load(select_detector): det = select_detector det_name = det.meta['name'] with TemporaryDirectory() as temp_dir: temp_dir += '/' save_detector(det, temp_dir) det_load = load_detector(temp_dir) det_load_name = det_load.meta['name'] assert det_load_name == det_name assert det_load.threshold == det.threshold == threshold if type(det_load) in [AdversarialVAE, OutlierVAE, OutlierVAEGMM]: assert det_load.samples == det.samples == samples if type(det_load) == AdversarialVAE: for layer in det_load.model.layers: assert not layer.trainable if type(det_load) == OutlierAEGMM: assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.aegmm, tf.keras.Model) assert det_load.aegmm.n_gmm == n_gmm elif type(det_load) == OutlierVAEGMM: assert isinstance(det_load.vaegmm.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.vaegmm, tf.keras.Model) assert det_load.vaegmm.latent_dim == latent_dim assert det_load.vaegmm.n_gmm == n_gmm elif type(det_load) in [AdversarialVAE, OutlierVAE]: assert isinstance(det_load.vae.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vae.decoder.decoder_net, tf.keras.Sequential) assert isinstance(det_load.vae, tf.keras.Model) assert det_load.vae.latent_dim == latent_dim elif type(det_load) == Mahalanobis: assert det_load.clip is None assert det_load.mean == det_load.C == det_load.n == 0 assert det_load.meta['detector_type'] == 'online'
def test_save_load(select_detector): det = select_detector det_name = det.meta['name'] # save and load functionality does not work for OutlierProphet and Python 3.6. # https://github.com/facebook/prophet/issues/1361 if sys.version_info.minor == 6 and isinstance(det, OutlierProphet): return with TemporaryDirectory() as temp_dir: temp_dir += '/' save_detector(det, temp_dir) det_load = load_detector(temp_dir) det_load_name = det_load.meta['name'] assert det_load_name == det_name if not type(det_load) in [OutlierProphet, KSDrift, MMDDrift]: assert det_load.threshold == det.threshold == threshold if type(det_load) in [OutlierVAE, OutlierVAEGMM]: assert det_load.samples == det.samples == samples if type(det_load) == AdversarialAE: for layer in det_load.model.layers: assert not layer.trainable if type(det_load) == MMDDrift: assert det_load.infer_sigma assert isinstance(det_load.permutation_test, Callable) if type(det_load) == KSDrift: assert det_load.n_features == latent_dim if type(det_load) == OutlierAEGMM: assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.aegmm, tf.keras.Model) assert det_load.aegmm.n_gmm == n_gmm elif type(det_load) == OutlierVAEGMM: assert isinstance(det_load.vaegmm.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.vaegmm, tf.keras.Model) assert det_load.vaegmm.latent_dim == latent_dim assert det_load.vaegmm.n_gmm == n_gmm elif type(det_load) in [AdversarialAE, OutlierAE]: assert isinstance(det_load.ae.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.ae.decoder.decoder_net, tf.keras.Sequential) assert isinstance(det_load.ae, tf.keras.Model) elif type(det_load) == OutlierVAE: assert isinstance(det_load.vae.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vae.decoder.decoder_net, tf.keras.Sequential) assert isinstance(det_load.vae, tf.keras.Model) assert det_load.vae.latent_dim == latent_dim elif type(det_load) == Mahalanobis: assert det_load.clip is None assert det_load.mean == det_load.C == det_load.n == 0 assert det_load.meta['detector_type'] == 'online' elif type(det_load) == OutlierProphet: assert det_load.model.interval_width == .7 assert det_load.model.growth == 'logistic' assert det_load.meta['data_type'] == 'time-series' elif type(det_load) == SpectralResidual: assert det_load.window_amp == 10 assert det_load.window_local == 10 elif type(det_load) == OutlierSeq2Seq: assert isinstance(det_load.seq2seq, tf.keras.Model) assert isinstance(det_load.seq2seq.threshold_net, tf.keras.Sequential) assert isinstance(det_load.seq2seq.encoder, EncoderLSTM) assert isinstance(det_load.seq2seq.decoder, DecoderLSTM) assert det_load.latent_dim == latent_dim assert det_load.threshold == threshold assert det_load.shape == (-1, seq_len, input_dim) elif type(det_load) in [KSDrift, MMDDrift]: assert isinstance(det_load.preprocess_fn, Callable) assert det_load.preprocess_fn.__name__ == 'uae' assert isinstance(det_load.preprocess_kwargs['encoder_net'], tf.keras.Sequential) assert det_load.p_val == p_val assert (det_load.X_ref == X_ref).all()
def test_save_load(select_detector): det = select_detector det_name = det.meta['name'] # save and load functionality does not work for OutlierProphet and Python 3.6. # https://github.com/facebook/prophet/issues/1361 if sys.version_info.minor == 6 and isinstance(det, OutlierProphet): return with TemporaryDirectory() as temp_dir: temp_dir += '/' save_detector(det, temp_dir) det_load = load_detector(temp_dir) det_load_name = det_load.meta['name'] assert det_load_name == det_name if not type(det_load) in [ OutlierProphet, ChiSquareDrift, ClassifierDrift, KSDrift, MMDDrift, TabularDrift ]: assert det_load.threshold == det.threshold == threshold if type(det_load) in [OutlierVAE, OutlierVAEGMM]: assert det_load.samples == det.samples == samples if type(det_load) == AdversarialAE or type( det_load) == ModelDistillation: for layer in det_load.model.layers: assert not layer.trainable if type(det_load) == OutlierAEGMM: assert isinstance(det_load.aegmm.encoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.aegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.aegmm, tf.keras.Model) assert det_load.aegmm.n_gmm == n_gmm elif type(det_load) == OutlierVAEGMM: assert isinstance(det_load.vaegmm.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vaegmm.decoder, tf.keras.Sequential) assert isinstance(det_load.vaegmm.gmm_density, tf.keras.Sequential) assert isinstance(det_load.vaegmm, tf.keras.Model) assert det_load.vaegmm.latent_dim == latent_dim assert det_load.vaegmm.n_gmm == n_gmm elif type(det_load) in [AdversarialAE, OutlierAE]: assert isinstance(det_load.ae.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.ae.decoder.decoder_net, tf.keras.Sequential) assert isinstance(det_load.ae, tf.keras.Model) elif type(det_load) == ModelDistillation: assert isinstance(det_load.model, tf.keras.Sequential) or isinstance( det_load.model, tf.keras.Model) assert (isinstance(det_load.distilled_model, tf.keras.Sequential) or isinstance(det_load.distilled_model, tf.keras.Model)) elif type(det_load) == OutlierVAE: assert isinstance(det_load.vae.encoder.encoder_net, tf.keras.Sequential) assert isinstance(det_load.vae.decoder.decoder_net, tf.keras.Sequential) assert isinstance(det_load.vae, tf.keras.Model) assert det_load.vae.latent_dim == latent_dim elif type(det_load) == Mahalanobis: assert det_load.clip is None assert det_load.mean == det_load.C == det_load.n == 0 assert det_load.meta['detector_type'] == 'online' elif type(det_load) == OutlierProphet: assert det_load.model.interval_width == .7 assert det_load.model.growth == 'logistic' assert det_load.meta['data_type'] == 'time-series' elif type(det_load) == SpectralResidual: assert det_load.window_amp == 10 assert det_load.window_local == 10 elif type(det_load) == OutlierSeq2Seq: assert isinstance(det_load.seq2seq, tf.keras.Model) assert isinstance(det_load.seq2seq.threshold_net, tf.keras.Sequential) assert isinstance(det_load.seq2seq.encoder, EncoderLSTM) assert isinstance(det_load.seq2seq.decoder, DecoderLSTM) assert det_load.latent_dim == latent_dim assert det_load.threshold == threshold assert det_load.shape == (-1, seq_len, input_dim) elif type(det_load) == KSDrift: assert det_load.n_features == latent_dim assert det_load.p_val == p_val assert (det_load.x_ref == X_ref).all() assert isinstance(det_load.preprocess_fn, Callable) assert det_load.preprocess_fn.func.__name__ == 'preprocess_drift' elif type(det_load) in [ChiSquareDrift, TabularDrift]: assert isinstance(det_load.x_ref_categories, dict) assert det_load.p_val == p_val x = X_ref_cat.copy() if isinstance( det_load, ChiSquareDrift) else X_ref_mix.copy() assert (det_load.x_ref == x).all() elif type(det_load) == MMDDrift: assert not det_load._detector.infer_sigma assert det_load._detector.n_permutations == n_permutations assert det_load._detector.p_val == p_val assert (det_load._detector.x_ref == X_ref).all() assert isinstance(det_load._detector.preprocess_fn, Callable) assert det_load._detector.preprocess_fn.func.__name__ == 'preprocess_drift' elif type(det_load) == ClassifierDrift: assert det_load._detector.p_val == p_val assert (det_load._detector.x_ref == X_ref).all() assert isinstance(det_load._detector.skf, StratifiedKFold) assert isinstance(det_load._detector.compile_kwargs, dict) assert isinstance(det_load._detector.train_kwargs, dict) assert isinstance(det_load._detector.model, tf.keras.Model) elif type(det_load) == LLR: assert isinstance(det_load.dist_s, tf.keras.Model) assert isinstance(det_load.dist_b, tf.keras.Model) assert not det_load.sequential assert not det_load.has_log_prob
def fetch_detector(filepath: Union[str, os.PathLike], detector_type: str, dataset: str, detector_name: str, model: str = None) -> Data: """ Fetch an outlier or adversarial detector from a google bucket, save it locally and return the initialised detector. Parameters ---------- filepath Local directory to save detector to. detector_type `outlier` or `adversarial`. dataset Dataset of pre-trained detector. E.g. `kddcup`, `cifar10` or `ecg`. detector_name Name of the detector in the bucket. model Classification model used for adversarial detection. Returns ------- Initialised pre-trained detector. """ # create path (if needed) filepath = Path(filepath) if not filepath.is_dir(): filepath.mkdir(parents=True, exist_ok=True) logger.warning( 'Directory {} does not exist and is now created.'.format(filepath)) # create url of detector url = 'https://storage.googleapis.com/seldon-models/alibi-detect/' if detector_type == 'adversarial': url = _join_url(url, ['ad', dataset, model, detector_name]) elif detector_type == 'outlier': url = _join_url(url, ['od', detector_name, dataset]) # fetch the metadata and state dict meta, state_dict = fetch_state_dict(url, filepath, save_state_dict=True) # load detector name = meta['name'] kwargs = {} # type: dict if name == 'OutlierAE': fetch_ae(url, filepath) elif name == 'OutlierAEGMM': fetch_aegmm(url, filepath) elif name == 'OutlierVAE': fetch_vae(url, filepath) elif name == 'OutlierVAEGMM': fetch_vaegmm(url, filepath) elif name == 'OutlierSeq2Seq': fetch_seq2seq(url, filepath) elif name == 'AdversarialAE': fetch_ad_ae(url, filepath, state_dict) if model == 'resnet56': kwargs = {'custom_objects': {'backend': backend}} elif name == 'ModelDistillation': fetch_ad_md(url, filepath) if model == 'resnet56': kwargs = {'custom_objects': {'backend': backend}} elif name == 'LLR': model_type = fetch_llr(url, filepath) if model_type == 'weights': kwargs = get_pixelcnn_default_kwargs() detector = load_detector(filepath, **kwargs) return detector
def load(self): self.model = load_detector(self.model_path) self.ready = True