def test_mmd(mmd_params): n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref = mmd_params np.random.seed(0) torch.manual_seed(0) x_ref = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) preprocess_fn, preprocess_kwargs = preprocess to_list = False if hasattr(preprocess_fn, '__name__') and preprocess_fn.__name__ == 'preprocess_list': if not preprocess_x_ref: return to_list = True x_ref = [_[None, :] for _ in x_ref] elif isinstance(preprocess_fn, Callable) and 'layer' in list(preprocess_kwargs.keys()) \ and preprocess_kwargs['model'].__name__ == 'HiddenOutput': model = MyModel(n_features) layer = preprocess_kwargs['layer'] preprocess_fn = partial(preprocess_fn, model=HiddenOutput(model=model, layer=layer)) else: preprocess_fn = None cd = MMDDriftTorch( x_ref=x_ref, p_val=.05, preprocess_x_ref=preprocess_x_ref if isinstance(preprocess_fn, Callable) else False, update_x_ref=update_x_ref, preprocess_fn=preprocess_fn, n_permutations=n_permutations ) x = x_ref.copy() preds = cd.predict(x, return_p_val=True) assert preds['data']['is_drift'] == 0 and preds['data']['p_val'] >= cd.p_val if isinstance(update_x_ref, dict): k = list(update_x_ref.keys())[0] assert cd.n == len(x) + len(x_ref) assert cd.x_ref.shape[0] == min(update_x_ref[k], len(x) + len(x_ref)) x_h1 = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) if to_list: x_h1 = [_[None, :] for _ in x_h1] preds = cd.predict(x_h1, return_p_val=True) if preds['data']['is_drift'] == 1: assert preds['data']['p_val'] < preds['data']['threshold'] == cd.p_val assert preds['data']['distance'] > preds['data']['distance_threshold'] else: assert preds['data']['p_val'] >= preds['data']['threshold'] == cd.p_val assert preds['data']['distance'] <= preds['data']['distance_threshold']
def __init__(self, x_ref: Union[np.ndarray, list], backend: str = 'tensorflow', p_val: float = .05, preprocess_x_ref: bool = True, update_x_ref: Optional[Dict[str, int]] = None, preprocess_fn: Optional[Callable] = None, kernel: Callable = None, sigma: Optional[np.ndarray] = None, configure_kernel_from_x_ref: bool = True, n_permutations: int = 100, device: Optional[str] = None, input_shape: Optional[tuple] = None, data_type: Optional[str] = None) -> None: """ Maximum Mean Discrepancy (MMD) data drift detector using a permutation test. Parameters ---------- x_ref Data used as reference distribution. backend Backend used for the MMD implementation. p_val p-value used for the significance of the permutation test. preprocess_x_ref Whether to already preprocess and store the reference data. update_x_ref Reference data can optionally be updated to the last n instances seen by the detector or via reservoir sampling with size n. For the former, the parameter equals {'last': n} while for reservoir sampling {'reservoir_sampling': n} is passed. preprocess_fn Function to preprocess the data before computing the data drift metrics. kernel Kernel used for the MMD computation, defaults to Gaussian RBF kernel. sigma Optionally set the GaussianRBF kernel bandwidth. Can also pass multiple bandwidth values as an array. The kernel evaluation is then averaged over those bandwidths. configure_kernel_from_x_ref Whether to already configure the kernel bandwidth from the reference data. n_permutations Number of permutations used in the permutation test. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. input_shape Shape of input data. data_type Optionally specify the data type (tabular, image or time-series). Added to metadata. """ super().__init__() backend = backend.lower() if backend == 'tensorflow' and not has_tensorflow or backend == 'pytorch' and not has_pytorch: raise ImportError( f'{backend} not installed. Cannot initialize and run the ' f'MMDDrift detector with {backend} backend.') elif backend not in ['tensorflow', 'pytorch']: raise NotImplementedError( f'{backend} not implemented. Use tensorflow or pytorch instead.' ) kwargs = locals() args = [kwargs['x_ref']] pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] [kwargs.pop(k, None) for k in pop_kwargs] if kernel is None: if backend == 'tensorflow': from alibi_detect.utils.tensorflow.kernels import GaussianRBF else: from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore kwargs.update({'kernel': GaussianRBF}) if backend == 'tensorflow' and has_tensorflow: kwargs.pop('device', None) self._detector = MMDDriftTF(*args, **kwargs) # type: ignore else: self._detector = MMDDriftTorch(*args, **kwargs) # type: ignore self.meta = self._detector.meta