def __init__(self, dataset: np.ndarray, categorical_indices: Optional[List[Index]] = None, feature_names: Optional[List[str]] = None) -> None: """ Constructs a ``Discretiser`` abstract class. """ # Must be overwritten in children classes self.feature_value_names = {} # type: Dict[Index, Dict[int, str]] self.feature_bin_boundaries = {} # type: Dict[Index, np.ndarray] assert _validate_input_discretiser( dataset, categorical_indices=categorical_indices, feature_names=feature_names), 'Invalid input.' self.is_structured = fuav.is_structured_array(dataset) self.dataset_dtype = dataset.dtype # Sort out column indices indices_num, indices_cat = fuat.indices_by_type(dataset) num_indices = set(indices_num) cat_indices = set(indices_cat) all_indices = num_indices.union(cat_indices) if categorical_indices is None: categorical_indices = cat_indices # type: ignore numerical_indices = num_indices else: if cat_indices.difference(categorical_indices): msg = ('Some of the string-based columns in the input dataset ' 'were not selected as categorical features via the ' 'categorical_indices parameter. String-based columns ' 'cannot be treated as numerical features, therefore ' 'they will be also treated as categorical features ' '(in addition to the ones selected with the ' 'categorical_indices parameter).') warnings.warn(msg, UserWarning) categorical_indices = cat_indices.union( # type: ignore categorical_indices) numerical_indices = all_indices.difference(categorical_indices) self.categorical_indices = sorted( list(categorical_indices)) # type: ignore self.numerical_indices = sorted(list(numerical_indices)) self.features_number = len(all_indices) if self.is_structured: indices = self.dataset_dtype.names else: indices = range(self.features_number) if feature_names is None: feature_names_map = {x: str(x) for x in indices} else: feature_names_map = dict(zip(indices, feature_names)) self.feature_names_map = feature_names_map
def __init__(self, data_set: np.ndarray, categorical_indices: Optional[List[Index]] = None, neighbours: int = 7, distance_function: Optional[DistanceFunction] = None, normalise_scores: bool = True) -> None: """ Initialises the ``DensityCheck`` class. """ # pylint: disable=too-many-arguments assert _validate_input_dc(data_set, categorical_indices, neighbours, distance_function, normalise_scores), 'Invalid input.' self.data_set = data_set self._is_structured = fuav.is_structured_array(self.data_set) # self.neighbours = neighbours if distance_function is None: if not _NUMPY_1_14 and self._is_structured: distance_function = self._mixed_distance_o # pragma: nocover else: distance_function = self._mixed_distance_n self._distance_function = distance_function # type: ignore # self.normalise_scores = normalise_scores # Sort out column indices feature_indices = fuat.indices_by_type(self.data_set) num_indices = set(feature_indices[0]) cat_indices = set(feature_indices[1]) all_indices = num_indices.union(cat_indices) if categorical_indices is None: _categorical_indices = cat_indices _numerical_indices = num_indices else: if cat_indices.difference(categorical_indices): msg = ('Some of the string-based columns in the input data ' 'set were not selected as categorical features via the ' 'categorical_indices parameter. String-based columns ' 'cannot be treated as numerical features, therefore ' 'they will be also treated as categorical features ' '(in addition to the ones selected with the ' 'categorical_indices parameter).') warnings.warn(msg, UserWarning) _categorical_indices = cat_indices.union(categorical_indices) else: _categorical_indices = categorical_indices # type: ignore _numerical_indices = all_indices.difference(_categorical_indices) self._categorical_indices = sorted(list(_categorical_indices)) self._numerical_indices = sorted(list(_numerical_indices)) self._samples_number = self.data_set.shape[0] self.distance_matrix = fud.get_distance_matrix(self.data_set, self._distance_function) assert self._samples_number == self.distance_matrix.shape[0] assert self.distance_matrix.shape[0] == self.distance_matrix.shape[1] self.scores = self._compute_scores() assert self._samples_number == self.scores.shape[0] self.scores_min = self.scores.min() self.scores_max = self.scores.max() if self.normalise_scores: if self.scores_min == self.scores_max: assert (self.scores == self.scores_min).all(), \ 'All distances/scores are equal.' self.scores[:] = 0 else: self.scores -= self.scores_min self.scores /= self.scores_max - self.scores_min
def __init__(self, dataset: np.ndarray, ground_truth: Optional[np.ndarray] = None, categorical_indices: Optional[np.ndarray] = None, int_to_float: bool = True) -> None: """ Constructs an ``Augmentation`` abstract class. """ # pylint: disable=too-many-locals assert _validate_input(dataset, ground_truth=ground_truth, categorical_indices=categorical_indices, int_to_float=int_to_float), 'Invalid input.' self.dataset = dataset self.data_points_number = dataset.shape[0] self.is_structured = fuav.is_structured_array(dataset) self.ground_truth = ground_truth # Sort out column indices indices = fuat.indices_by_type(dataset) num_indices = set(indices[0]) cat_indices = set(indices[1]) all_indices = num_indices.union(cat_indices) if categorical_indices is None: categorical_indices = cat_indices numerical_indices = num_indices else: if cat_indices.difference(categorical_indices): msg = ('Some of the string-based columns in the input dataset ' 'were not selected as categorical features via the ' 'categorical_indices parameter. String-based columns ' 'cannot be treated as numerical features, therefore ' 'they will be also treated as categorical features ' '(in addition to the ones selected with the ' 'categorical_indices parameter).') warnings.warn(msg, UserWarning) categorical_indices = cat_indices.union(categorical_indices) numerical_indices = all_indices.difference(categorical_indices) self.categorical_indices = sorted(list(categorical_indices)) self.numerical_indices = sorted(list(numerical_indices)) self.features_number = len(all_indices) # Sort out the dtype of the sampled array. ntype = np.dtype(np.float64) if int_to_float else np.dtype(np.int64) if self.is_structured: sample_dtype = [] for column_name in self.dataset.dtype.names: if column_name in self.numerical_indices: new_dtype = fuat.generalise_dtype( self.dataset.dtype[column_name], ntype) sample_dtype.append((column_name, new_dtype)) elif column_name in self.categorical_indices: sample_dtype.append( (column_name, self.dataset.dtype[column_name])) else: assert False, 'Unknown column name.' # pragma: nocover else: if fuav.is_numerical_array(self.dataset): sample_dtype = fuat.generalise_dtype(self.dataset.dtype, ntype) else: sample_dtype = self.dataset.dtype self.sample_dtype = sample_dtype
def fit(self, X: np.ndarray, y: np.ndarray) -> None: """ Fits the model. Parameters ---------- X : numpy.ndarray The KNN training data. y : numpy.ndarray The KNN training labels. Raises ------ IncorrectShapeError Either the ``X`` array is not 2-dimensional, the ``y`` array is not 1-dimensional, the number of rows in ``X`` is not the same as the number of elements in ``y`` or the ``X`` array has 0 rows or 0 columns. PrefittedModelError Trying to fit the model when it has already been fitted. Usually raised when calling the ``fit`` method for the second time without clearing the model first. TypeError Trying to fit a KNN predictor in a regressor mode with non-numerical target variable. """ if self._is_fitted: raise PrefittedModelError('This model has already been fitted.') if not fuav.is_2d_array(X): raise IncorrectShapeError('The training data must be a 2-' 'dimensional array.') if not fuav.is_1d_array(y): raise IncorrectShapeError('The training data labels must be a 1-' 'dimensional array.') if X.shape[0] == 0: raise IncorrectShapeError('The data array has to have at least ' 'one data point.') # If the array is structured the fuav.is_2d_array function takes care # of checking whether there is at least one column if not fuav.is_structured_array(X) and X.shape[1] == 0: raise IncorrectShapeError('The data array has to have at least ' 'one feature.') if X.shape[0] != y.shape[0]: raise IncorrectShapeError('The number of samples in X must be the ' 'same as the number of labels in y.') if not self._is_classifier and not fuav.is_numerical_array(y): raise TypeError('Regressor can only be fitted for a numerical ' 'target vector.') numerical_indices, categorical_indices = fuat.indices_by_type(X) self._numerical_indices = numerical_indices self._categorical_indices = categorical_indices self._is_structured = fuav.is_structured_array(X) self._X = X self._y = y if self._is_classifier: unique_y, unique_y_counts = np.unique(self._y, return_counts=True) # Order labels lexicographically. unique_y_sort_index = np.argsort(unique_y) self._unique_y = unique_y[unique_y_sort_index] self._unique_y_counts = unique_y_counts[unique_y_sort_index] # How many other labels have the same count. top_y_index = self._unique_y_counts == np.max( self._unique_y_counts) top_y_unique_sorted = np.sort(self._unique_y[top_y_index]) self._majority_label = top_y_unique_sorted[0] self._unique_y_probabilities = (self._unique_y_counts / self._y.shape[0]) else: self._majority_label = self._y.mean() self._unique_y = np.ndarray((0, )) self._unique_y_counts = np.ndarray((0, )) self._unique_y_probabilities = np.ndarray((0, )) self._X_n = self._X.shape[0] self._is_fitted = True
def describe_array( array: np.ndarray, include: Optional[Union[str, int, List[Union[str, int]]]] = None, exclude: Optional[Union[str, int, List[Union[str, int]]]] = None, **kwargs: bool ) -> Dict[Union[str, int], Union[str, int, float, bool, np.ndarray, Dict[str, Union[str, int, float, bool, np.ndarray]]] ]: # yapf: disable """ Describes categorical (textual) and numerical columns in the input array. The details of numerical and categorical descriptions can be found in :func:`fatf.transparency.data.describe_functions.describe_numerical_array` and :func:`fatf.transparency.data.describe_functions.\ describe_categorical_array` functions documentation respectively. To filter out the columns that will be described you can use ``include`` and ``exclude`` parameters. Either of these can be a list with columns indices, a string or an integer when excluding or including just one column; or one of the keywords: ``'numerical'`` or ``'categorical'``, to indicate that only numerical or categorical columns should be included/ excluded. By default all columns are described. Parameters ---------- array : numpy.ndarray The array to be described. include : Union[str, int, List[Union[str, int]]], optional (default=None) A list of column indices to be included in the description. If ``None`` (the default value), all of the columns will be included. Alternatively this can be set to a single index (either a string or an integer) to compute statistics just for this one column. It is also possible to set it to ``'numerical'`` or ``'categorical'`` to just include numerical or categorical columns respectively. exclude : Union[str, int, List[Union[str, int]]], optional (default=None) A list of column indices to be excluded from the description. If ``None`` (the default value), none of the columns will be excluded. Alternatively this can be set to a single index (either a string or an integer) to exclude just one column. It is also possible to set it to ``'numerical'`` or ``'categorical'`` to exclude wither all numerical or all categorical columns respectively. **kwargs : bool Keyword arguments that are passed to the :func:`fatf.transparency.\ data.describe_functions.describe_numerical_array` function responsible for describing numerical arrays. Warns ----- UserWarning When using ``include`` or ``exclude`` parameters for 1-dimensional input arrays (in which case these parameters are ignored). Raises ------ IncorrectShapeError The input array is neither 1- not 2-dimensional. RuntimeError None of the columns were selected to be described. ValueError The input array is not of a base type (textual and numerical elements). The input array has 0 columns. Returns ------- description : Dict[Union[str, int], Dict[str, \ Union[str, int, float bool, np.ndarray]]] For 2-dimensional arrays a dictionary describing every column under a key corresponding to its index in the input array. For a 1-dimensional input array a dictionary describing that array. """ # pylint: disable=too-many-locals,too-many-branches is_1d = fuav.is_1d_like(array) if is_1d: array = fuat.as_unstructured(array) is_2d = False else: is_2d = fuav.is_2d_array(array) if not is_1d and not is_2d: raise IncorrectShapeError('The input array should be 1- or ' '2-dimensional.') if not fuav.is_base_array(array): raise ValueError('The input array should be of a base type (a mixture ' 'of numerical and textual types).') if is_1d: if include is not None or exclude is not None: warnings.warn( 'The input array is 1-dimensional. Ignoring include and ' 'exclude parameters.', category=UserWarning) if fuav.is_numerical_array(array): description = describe_numerical_array(array, **kwargs) elif fuav.is_textual_array(array): description = describe_categorical_array(array) else: # pragma: no cover assert False, 'A base array should either be numerical or textual.' elif is_2d: numerical_indices, categorical_indices = fuat.indices_by_type(array) is_structured_array = fuav.is_structured_array(array) if (numerical_indices.shape[0] + categorical_indices.shape[0]) == 0: raise ValueError('The input array cannot have 0 columns.') numerical_indices_set = set(numerical_indices) categorical_indices_set = set(categorical_indices) all_indices = categorical_indices_set.union(numerical_indices_set) # Indices to be included include_indices = _filter_include_indices(categorical_indices_set, numerical_indices_set, include, all_indices) categorical_indices_set, numerical_indices_set = include_indices # Indices to be included exclude_indices = _filter_exclude_indices(categorical_indices_set, numerical_indices_set, exclude, all_indices) categorical_indices_set, numerical_indices_set = exclude_indices all_indices = numerical_indices_set.union(categorical_indices_set) if len(all_indices) == 0: # pylint: disable=len-as-condition raise RuntimeError('None of the columns were selected to be ' 'described.') description = dict() for idx in numerical_indices_set: if is_structured_array: description[idx] = describe_numerical_array( # type: ignore array[idx], **kwargs) else: description[idx] = describe_numerical_array( # type: ignore array[:, idx], **kwargs) for idx in categorical_indices_set: if is_structured_array: description[idx] = describe_categorical_array( # type: ignore array[idx]) else: description[idx] = describe_categorical_array( # type: ignore array[:, idx]) else: # pragma: no cover assert False, 'The input array can only be 1- or 2-dimensional.' return description # type: ignore
def test_indices_by_type(): """ Tests :func:`fatf.utils.array.tools.indices_by_type` function. """ # pylint: disable=too-many-locals,too-many-statements # Test any object and shape type_error = 'The input should be a numpy array-like.' incorrect_shape_error = 'The input array should be 2-dimensional.' value_error = ('indices_by_type only supports input arrays that hold base ' 'numpy types, i.e. numerical and string-like -- numpy void ' 'and object-like types are not allowed.') with pytest.raises(TypeError) as exin: fuat.indices_by_type(None) assert str(exin.value) == type_error with pytest.raises(IncorrectShapeError) as exin: fuat.indices_by_type(np.empty((0, ))) assert str(exin.value) == incorrect_shape_error with pytest.raises(ValueError) as exin: fuat.indices_by_type(NOT_NUMERICAL_NP_ARRAY) assert str(exin.value) == value_error # Empty array i_n, i_c = fuat.indices_by_type(np.empty((22, 0))) assert np.array_equal([], i_n) assert np.array_equal([], i_c) # All numerical array array_all_numerical = np.ones((22, 4)) array_all_numerical_indices_numerical = np.array([0, 1, 2, 3]) array_all_numerical_indices_categorical = np.array([], dtype=int) i_n, i_c = fuat.indices_by_type(array_all_numerical) assert np.array_equal(array_all_numerical_indices_numerical, i_n) assert np.array_equal(array_all_numerical_indices_categorical, i_c) # All categorical -- single type -- array array_all_categorical = np.ones((22, 4), dtype='U4') array_all_categorical_indices_numerical = np.array([]) array_all_categorical_indices_categorical = np.array([0, 1, 2, 3]) i_n, i_c = fuat.indices_by_type(array_all_categorical) assert np.array_equal(array_all_categorical_indices_numerical, i_n) assert np.array_equal(array_all_categorical_indices_categorical, i_c) # Mixture array array_mixture_1 = np.ones((22, ), dtype=[('a', 'U4'), ('b', 'U4'), ('c', 'U4'), ('d', 'U4')]) # yapf: disable array_mixture_1_indices_numerical = np.array([]) array_mixture_1_indices_categorical = np.array(['a', 'b', 'c', 'd'], dtype='U1') #### i_n, i_c = fuat.indices_by_type(array_mixture_1) assert np.array_equal(array_mixture_1_indices_numerical, i_n) assert np.array_equal(array_mixture_1_indices_categorical, i_c) array_mixture_2 = np.ones((22, ), dtype=[('a', 'U4'), ('b', 'f'), ('c', 'U4'), ('d', int)]) # yapf: disable array_mixture_2_indices_numerical = np.array(['b', 'd'], dtype='U1') array_mixture_2_indices_categorical = np.array(['a', 'c'], dtype='U1') i_n, i_c = fuat.indices_by_type(array_mixture_2) assert np.array_equal(array_mixture_2_indices_numerical, i_n) assert np.array_equal(array_mixture_2_indices_categorical, i_c) glob_indices_numerical = np.array([0, 1]) glob_indices_categorical = np.array([]) i_n, i_c = fuat.indices_by_type(NUMERICAL_NP_ARRAY) assert np.array_equal(glob_indices_numerical, i_n) assert np.array_equal(glob_indices_categorical, i_c) # glob_indices_numerical = np.array([0, 1, 2]) glob_indices_categorical = np.array([]) i_n, i_c = fuat.indices_by_type(WIDE_NP_ARRAY) assert np.array_equal(glob_indices_numerical, i_n) assert np.array_equal(glob_indices_categorical, i_c) # glob_indices_numerical = np.array(['numbers', 'complex']) glob_indices_categorical = np.array([]) i_n, i_c = fuat.indices_by_type(NUMERICAL_STRUCTURED_ARRAY) assert np.array_equal(glob_indices_numerical, i_n) assert np.array_equal(glob_indices_categorical, i_c) # glob_indices_numerical = np.array(['numerical']) glob_indices_categorical = np.array(['categorical']) i_n, i_c = fuat.indices_by_type(NOT_NUMERICAL_STRUCTURED_ARRAY) assert np.array_equal(glob_indices_numerical, i_n) assert np.array_equal(glob_indices_categorical, i_c) # glob_indices_numerical = np.array(['numbers', 'complex', 'anybody']) glob_indices_categorical = np.array([]) i_n, i_c = fuat.indices_by_type(WIDE_STRUCTURED_ARRAY) assert np.array_equal(glob_indices_numerical, i_n) assert np.array_equal(glob_indices_categorical, i_c)