def test_list_lengths(self): list_lengths = arrow_util.ListLengthsFromListArray( pa.array([], type=pa.list_(pa.int64()))) self.assertTrue(list_lengths.equals(pa.array([], type=pa.int32()))) list_lengths = arrow_util.ListLengthsFromListArray( pa.array([[1., 2.], [], [3.]])) self.assertTrue(list_lengths.equals(pa.array([2, 0, 1], type=pa.int32()))) list_lengths = arrow_util.ListLengthsFromListArray( pa.array([[1., 2.], None, [3.]])) self.assertTrue(list_lengths.equals(pa.array([2, 0, 1], type=pa.int32())))
def update(self, feature_column, feature_type, num_values_quantiles_combiner, weight_column=None): """Update the partial common statistics using the input value.""" # All the values in this column is null and we cannot deduce the type of # the feature. This is not an error as this feature might have some values # in other batches. if feature_type is None: return if self.type is None: self.type = feature_type elif self.type != feature_type: raise TypeError('Cannot determine the type of feature %s. ' 'Found values of types %s and %s.' % (feature_column.name, self.type, feature_type)) # np.max / np.min below cannot handle empty arrays. And there's nothing # we can collect in this case. if not feature_column: return if weight_column and (feature_column.data.num_chunks != weight_column.data.num_chunks): raise ValueError( 'Expected the feature column {} and weight column {} to have the ' 'same number of chunks.'.format(feature_column.name, weight_column.name)) weight_chunks = weight_column.data.iterchunks( ) if weight_column else [] for feature_array, weight_array in six.moves.zip_longest( feature_column.data.iterchunks(), weight_chunks, fillvalue=None): num_values = arrow_util.ListLengthsFromListArray( feature_array).to_numpy() none_mask = arrow_util.GetArrayNullBitmapAsByteArray( feature_array).to_numpy().view(np.bool) num_values_not_none = num_values[~none_mask] self.num_non_missing += len( feature_array) - feature_array.null_count self.max_num_values = max(np.max(num_values_not_none), self.max_num_values) self.min_num_values = min(np.min(num_values_not_none), self.min_num_values) self.total_num_values += np.sum(num_values_not_none) self.num_values_summary = num_values_quantiles_combiner.add_input( self.num_values_summary, [num_values_not_none]) if weight_array: weights = (arrow_util.FlattenListArray( weight_array).to_numpy().astype(np.float32, copy=False)) if weights.size != num_values.size: raise ValueError('Weight feature must not be missing.') self.weighted_total_num_values += np.sum(num_values * weights) self.weighted_num_non_missing += np.sum(weights[~none_mask])
def _flatten_and_impute( examples_table: pa.Table, categorical_features: Set[types.FeaturePath] ) -> Dict[types.FeaturePath, np.ndarray]: """Flattens and imputes the values in the input Arrow table. Replaces missing values with CATEGORICAL_FEATURE_IMPUTATION_FILL_VALUE for categorical features and 10*max(feature_values) for numeric features. We impute missing values with an extreme value that is far from observed values so it does not incorrectly impact KNN results. 10*max(feature_values) is used instead of sys.max_float because max_float is large enough to cause unexpected float arithmetic errors. Args: examples_table: Arrow table containing a batch of examples where all features are univalent. categorical_features: Set of categorical feature names. Returns: A Dict[FeaturePath, np.ndarray] where the key is the feature path and the value is a 1D numpy array corresponding to the feature values. """ num_rows = examples_table.num_rows result = {} for feature_column in examples_table.itercolumns(): feature_path = types.FeaturePath([feature_column.name]) # Assume we have only a single chunk. feature_array = feature_column.data.chunk(0) # to_pandas returns a readonly array. Create a copy as we will be imputing # the NaN values. non_missing_values = np.copy( arrow_util.primitive_array_to_numpy(feature_array.flatten())) non_missing_parent_indices = arrow_util.primitive_array_to_numpy( arrow_util.GetFlattenedArrayParentIndices(feature_array)) is_categorical_feature = feature_path in categorical_features result_dtype = non_missing_values.dtype if non_missing_parent_indices.size < num_rows and is_categorical_feature: result_dtype = np.object flattened_array = np.ndarray(shape=num_rows, dtype=result_dtype) num_values = arrow_util.primitive_array_to_numpy( arrow_util.ListLengthsFromListArray(feature_array)) missing_parent_indices = np.where(num_values == 0)[0] if feature_path in categorical_features: imputation_fill_value = CATEGORICAL_FEATURE_IMPUTATION_FILL_VALUE else: # Also impute any NaN values. nan_mask = np.isnan(non_missing_values) imputation_fill_value = sys.maxsize if not np.all(nan_mask): imputation_fill_value = non_missing_values[~nan_mask].max( ) * 10 non_missing_values[nan_mask.nonzero()[0]] = imputation_fill_value flattened_array[non_missing_parent_indices] = non_missing_values if missing_parent_indices.any(): flattened_array[missing_parent_indices] = imputation_fill_value result[feature_path] = flattened_array return result
def update(self, feature_path: types.FeaturePath, feature_array: pa.Array, feature_type: types.FeatureNameStatisticsType, num_values_quantiles_combiner: Any, weights: Optional[np.ndarray] = None) -> None: """Update the partial common statistics using the input value.""" # All the values in this column is null and we cannot deduce the type of # the feature. This is not an error as this feature might have some values # in other batches. if feature_type is None: return if self.type is None: self.type = feature_type elif self.type != feature_type: raise TypeError('Cannot determine the type of feature %s. ' 'Found values of types %s and %s.' % (feature_path, self.type, feature_type)) # np.max / np.min below cannot handle empty arrays. And there's nothing # we can collect in this case. if not feature_array: return num_values = arrow_util.primitive_array_to_numpy( arrow_util.ListLengthsFromListArray(feature_array)) none_mask = arrow_util.primitive_array_to_numpy( arrow_util.GetArrayNullBitmapAsByteArray(feature_array)).view( np.bool) self.num_non_missing += len(feature_array) - feature_array.null_count num_values_not_none = num_values[~none_mask] # We do this check to avoid failing in np.min/max with empty array. if num_values_not_none.size == 0: return # Use np.maximum.reduce(num_values_not_none, initial=self.max_num_values) # once we upgrade to numpy 1.16 self.max_num_values = max(np.max(num_values_not_none), self.max_num_values) self.min_num_values = min(np.min(num_values_not_none), self.min_num_values) self.total_num_values += np.sum(num_values_not_none) self.num_values_summary = num_values_quantiles_combiner.add_input( self.num_values_summary, [num_values_not_none]) if weights is not None: if weights.size != num_values.size: raise ValueError('Weight feature must not be missing.') self.weighted_total_num_values += np.sum(num_values * weights) self.weighted_num_non_missing += np.sum(weights[~none_mask])
def _get_univalent_values_with_parent_indices( self, examples_table: pa.Table) -> Dict[types.FeatureName, pd.DataFrame]: """Extracts univalent values for each feature along with parent indices.""" result = {} for feature_column in examples_table.itercolumns(): feature_name = feature_column.name if (self._features_needed is not None and feature_name not in self._features_needed): continue feature_type = stats_util.get_feature_type_from_arrow_type( feature_name, feature_column.type) # Only consider crosses of numeric features. # TODO(zhuo): Support numeric features nested under structs. if feature_type in (statistics_pb2.FeatureNameStatistics.STRING, statistics_pb2.FeatureNameStatistics.STRUCT): continue # Assume we have only a single chunk. assert feature_column.data.num_chunks == 1 feat_arr = feature_column.data.chunk(0) value_lengths = arrow_util.primitive_array_to_numpy( arrow_util.ListLengthsFromListArray(feat_arr)) univalent_parent_indices = set((value_lengths == 1).nonzero()[0]) # If there are no univalent values, continue to the next feature. if not univalent_parent_indices: continue non_missing_values = arrow_util.primitive_array_to_numpy( feat_arr.flatten()) value_parent_indices = arrow_util.primitive_array_to_numpy( arrow_util.GetFlattenedArrayParentIndices(feat_arr)) if feature_type == statistics_pb2.FeatureNameStatistics.FLOAT: # Remove any NaN values if present. non_nan_mask = ~np.isnan(non_missing_values) non_missing_values = non_missing_values[non_nan_mask] value_parent_indices = value_parent_indices[non_nan_mask] df = pd.DataFrame({ feature_name: non_missing_values, 'parent_index': value_parent_indices }) # Only keep the univalent feature values. df = df[df['parent_index'].isin(univalent_parent_indices)] result[feature_name] = df return result
def add_input( self, accumulator: Dict[types.FeaturePath, _PartialSparseFeatureStats], input_table: pa.Table ) -> Dict[types.FeaturePath, _PartialSparseFeatureStats]: """Returns result of folding a batch of inputs into the current accumulator. Args: accumulator: The current accumulator. input_table: An Arrow Table whose columns are features and rows are examples. Returns: The accumulator after updating the statistics for the batch of inputs. """ component_feature_value_list_lengths = dict() component_feature_num_missing = dict() batch_example_count = input_table.num_rows # Do a single pass through the input table to determine the value list # lengths and number missing for every feature that is an index or value # feature in any sparse feature in the schema. for feature_path, leaf_array, _ in arrow_util.enumerate_arrays( input_table, weight_column=None, enumerate_leaves_only=True): if (feature_path in self._all_index_feature_paths or feature_path in self._all_value_feature_paths): # If the column is a NullArray, skip it when populating the # component_feature_ dicts. Features that are missing from those dicts # are treated as entirely missing for the batch. if not pa.types.is_null(leaf_array.type): component_feature_value_list_lengths[ feature_path] = arrow_util.primitive_array_to_numpy( arrow_util.ListLengthsFromListArray(leaf_array)) component_feature_num_missing[ feature_path] = leaf_array.null_count # Now create a partial sparse feature stats object for each sparse feature # using the value list lengths and numbers missing information collected # above. for feature_path in self._sparse_feature_component_paths: value_feature_path = self._sparse_feature_component_paths[ feature_path].value_feature index_feature_paths = self._sparse_feature_component_paths[ feature_path].index_features missing_value_count = component_feature_num_missing.get( value_feature_path) # If this batch does not have the value feature at all, # missing_value_count is the number of examples in the batch. # Also populate the value list lengths for the value feature with all 0s # since a missing feature is considered to have a value list length of 0. if missing_value_count is None: missing_value_count = batch_example_count component_feature_value_list_lengths[ value_feature_path] = np.full(batch_example_count, 0) missing_index_counts = collections.Counter() min_length_diff = dict() max_length_diff = dict() for index_feature_path in index_feature_paths: missing_index_count = component_feature_num_missing.get( index_feature_path) # If this batch does not have this index feature at all, # missing_index_count for that index feature is the number of # examples in the batch. # Also populate the value list lengths for the index feature with all 0s # since a missing feature is considered to have a value list length of # 0. if missing_index_count is None: missing_index_counts[ index_feature_path] = batch_example_count component_feature_value_list_lengths[ index_feature_path] = np.full(batch_example_count, 0) else: missing_index_counts[ index_feature_path] = missing_index_count length_differences = np.subtract( component_feature_value_list_lengths[index_feature_path], component_feature_value_list_lengths[value_feature_path]) min_length_diff[index_feature_path] = np.min( length_differences) max_length_diff[index_feature_path] = np.max( length_differences) stats_for_feature = _PartialSparseFeatureStats( missing_value_count, missing_index_counts, min_length_diff, max_length_diff) existing_stats_for_feature = accumulator.get(feature_path) if existing_stats_for_feature is None: accumulator[feature_path] = stats_for_feature else: accumulator[feature_path] += stats_for_feature return accumulator