def _compute_tensor_to_feature_dict(input_fn, feature_columns, dtype): """Computes a feature_name-to-tensor dict for the given features. Args: input_fn: See the same argument in 'save_quantiles_for_keypoints'. feature_columns: See the same argument in 'save_quantiles_for_keypoints'. dtype: See the same argument in 'save_quantiles_for_keypoints'. Returns: A str->tensor dict mapping each feature name to the tensor containing its feature values for the current batch. The dict contains all the features returned by input_fn if feature_columns are none, or only those features included in 'feature_columns', otherwise. If a non-None label is returned by 'input_fn', it will also be included in the dict. """ if feature_columns is not None: transformed_columns_to_tensors, label = input_fn() features_to_tensors = { f_col.name: tools.input_from_feature_column(transformed_columns_to_tensors, f_col, dtype) for f_col in feature_columns } else: features_to_tensors, label = input_fn() if label is None: return features_to_tensors if _LABEL_FEATURE_NAME in features_to_tensors: raise ValueError( ("Can't save a label as there's already a feature named: '%s'." " Try renaming that feature. ") % _LABEL_FEATURE_NAME) features_to_tensors[_LABEL_FEATURE_NAME] = label return features_to_tensors
def _materialize_feature_column(self, feature_column, x): """Creates input_fn with x then transform and materialize feature_column.""" input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={'x': x}, y=None, batch_size=self._test_data.num_examples, num_epochs=1, shuffle=False) with tf.Graph().as_default(): features = input_fn() input_tensor = tools.input_from_feature_column(features, feature_column) materialized = self._materialize_locally(input_tensor) return materialized
def input_calibration_layer(columns_to_tensors, num_keypoints, feature_columns=None, keypoints_initializers=None, keypoints_initializer_fns=None, bound=False, monotonic=None, missing_input_values=None, missing_output_values=None, dtype=dtypes.float32, **regularizer_amounts): """Creates a calibration layer for the given input and feature_columns. Returns a tensor with the calibrated values of the given features, a list of the names of the features in the order they feature in the returned, and a list of projection ops, that must be applied at each step (or every so many steps) to project the model to a feasible space: used for bounding the outputs or for imposing monotonic -- the list will be empty if bound and monotonic are not set. Args: columns_to_tensors: A mapping from feature name to tensors. 'string' key means a base feature (not-transformed). If feature_columns is not set these are the features calibrated. Otherwise the transformed feature_columns are the ones calibrated. num_keypoints: Number of keypoints to use. Either a single int, or a dict mapping feature names to num_keypoints. If a value of the dict is 0 or None the correspondent feature won't be calibrated. feature_columns: Optional. If set to a set of FeatureColumns, these will be the features used and calibrated. keypoints_initializers: For evaluation or inference (or when resuming training from a checkpoint) the values will be loaded from disk, so they don't need to be given (leave it as None). Either a tuple of two tensors of shape [num_keypoints], or a dict mapping feature names to pair of tensors of shape [num_keypoints[feature_name]]. See load_keypoints_from_quantiles or uniform_keypoints_for_signal on how to generate these (module keypoints_initialization). keypoints_initializer_fns: Like keypoints_initializers but using lambda initializers. They should be compatible with tf.get_variable. If this is set, then keypoints_initializers must be None. bound: boolean whether output of calibration must be bound. Alternatively a dict mapping feature name to boundness. monotonic: whether calibration has to be kept monotonic: None or 0 means no monotonic. Positive or negative values mean increasing or decreasing monotonic respectively. Alternatively a dict mapping feature name to monotonic. missing_input_values: If set, and if the input has this value it is assumed to be missing and the output will either be calibrated to some value between `[calibration_output_min, calibration_output_max]` or set to a fixed value set by missing_output_value. Limitation: it only works for scalars. Either one value for all inputs, or a dict mapping feature name to missing_input_value for the respective feature. missing_output_values: Requires missing_input_value also to be set. If set if will convert missing input to this value. Either one value for all inputs, or a dict mapping feature name to missing_input_value for the respective feature. dtype: If any of the scalars are not given as tensors, they are converted to tensors with this dtype. **regularizer_amounts: Keyword args of regularization amounts passed to regularizers.calibrator_regularization(). Keyword names should be among supported regularizers.CALIBRATOR_REGULARIZERS and values should be either float or {feature_name: float}. If float, then same value is applied to all features. Returns: A tuple of: * calibrated tensor of shape [batch_size, sum(features dimensions)]. * list of the feature names in the order they feature in the calibrated tensor. A name may appear more than once if the feature is multi-dimension (for instance a multi-dimension embedding) * list of projection ops, that must be applied at each step (or every so many steps) to project the model to a feasible space: used for bounding the outputs or for imposing monotonicity. Empty if none are requested. * None or tensor with regularization loss. Raises: ValueError: if dtypes are incompatible. """ with ops.name_scope('input_calibration_layer'): feature_names = tools.get_sorted_feature_names(columns_to_tensors, feature_columns) num_keypoints = tools.cast_to_dict(num_keypoints, feature_names, 'num_keypoints') bound = tools.cast_to_dict(bound, feature_names, 'bound') monotonic = tools.cast_to_dict(monotonic, feature_names, 'monotonic') keypoints_initializers = tools.cast_to_dict(keypoints_initializers, feature_names, 'keypoints_initializers') keypoints_initializer_fns = tools.cast_to_dict( keypoints_initializer_fns, feature_names, 'keypoints_initializer_fns') missing_input_values = tools.cast_to_dict(missing_input_values, feature_names, 'missing_input_values') missing_output_values = tools.cast_to_dict(missing_output_values, feature_names, 'missing_output_values') regularizer_amounts = { regularizer_name: tools.cast_to_dict(regularizer_amounts[regularizer_name], feature_names, regularizer_name) for regularizer_name in regularizer_amounts } per_dimension_feature_names = [] # Get uncalibrated tensors, either from columns_to_tensors, or using # feature_columns. if feature_columns is None: uncalibrated_features = [ columns_to_tensors[name] for name in feature_names ] else: transformed_columns_to_tensors = columns_to_tensors.copy() dict_feature_columns = { f_col.name: f_col for f_col in feature_columns } uncalibrated_features = [ tools.input_from_feature_column(transformed_columns_to_tensors, dict_feature_columns[name], dtype) for name in feature_names ] projection_ops = [] calibrated_splits = [] total_regularization = None for feature_idx in range(len(feature_names)): name = feature_names[feature_idx] uncalibrated_feature = uncalibrated_features[feature_idx] if uncalibrated_feature.shape.ndims == 1: feature_dim = 1 uncalibrated_splits = [uncalibrated_feature] elif uncalibrated_feature.shape.ndims == 2: feature_dim = uncalibrated_feature.shape.dims[1].value uncalibrated_splits = array_ops.unstack(uncalibrated_feature, axis=1) else: raise ValueError( 'feature {}: it has rank {}, but only ranks 1 or 2 are ' 'supported; feature shape={}'.format( name, uncalibrated_feature.shape.ndims, uncalibrated_feature.shape)) missing_input_value = missing_input_values[name] missing_output_value = missing_output_values[name] feature_regularizer_amounts = { regularizer_name: regularizer_amounts[regularizer_name][name] for regularizer_name in regularizer_amounts } # FutureWork: make the interpolation ops handle multi-dimension values, # so this step is not needed. for dim_idx in range(feature_dim): per_dimension_feature_names += [name] split_name = name if feature_dim > 1: split_name = '{}_dim_{}'.format(name, dim_idx) uncalibrated = uncalibrated_splits[dim_idx] if not num_keypoints[name]: # No calibration for this feature: calibrated_splits += [uncalibrated] if (missing_input_value is not None or missing_output_value is not None): raise ValueError( 'feature %s: cannot handle missing values if feature is not ' 'calibrated, missing_input_value=%s, missing_output_value=%s' % (name, missing_input_value, missing_output_value)) else: calibrated, projection, reg = one_dimensional_calibration_layer( uncalibrated, num_keypoints[name], signal_name=split_name, keypoints_initializers=keypoints_initializers[name], keypoints_initializer_fns=keypoints_initializer_fns[ name], bound=bound[name], monotonic=monotonic[name], missing_input_value=missing_input_value, missing_output_value=missing_output_value, **feature_regularizer_amounts) calibrated_splits += [calibrated] if projection is not None: projection_ops += [projection] total_regularization = tools.add_if_not_none( total_regularization, reg) all_calibrated = array_ops.stack(calibrated_splits, axis=1, name='stack_calibrated') return (all_calibrated, per_dimension_feature_names, projection_ops, total_regularization)
def _base_model_fn(self, features, labels, mode, config): # pylint: disable=unused-argument """Creates the prediction, loss, and train ops. Args: features: A dictionary of tensors keyed by the feature name. labels: A tensor representing the label. mode: The execution mode, as defined in tf.estimator.ModeKeys. config: Optional configuration object. Will receive what is passed to Estimator in `config` parameter, or the default `config`. Allows updating things in your model_fn based on configuration such as `num_ps_replicas`. Returns: ModelFnOps, with the predictions, loss, and train_op. Raises: ValueError: if incompatible parameters are given. """ with tf.compat.v1.variable_scope(self._name): if self._feature_columns is None: columns_to_tensors = features.copy() else: with tf.compat.v1.variable_scope( "feature_column_transformation"): columns_to_tensors = { feature_column.name: tools.input_from_feature_column( features.copy(), feature_column, self._dtype) for feature_column in self._feature_columns } (prediction, projection_ops, regularization) = self.prediction_builder(columns_to_tensors, mode, self._hparams, self._dtype) def _train_op_fn(loss): """Returns train_op tensor if TRAIN mode, or None.""" train_op = None if mode == tf.estimator.ModeKeys.TRAIN: if regularization is not None: loss += regularization tf.compat.v1.summary.scalar("loss_with_regularization", loss) optimizer = self._optimizer if optimizer is None: optimizer = tf.compat.v1.train.AdamOptimizer if callable(optimizer): optimizer = optimizer() train_op = optimizer.minimize( loss, global_step=tf.compat.v1.train.get_global_step(), name=_TRAIN_OP_NAME) self._projection_hook.set_projection_ops(projection_ops) return train_op # Use head to generate model_fn outputs. estimator_spec = self._head.create_estimator_spec( features=features, labels=labels, mode=mode, train_op_fn=_train_op_fn, logits=prediction) # Update chief worker's training session run hooks to include # projection_hook. This means that in a distributed setting, only the # chief worker will run the projection op after its own update and without # synchronization with other workers. Thus, the parameters may temporary # leave the feasible space. if mode == tf.estimator.ModeKeys.TRAIN: updated_training_chief_hooks = ( estimator_spec.training_chief_hooks + (self._projection_hook, )) estimator_spec = estimator_spec._replace( training_chief_hooks=updated_training_chief_hooks) return estimator_spec
def save_quantiles_for_keypoints(input_fn, save_dir, feature_columns=None, num_steps=1, override=True, num_quantiles=1000, dtype=dtypes.float32): """Calculates and saves quantiles for given features. These values can later be retrieved and used by keypoints_from_quantiles() below. Repeated values are discarded before the quantiles are calculated. That means that the quantiles of a very skewed distribution (for instance where 99% of the values are 0), will be different. But for the purpose of calibration this approach is more useful. Nothing is returned, the values are simply saved in the given location. This function can be called as a preprocessing step before actual training starts. Typically one will run this in a separate process locally, before starting training for instance. Args: input_fn: Similar to input_fn provided to Estimators. Typically one doesn't need to go over the full data to get good quantiles. Typically some 100 random examples per quantile is good enough for the purpose of calibration. If you don't have too much data, just use everything. If input_fn returns a target (used in training) it is ignored. save_dir: Where to save these quantiles. Since when optimizing hyper-parameters we train various models, we can share the quantiles information generated here. So this should be a directory that can be accessed by all training sessions. A subdirectory called "quantiles" will be created, and inside one file per feature is created: named after the feature name, and with the quantiles stored in JSON format. feature_columns: If set, quantiles are generated for these feature columns. The file name used to save the quantiles uses a hash of the names of the feature_columns, so it can support different quantiles sets for different parts of the model if needed. If not set quantiles will be generated for all features returned by input_fn. num_steps: number of steps to take over input_fn to gather enough data to create quantiles. Set to 0 or None to run until queue is exhausted, like if you used num_epochs in your input_fn. override: if False it won't regenerate quantiles for files that are already there. This works as long as the features definition/distribution hasn't change from one run to another. num_quantiles: This value should be larger than the maximum number of keypoints that will be considered for calibrating these features. If there are not enough quantiles for the keypoints, the system is robust and will simply interpolate the missing quantiles. Similarly if there are not enough examples to represent the quantiles, it will interpolate the quantiles from the examples given. dtype: Deafult dtype to use, in particular for categorical values. Returns: Nothing, results are saved to disk. Raises: errors.OpError: For I/O errors. FutureWork: * Use Munro-Paterson algorithm to calculate quantiles in a streaming fashion. See Squawd library. * Add support to weighted examples. * Handle cases where there are not enough different values in quantiles. """ subdir = os.path.join(save_dir, _QUANTILES_SUBDIRECTORY) file_io.recursive_create_dir(subdir) with ops.Graph().as_default(): tensors = None if feature_columns is not None: # Features from feature_columns. if not override: # Remove feature_columns for which we already have the quantiles. missing_feature_columns = [] for f_col in feature_columns: try: _ = _load_quantiles(subdir, f_col.name) except errors.NotFoundError: missing_feature_columns += [f_col] feature_columns = missing_feature_columns if not feature_columns: return transformed_columns_to_tensors, unused_label = input_fn() tensors = { f_col.name: tools.input_from_feature_column(transformed_columns_to_tensors, f_col, dtype) for f_col in feature_columns } else: # Features directly from columns_to_tensors. columns_to_tensors, unused_label = input_fn() tensors = {} if override: tensors = columns_to_tensors else: for name, tensor in columns_to_tensors.items(): try: _ = _load_quantiles(subdir, name) except errors.NotFoundError: tensors[name] = tensor if not tensors: # All features already calculated. return # Here a dict of feature_name to tensor is in tensors. arrays = _materialize_locally(tensors, num_steps) percentiles = np.linspace(0., 100., num_quantiles) for key, values in arrays.items(): values = np.unique(values) quantiles = np.percentile(values, percentiles, interpolation="nearest") quantiles = list(quantiles) _save_quantiles(subdir, key, quantiles)