def testLoadKeypointsFromQuantilesLoadingLabelQuantiles(self): input_fn, unused_feature_names, unused_feature_columns = self._BuildInputs( x0=np.random.uniform(0.0, 2.0, 100000), x1=[0], x2=[0], label=np.random.uniform(0.0, 100.0, 100000)) save_dir = os.path.join( self.get_temp_dir(), 'load_keypoints_from_quantiles_loading_label_quantiles') keypoints_initialization.save_quantiles_for_keypoints(input_fn, save_dir, override=True) with tf.Graph().as_default() as g, self.session(graph=g) as session: result = keypoints_initialization.load_keypoints_from_quantiles( feature_names=['x0'], save_dir=save_dir, num_keypoints=5, use_label_quantiles_for_outputs=True) result = session.run(result) self.assertAllClose( { 'x0': [ np.array([0.0, 0.5, 1.0, 1.5, 2.0]), np.array([0.0, 25.0, 50.0, 75.0, 100.0]) ] }, result, atol=0.2, msg='load_keypoints_from_quantiles didn\'t produce expected labels' )
def testQuantileInitWithMissingInputValuesDict(self): num_examples = 10 x0 = np.linspace(-1.0, 1.0, num_examples) x1 = np.linspace(0.0, 1.0, num_examples) x2 = np.linspace(0.0, 1.0, num_examples) input_fn, feature_names, feature_columns = self._BuildInputs( x0, x1, x2) save_dir = os.path.join(self.get_temp_dir(), 'exclude_input_values_dict') keypoints_initialization.save_quantiles_for_keypoints( input_fn, save_dir, feature_columns=feature_columns, num_quantiles=num_examples, override=True) with ops.Graph().as_default() as g: # Check by using load_keypoints_from_quantiles. keypoints_init = keypoints_initialization.load_keypoints_from_quantiles( feature_names, save_dir, num_keypoints=3, output_min={ 'x0': 0., 'x1': 0., 'x2': 0. }, output_max={ 'x0': 1., 'x1': 1., 'x2': 1. }, missing_input_values_dict={ 'x0': -1.0, 'x1': 0.0, 'x2': None }, ) with self.test_session(graph=g) as sess: keypoints_init = sess.run(keypoints_init) self.assertAllClose(keypoints_init['x0'][0], [-0.778, 0.111, 1.0], atol=0.1) self.assertAllClose(keypoints_init['x0'][1], [0.0, 0.5, 1.0], atol=0.01) self.assertAllClose(keypoints_init['x1'][0], [0.111, 0.556, 1.0], atol=0.1) self.assertAllClose(keypoints_init['x1'][1], [0.0, 0.5, 1.0], atol=0.01) self.assertAllClose(keypoints_init['x2'][0], [0.0, 0.444, 1.0], atol=0.01) self.assertAllClose(keypoints_init['x2'][1], [0.0, 0.5, 1.0], atol=0.01)
def testLoadKeypointsFromQuantilesRaises(self, use_label_quantiles_for_outputs, output_min, output_max, msg): input_fn, unused_feature_names, unused_feature_columns = self._BuildInputs( x0=np.random.uniform(0.0, 2.0, 100000), x1=[0], x2=[0], label=np.random.uniform(0.0, 100.0, 100000)) save_dir = os.path.join( self.get_temp_dir(), 'load_keypoints_from_quantiles_loading_label_quantiles') keypoints_initialization.save_quantiles_for_keypoints(input_fn, save_dir, override=True) with self.assertRaises(ValueError, msg=msg): keypoints_initialization.load_keypoints_from_quantiles( use_label_quantiles_for_outputs=use_label_quantiles_for_outputs, output_min=output_min, output_max=output_max, feature_names=['x0'], save_dir=save_dir, num_keypoints=5)
def testQuantileInitWithReversedDict(self): num_examples = 100 x0 = np.linspace(0.0, 10.0, num_examples) x1 = np.linspace(0.0, 10.0, num_examples) x2 = np.linspace(0.0, 1.0, num_examples) input_fn, feature_names, feature_columns = self._BuildInputs( x0, x1, x2) save_dir = os.path.join(self.get_temp_dir(), 'reversed_dict') keypoints_initialization.save_quantiles_for_keypoints( input_fn, save_dir, feature_columns=feature_columns, num_quantiles=100, override=True) reversed_dict = {'x0': False, 'x1': True, 'x2': False} with ops.Graph().as_default() as g: # Check by using load_keypoints_from_quantiles. keypoints_init = keypoints_initialization.load_keypoints_from_quantiles( feature_names, save_dir, num_keypoints=3, output_min={ 'x0': 0., 'x1': 0., 'x2': 0. }, output_max={ 'x0': 1., 'x1': 1., 'x2': 1. }, reversed_dict=reversed_dict) with self.test_session(graph=g) as sess: keypoints_init = sess.run(keypoints_init) self.assertAllClose(keypoints_init['x0'][0], [0.0, 5.0, 10.0], atol=0.1) self.assertAllClose(keypoints_init['x0'][1], [0.0, 0.5, 1.0], atol=0.01) self.assertAllClose(keypoints_init['x1'][0], [0.0, 5.0, 10.0], atol=0.1) self.assertAllClose(keypoints_init['x1'][1], [1.0, 0.5, 0.0], atol=0.01) self.assertAllClose(keypoints_init['x2'][0], [0.0, 0.5, 1.0], atol=0.01) self.assertAllClose(keypoints_init['x2'][1], [0.0, 0.5, 1.0], atol=0.01)
def _CheckSaveQuantilesForKeypoints(self, name, num_examples, num_steps, x0, x1, x2, use_feature_columns, override): input_fn, feature_names, feature_columns = self._BuildInputs( x0, x1, x2) save_dir = os.path.join(self.get_temp_dir(), name) keypoints_initialization.save_quantiles_for_keypoints( input_fn, save_dir, feature_columns=(feature_columns if use_feature_columns else None), num_quantiles=5, override=override) # Check by reading files directly. subdir = os.path.join(save_dir, keypoints_initialization._QUANTILES_SUBDIRECTORY) quantiles_x0 = keypoints_initialization._load_quantiles(subdir, 'x0') quantiles_x1 = keypoints_initialization._load_quantiles(subdir, 'x1') quantiles_x2 = keypoints_initialization._load_quantiles(subdir, 'x2') self.assertAllClose(quantiles_x0, [0, 2.5**2, 5.**2, 7.5**2, 100.], atol=0.2) self.assertAllClose( quantiles_x1, [1., math.pow(10., 0.5), 10.0, math.pow(10., 1.5), 100.], atol=0.2) # x2 should start with [0,0,...] and end in [..., 1, 1], the middle value # can be either 0 or 1. self.assertAllClose(quantiles_x2[0:2], [0., 0.], atol=1e-3) self.assertAllClose(quantiles_x2[-2:], [1., 1.], atol=1e-3) # New graph is needed because default graph is changed by save # keypoints, and self.test_session() will by default try to reuse a cached # session, with a different graph. with ops.Graph().as_default() as g: # Check by using load_keypoints_from_quantiles. keypoints_init = keypoints_initialization.load_keypoints_from_quantiles( feature_names, save_dir, 3, output_min={ 'x0': 0., 'x1': 1., 'x2': 7. }, output_max={ 'x0': 1., 'x1': 10., 'x2': 13. }) with self.test_session(graph=g) as sess: keypoints_init = sess.run(keypoints_init) self.assertAllClose(keypoints_init['x0'][0], [0, 5.**2, 100.], atol=0.2) self.assertAllClose(keypoints_init['x0'][1], [0., 0.5, 1.]) self.assertAllClose(keypoints_init['x1'][0], [1., 10.0, 100.], atol=0.2) self.assertAllClose(keypoints_init['x1'][1], [1., 5.5, 10.]) # Notice x2 only has 2 unique values, so it should have lowered the # num_keypoints to 2. self.assertAllClose([0., 1.0], keypoints_init['x2'][0], atol=1e-3) self.assertAllClose([7., 13.0], keypoints_init['x2'][1], atol=1e-3) # Check that load_keypoints_from_quantiles don't generate anything # if num_keypoints is 0 or unset. with ops.Graph().as_default() as g: # Check by using load_keypoints_from_quantiles. keypoints_init = keypoints_initialization.load_keypoints_from_quantiles( feature_names, save_dir, { 'x0': 3, 'x2': 3, 'x1': 0 }, output_min={ 'x0': 0., 'x1': 1., 'x2': 7. }, output_max={ 'x0': 1., 'x1': 10., 'x2': 13. }) with self.test_session(graph=g) as sess: keypoints_init = sess.run(keypoints_init) self.assertTrue('x0' in keypoints_init) self.assertTrue('x2' in keypoints_init) self.assertTrue('x1' not in keypoints_init)
def input_calibration_layer_from_hparams(columns_to_tensors, feature_columns, hparams, quantiles_dir=None, keypoints_initializers=None, name=None, dtype=dtypes.float32): """Creates a calibration layer for the input using hyper-parameters. Similar to `input_calibration_layer` but reads its parameters from a `CalibratedHParams` object. Args: columns_to_tensors: A mapping from feature name to tensors. 'string' key means a base feature (not-transformed). If feature_columns is not set these are the features calibrated. Otherwise the transformed feature_columns are the ones calibrated. feature_columns: An iterable containing all the feature columns used by the model. Optional, if not set the model will use all features given in columns_to_tensors. All items in the set should be instances of classes derived from `FeatureColumn`. hparams: Hyper-parameters, need to inherit from `CalibratedHParams`. See `CalibratedHParams` and `input_calibration_layer` for descriptions of how these hyper-parameters work. quantiles_dir: location where quantiles for the data was saved. Typically the same directory as the training data. These quantiles can be generated with `pwl_calibration_layers.calculate_quantiles_for_keypoints`, maybe in a separate invocation of your program. Different models that share the same quantiles information -- so this needs to be generated only once when hyper-parameter tuning. If you don't want to use quantiles, you can set `keypoints_initializers` instead. keypoints_initializers: if you know the distribution of your input features you can provide that directly instead of `quantiles_dir`. See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be a pair of tensors with keypoints inputs and outputs to use for initialization (must match `num_keypoints` configured in `hparams`). Alternatively can be given as a dict mapping feature name to pairs, for initialization per feature. If `quantiles_dir` and `keypoints_initializer` are set, the later takes precendence, and the features for which `keypoints_initializers` are not defined fallback to using the quantiles found in `quantiles_dir`. name: Name scope for layer. dtype: If any of the scalars are not given as tensors, they are converted to tensors with this dtype. Returns: A tuple of: * calibrated tensor of shape [batch_size, sum(features dimensions)]. * list of the feature names in the order they feature in the calibrated tensor. A name may appear more than once if the feature is multi-dimension (for instance a multi-dimension embedding) * list of projection ops, that must be applied at each step (or every so many steps) to project the model to a feasible space: used for bounding the outputs or for imposing monotonicity. Empty if none are requested. * None or tensor with regularization loss. Raises: ValueError: if dtypes are incompatible. """ with ops.name_scope(name or "input_calibration_layer_from_hparams"): # Sort out list of feature names. unique_feature_names = tools.get_sorted_feature_names( columns_to_tensors=columns_to_tensors, feature_columns=feature_columns) # Get per-feature parameters. num_keypoints = _get_per_feature_dict(hparams, "num_keypoints") calibration_output_min = _get_per_feature_dict( hparams, "calibration_output_min") calibration_output_max = _get_per_feature_dict( hparams, "calibration_output_max") calibration_bound = _get_per_feature_dict(hparams, "calibration_bound") monotonicity = _get_per_feature_dict(hparams, "monotonicity") missing_input_values = _get_per_feature_dict(hparams, "missing_input_value") missing_output_values = _get_per_feature_dict(hparams, "missing_output_value") # Define keypoints_initializers to use in this invocation of model_fn. kp_init = None if quantiles_dir is not None: # Skip features for which an explicit initializer was given. if isinstance(keypoints_initializers, dict): quantiles_feature_names = [] for name in unique_feature_names: if name not in keypoints_initializers: quantiles_feature_names.append(name) else: quantiles_feature_names = unique_feature_names # Reverse initial output keypoints for decreasing monotonic features. reversed_dict = {} for feature_name in quantiles_feature_names: if monotonicity[feature_name] == -1: reversed_dict[feature_name] = True else: reversed_dict[feature_name] = False # Read initializers from quantiles_dir, for those not already # defined. # # Notice that output_min and output_max won't matter much if # they are not bounded, since they will be adjusted during training. kp_init = keypoints_initialization.load_keypoints_from_quantiles( feature_names=quantiles_feature_names, save_dir=quantiles_dir, num_keypoints=num_keypoints, output_min=calibration_output_min, output_max=calibration_output_max, reversed_dict=reversed_dict, missing_input_values_dict=missing_input_values, dtype=dtype) # Merge with explicit initializers. if isinstance(keypoints_initializers, dict): kp_init.update(keypoints_initializers) else: # Take given initializers. kp_init = keypoints_initializers # Update num_keypoints according to keypoints actually used by the # initialization functions: some initialization functions may change # them, for instance if there are not enough unique values. if isinstance(kp_init, dict): # One initializer (kp_init) per feature. for (feature_name, initializers) in six.iteritems(kp_init): kp_init_keypoints = initializers[0].shape.as_list()[0] num_keypoints[feature_name] = _update_keypoints( feature_name, num_keypoints[feature_name], kp_init_keypoints) else: # Check generic initializer (kp_init). kp_init_keypoints = kp_init[0].shape.as_list()[0] for feature_name in six.iterkeys(num_keypoints): num_keypoints[feature_name] = _update_keypoints( feature_name, num_keypoints[feature_name], kp_init_keypoints) # Setup the regularization. calibration_l1_regs = _get_per_feature_dict(hparams, "calibration_l1_reg") calibration_l2_regs = _get_per_feature_dict(hparams, "calibration_l2_reg") calibration_l1_laplacian_regs = _get_per_feature_dict( hparams, "calibration_l1_laplacian_reg") calibration_l2_laplacian_regs = _get_per_feature_dict( hparams, "calibration_l2_laplacian_reg") return pwl_calibration_layers.input_calibration_layer( columns_to_tensors=columns_to_tensors, feature_columns=feature_columns, num_keypoints=num_keypoints, keypoints_initializers=kp_init, bound=calibration_bound, monotonic=monotonicity, missing_input_values=missing_input_values, missing_output_values=missing_output_values, l1_reg=calibration_l1_regs, l2_reg=calibration_l2_regs, l1_laplacian_reg=calibration_l1_laplacian_regs, l2_laplacian_reg=calibration_l2_laplacian_regs)
def input_calibration_layer_from_hparams(columns_to_tensors, hparams, quantiles_dir=None, keypoints_initializers=None, name=None, dtype=dtypes.float32): """Creates a calibration layer for the input using hyper-parameters. Similar to `input_calibration_layer` but reads its parameters from a `CalibratedHParams` object. Args: columns_to_tensors: A mapping from feature name to tensors. hparams: Hyper-parameters, need to inherit from `CalibratedHParams`. See `CalibratedHParams` and `input_calibration_layer` for descriptions of how these hyper-parameters work. quantiles_dir: location where quantiles for the data was saved. Typically the same directory as the training data. These quantiles can be generated with `pwl_calibration_layers.calculate_quantiles_for_keypoints`, maybe in a separate invocation of your program. Different models that share the same quantiles information -- so this needs to be generated only once when hyper-parameter tuning. If you don't want to use quantiles, you can set `keypoints_initializers` instead. keypoints_initializers: if you know the distribution of your input features you can provide that directly instead of `quantiles_dir`. See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be a pair of tensors with keypoints inputs and outputs to use for initialization (must match `num_keypoints` configured in `hparams`). Alternatively can be given as a dict mapping feature name to pairs, for initialization per feature. If `quantiles_dir` and `keypoints_initializer` are set, the latter takes precendence, and the features for which `keypoints_initializers` are not defined fallback to using the quantiles found in `quantiles_dir`. name: Name scope for layer. dtype: If any of the scalars are not given as tensors, they are converted to tensors with this dtype. Returns: A tuple of: * calibrated tensor of shape [batch_size, sum(features dimensions)]. * list of the feature names in the order they appear in the calibrated tensor. A name may appear more than once if the feature is multi-dimension (for instance a multi-dimension embedding) * list of projection ops, that must be applied at each step (or every so many steps) to project the model to a feasible space: used for bounding the outputs or for imposing monotonicity. Empty if none are requested. * tensor with regularization loss, or None for no regularization. Raises: ValueError: if dtypes are incompatible. """ with ops.name_scope(name or "input_calibration_layer_from_hparams"): # Sort out list of feature names. unique_feature_names = tools.get_sorted_feature_names( columns_to_tensors=columns_to_tensors) # Get per-feature parameters. num_keypoints = _get_per_feature_dict(hparams, "num_keypoints") calibration_output_min = _get_per_feature_dict( hparams, "calibration_output_min") calibration_output_max = _get_per_feature_dict( hparams, "calibration_output_max") calibration_bound = _get_per_feature_dict(hparams, "calibration_bound") monotonicity = _get_per_feature_dict(hparams, "monotonicity") missing_input_values = _get_per_feature_dict(hparams, "missing_input_value") missing_output_values = _get_per_feature_dict(hparams, "missing_output_value") # Convert keypoints_initializers to a dict if needed, or otherwise make a # copy of the original keypoints_initializers dict. if keypoints_initializers is None: keypoints_initializers = {} elif not isinstance(keypoints_initializers, dict): keypoints_initializers = { name: keypoints_initializers for name in unique_feature_names } else: keypoints_initializers = keypoints_initializers.copy() # If quantiles_dir is given, add any missing keypoint initializers with # keypoints based on quantiles. if quantiles_dir is not None: quantiles_feature_names = [ name for name in unique_feature_names if name not in keypoints_initializers ] # Reverse initial output keypoints for decreasing monotonic features. reversed_dict = { feature_name: (monotonicity[feature_name] == -1) for feature_name in quantiles_feature_names } # Read initializers from quantiles_dir, for those not already # defined. # # Notice that output_min and output_max won't matter much if # they are not bounded, since they will be adjusted during training. quantiles_init = keypoints_initialization.load_keypoints_from_quantiles( feature_names=quantiles_feature_names, save_dir=quantiles_dir, num_keypoints=num_keypoints, output_min=calibration_output_min, output_max=calibration_output_max, reversed_dict=reversed_dict, missing_input_values_dict=missing_input_values, dtype=dtype) # Merge with explicit initializers. keypoints_initializers.update(quantiles_init) # Update num_keypoints according to keypoints actually used by the # initialization functions: some initialization functions may change # them, for instance if there are not enough unique values. for (feature_name, initializers) in six.iteritems(keypoints_initializers): kp_init_keypoints = initializers[0].shape.as_list()[0] num_keypoints[feature_name] = _update_keypoints( feature_name, num_keypoints[feature_name], kp_init_keypoints) # Setup the regularization. regularizer_amounts = {} for regularizer_name in regularizers.CALIBRATOR_REGULARIZERS: regularizer_amounts[regularizer_name] = _get_per_feature_dict( hparams, "calibration_{}".format(regularizer_name)) return pwl_calibration_layers.input_calibration_layer( columns_to_tensors=columns_to_tensors, num_keypoints=num_keypoints, keypoints_initializers=keypoints_initializers, bound=calibration_bound, monotonic=monotonicity, missing_input_values=missing_input_values, missing_output_values=missing_output_values, **regularizer_amounts)