def test_get_config(self, output_dim, initializer, scale, trainable): rff_layer = kernel_layers.RandomFourierFeatures( output_dim, initializer, scale=scale, trainable=trainable, name='random_fourier_features', ) expected_initializer = initializer if not isinstance(initializer, six.string_types): expected_initializer = initializers.serialize(initializer) expected_dtype = ('float32' if base_layer_utils.v2_dtype_behavior_enabled() else None) expected_config = { 'output_dim': output_dim, 'kernel_initializer': expected_initializer, 'scale': scale, 'name': 'random_fourier_features', 'trainable': trainable, 'dtype': expected_dtype, } self.assertLen(expected_config, len(rff_layer.get_config())) self.assertSameElements(list(expected_config.items()), list(rff_layer.get_config().items()))
def test_get_config(self, output_dim, initializer, scale, trainable): rff_layer = kernel_layers.RandomFourierFeatures( output_dim, initializer, scale=scale, trainable=trainable, name="random_fourier_features", ) expected_initializer = initializer if not isinstance(initializer, str): expected_initializer = initializers.serialize(initializer) expected_dtype = ("float32" if base_layer_utils.v2_dtype_behavior_enabled() else None) expected_config = { "output_dim": output_dim, "kernel_initializer": expected_initializer, "scale": scale, "name": "random_fourier_features", "trainable": trainable, "dtype": expected_dtype, } self.assertLen(expected_config, len(rff_layer.get_config())) self.assertSameElements(list(expected_config.items()), list(rff_layer.get_config().items()))
def global_policy(): """Returns the global dtype policy. The global policy is the default `tf.keras.mixed_precision.Policy` used for layers, if no policy is passed to the layer constructor. If no policy has been set with `keras.mixed_precision.set_global_policy`, this will return a policy constructed from `tf.keras.backend.floatx()` (floatx defaults to float32). >>> tf.keras.mixed_precision.global_policy() <Policy "float32"> >>> tf.keras.layers.Dense(10).dtype_policy # Defaults to the global policy <Policy "float32"> If TensorFlow 2 behavior has been disabled with `tf.compat.v1.disable_v2_behavior()`, this will instead return a special "_infer" policy which infers the dtype from the dtype of the first input the first time the layer is called. This behavior matches the behavior that existed in TensorFlow 1. See `tf.keras.mixed_precision.Policy` for more information on policies. Returns: The global Policy. """ if _global_policy is None: if base_layer_utils.v2_dtype_behavior_enabled(): return Policy(backend.floatx()) else: return Policy('_infer') return _global_policy
def __init__(self, name=None, dtype=None, **kwargs): super().__init__(name=name, dtype=dtype, **kwargs) self.stateful = True # All metric layers are stateful. self.built = True if not base_layer_utils.v2_dtype_behavior_enabled(): # We only do this when the V2 behavior is not enabled, as when it is # enabled, the dtype already defaults to floatx. self._dtype = (backend.floatx() if dtype is None else tf.as_dtype(dtype).name)
def set_policy(policy): """Sets the global dtype policy. The global policy is the default `tf.keras.mixed_precision.Policy` used for layers, if no policy is passed to the layer constructor. >>> tf.keras.mixed_precision.set_global_policy('mixed_float16') >>> tf.keras.mixed_precision.global_policy() <Policy "mixed_float16"> >>> tf.keras.layers.Dense(10).dtype_policy <Policy "mixed_float16"> >>> # Global policy is not used if a policy is directly passed to constructor >>> tf.keras.layers.Dense(10, dtype='float64').dtype_policy <Policy "float64"> >>> tf.keras.mixed_precision.set_global_policy('float32') If no global policy is set, layers will instead default to a Policy constructed from `tf.keras.backend.floatx()`. To use mixed precision, the global policy should be set to `'mixed_float16'` or `'mixed_bfloat16'`, so that every layer uses a 16-bit compute dtype and float32 variable dtype by default. Only floating point policies can be set as the global policy, such as `'float32'` and `'mixed_float16'`. Non-floating point policies such as `'int32'` and `'complex64'` cannot be set as the global policy because most layers do not support such policies. See `tf.keras.mixed_precision.Policy` for more information. Args: policy: A Policy, or a string that will be converted to a Policy. Can also be None, in which case the global policy will be constructed from `tf.keras.backend.floatx()` """ global _global_policy if not base_layer_utils.v2_dtype_behavior_enabled(): raise ValueError( 'The global policy can only be set in TensorFlow 2 or if ' 'V2 dtype behavior has been set. To enable V2 dtype ' 'behavior, call ' '"tf.compat.v1.keras.layers.enable_v2_dtype_behavior()"') if policy is not None and not isinstance(policy, Policy): policy = Policy(policy) is_mixed_policy = (policy is not None and policy.compute_dtype != policy.variable_dtype) if is_mixed_policy: _check_if_mixed_precision_graph_rewrite_is_enabled(policy) if (policy is not None and policy.compute_dtype is not None and not tf.as_dtype(policy.compute_dtype).is_floating): raise ValueError( 'set_policy can only be used to set the global policy to ' 'floating-point policies, such as "float32" and ' '"mixed_float16", but got policy: %s' % (policy.name, )) _global_policy = policy mixed_precision_global_state.using_mixed_precision_policy = is_mixed_policy
def test_policy_scope(self): if base_layer_utils.v2_dtype_behavior_enabled(): default_policy = 'float32' else: default_policy = '_infer' with mp_policy.policy_scope('mixed_float16'): self.assertEqual(mp_policy.global_policy().name, 'mixed_float16') with mp_policy.policy_scope('_infer'): self.assertEqual(mp_policy.global_policy().name, '_infer') self.assertEqual(mp_policy.global_policy().name, 'mixed_float16') self.assertEqual(mp_policy.global_policy().name, default_policy)
def test_policy_scope(self): if base_layer_utils.v2_dtype_behavior_enabled(): default_policy = "float32" else: default_policy = "_infer" with mp_policy.policy_scope("mixed_float16"): self.assertEqual(mp_policy.global_policy().name, "mixed_float16") with mp_policy.policy_scope("_infer"): self.assertEqual(mp_policy.global_policy().name, "_infer") self.assertEqual(mp_policy.global_policy().name, "mixed_float16") self.assertEqual(mp_policy.global_policy().name, default_policy)
def __init__( self, input_dim, output_dim, embeddings_initializer="uniform", embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None, **kwargs, ): if "input_shape" not in kwargs: if input_length: kwargs["input_shape"] = (input_length,) else: kwargs["input_shape"] = (None,) if input_dim <= 0 or output_dim <= 0: raise ValueError( "Both `input_dim` and `output_dim` should be positive, " f"Received input_dim = {input_dim} " f"and output_dim = {output_dim}" ) if ( not base_layer_utils.v2_dtype_behavior_enabled() and "dtype" not in kwargs ): # In TF1, the dtype defaults to the input dtype which is typically # int32, so explicitly set it to floatx kwargs["dtype"] = backend.floatx() # We set autocast to False, as we do not want to cast floating- point # inputs to self.dtype. In call(), we cast to int32, and casting to # self.dtype before casting to int32 might cause the int32 values to be # different due to a loss of precision. kwargs["autocast"] = False super().__init__(**kwargs) self.input_dim = input_dim self.output_dim = output_dim self.embeddings_initializer = initializers.get(embeddings_initializer) self.embeddings_regularizer = regularizers.get(embeddings_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.embeddings_constraint = constraints.get(embeddings_constraint) self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length
def test_global_policy(self): if base_layer_utils.v2_dtype_behavior_enabled(): default_policy = 'float32' else: default_policy = '_infer' self.assertEqual(mp_policy.global_policy().name, default_policy) try: mp_policy.set_global_policy('mixed_float16') self.assertEqual(mp_policy.global_policy().name, 'mixed_float16') with tf.Graph().as_default(): # Policies are not associated with a graph self.assertEqual(mp_policy.global_policy().name, 'mixed_float16') mp_policy.set_global_policy('_infer') self.assertEqual(mp_policy.global_policy().name, '_infer') policy = mp_policy.Policy('mixed_bfloat16') mp_policy.set_global_policy(policy) self.assertIs(mp_policy.global_policy(), policy) finally: mp_policy.set_global_policy(None)
def __init__(self, input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None, **kwargs): if 'input_shape' not in kwargs: if input_length: kwargs['input_shape'] = (input_length, ) else: kwargs['input_shape'] = (None, ) if input_dim <= 0 or output_dim <= 0: raise ValueError( 'Both `input_dim` and `output_dim` should be positive, ' 'found input_dim {} and output_dim {}'.format( input_dim, output_dim)) if (not base_layer_utils.v2_dtype_behavior_enabled() and 'dtype' not in kwargs): # In TF1, the dtype defaults to the input dtype which is typically int32, # so explicitly set it to floatx kwargs['dtype'] = K.floatx() # We set autocast to False, as we do not want to cast floating- point inputs # to self.dtype. In call(), we cast to int32, and casting to self.dtype # before casting to int32 might cause the int32 values to be different due # to a loss of precision. kwargs['autocast'] = False super(Embedding, self).__init__(**kwargs) self.input_dim = input_dim self.output_dim = output_dim self.embeddings_initializer = initializers.get(embeddings_initializer) self.embeddings_regularizer = regularizers.get(embeddings_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.embeddings_constraint = constraints.get(embeddings_constraint) self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length
def test_global_policy(self): if base_layer_utils.v2_dtype_behavior_enabled(): default_policy = "float32" else: default_policy = "_infer" self.assertEqual(mp_policy.global_policy().name, default_policy) try: mp_policy.set_global_policy("mixed_float16") self.assertEqual(mp_policy.global_policy().name, "mixed_float16") # Policies are not associated with a graph with tf.Graph().as_default(): self.assertEqual( mp_policy.global_policy().name, "mixed_float16" ) mp_policy.set_global_policy("_infer") self.assertEqual(mp_policy.global_policy().name, "_infer") policy = mp_policy.Policy("mixed_bfloat16") mp_policy.set_global_policy(policy) self.assertIs(mp_policy.global_policy(), policy) finally: mp_policy.set_global_policy(None)