def test_axes_initialization(self): input_spec.InputSpec(shape=[1, None, 2, 3], axes={3: 5, '2': 2}) with self.assertRaisesRegexp(ValueError, 'Axis 4 is greater than'): input_spec.InputSpec(shape=[1, None, 2, 3], axes={4: 5}) with self.assertRaisesRegexp(TypeError, 'keys in axes must be integers'): input_spec.InputSpec(shape=[1, None, 2, 3], axes={'string': 5})
def test_undefined_shapes(self): spec = input_spec.InputSpec(max_ndim=5) with self.assertRaisesRegexp(ValueError, 'unknown TensorShape'): input_spec.to_tensor_shape(spec).as_list() spec = input_spec.InputSpec(min_ndim=5, max_ndim=5) with self.assertRaisesRegexp(ValueError, 'unknown TensorShape'): input_spec.to_tensor_shape(spec).as_list()
def test_defined_ndims(self): spec = input_spec.InputSpec(ndim=5) self.assertAllEqual([None] * 5, input_spec.to_tensor_shape(spec).as_list()) spec = input_spec.InputSpec(ndim=0) self.assertAllEqual([], input_spec.to_tensor_shape(spec).as_list()) spec = input_spec.InputSpec(ndim=3, axes={1: 3, -1: 2}) self.assertAllEqual([None, 3, 2], input_spec.to_tensor_shape(spec).as_list())
def __init__(self, num_units, activation=None, reuse=None, kernel_initializer=None, bias_initializer=None, name=None, dtype=None, **kwargs): super(AUGRUCell, self).__init__(_reuse=reuse, name=name, dtype=dtype, **kwargs) # _check_supported_dtypes(self.dtype) if context.executing_eagerly() and context.num_gpus() > 0: logging.warn( "%s: Note that this cell is not optimized for performance. " "Please use tf.contrib.cudnn_rnn.CudnnGRU for better " "performance on GPU.", self) # Inputs must be 2-dimensional. self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units if activation: self._activation = activations.get(activation) else: self._activation = math_ops.tanh self._kernel_initializer = initializers.get(kernel_initializer) self._bias_initializer = initializers.get(bias_initializer)
def __init__(self, num_units, forget_bias=1.0, cell_clip=None, use_peephole=False, reuse=None, dtype=None, name="lstm_fused_cell"): """Initialize the LSTM cell. Args: num_units: int, The number of units in the LSTM cell. forget_bias: float, The bias added to forget gates (see above). cell_clip: clip the cell to this value. Defaults is no cell clipping. use_peephole: Whether to use peephole connections or not. reuse: (optional) boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. dtype: the dtype of variables of this layer. name: String, the name of the layer. Layers with the same name will share weights, but to avoid mistakes we require reuse=True in such cases. By default this is "lstm_cell", for variable-name compatibility with `tf.compat.v1.nn.rnn_cell.LSTMCell`. """ super(LSTMBlockFusedCell, self).__init__( _reuse=reuse, name=name, dtype=dtype) self._num_units = num_units self._forget_bias = forget_bias self._cell_clip = cell_clip if cell_clip is not None else -1 self._use_peephole = use_peephole # Inputs must be 3-dimensional. self.input_spec = input_spec.InputSpec(ndim=3)
def __init__(self, units, activation=None, use_bias=True, kernel_initializer=None, bias_initializer=init_ops.zeros_initializer(), kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, trainable=True, name=None, **kwargs): super(MaskedFullyConnected, self).__init__( trainable=trainable, name=name, activity_regularizer=activity_regularizer, **kwargs) self.units = units self.activation = activation self.use_bias = use_bias self.kernel_initializer = kernel_initializer self.bias_initializer = bias_initializer self.kernel_regularizer = kernel_regularizer self.bias_regularizer = bias_regularizer self.input_spec = input_spec.InputSpec(min_ndim=2)
def __init__(self, num_units, activation=None, reuse=None, kernel_initialier=None, bias_initializer=None, name=None, dtype=None, **kwargs): super(GRUCell, self).__init__(_reuse=reuse, name=name, dtype=dtype, **kwargs) _check_supported_dtypes(self.dtype) if context.executing_eagerly() and context.num_gpus() > 0: logging.warn("This is not optimized for performance.") self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units if activation: self._activation = activations.get(activation) else: self._activation = math_ops.tanh self._kernel_initializer = initializers.get(kernel_initialier) self._bias_initializer = initializers.get(bias_initializer)
def __init__(self, num_units, list_size=10, num_lists=10, activation=None, reuse=None, kernel_initializer=None, bias_initializer=None, name=None, dtype=None, **kwargs): super(NestedListOperationCell, self).__init__(_reuse=reuse, name=name, dtype=dtype, **kwargs) if context.executing_eagerly() and context.num_gpus() > 0: logging.warn( "%s: Note that this cell is not optimized for performance. " "Please use tf.contrib.cudnn_rnn.CudnnGRU for better " "performance on GPU.", self) # Inputs must be 2-dimensional. self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._list_size = list_size self._num_lists = num_lists if activation: self._activation = activations.get(activation) else: self._activation = tf.sigmoid self._kernel_initializer = initializers.get(kernel_initializer) self._bias_initializer = initializers.get(bias_initializer)
def test_input_spec_dtype(self): # Test the InputSpec's dtype is compared against the inputs before the layer # casts them, not after. layer = mp_test_util.MultiplyLayer(dtype='float64') layer.input_spec = input_spec.InputSpec(dtype='float16') # Test passing Eager tensors x = array_ops.ones((2, 2), dtype='float16') layer(x) x = array_ops.ones((2, 2), dtype='float64') with self.assertRaisesRegex( ValueError, 'expected dtype=float16, found dtype=.*float64'): layer(x) # Test passing symbolic tensors x = layers.Input((2, ), dtype='float16') y = layer(x) model = models.Model(x, y) model(array_ops.ones((2, 2))) x = layers.Input((2, ), dtype='float64') with self.assertRaisesRegex( ValueError, 'expected dtype=float16, found dtype=.*float64'): # In TF2, the error is only raised when the model is run y = layer(x) model = models.Model(x, y) model(array_ops.ones((2, 2)))
def __init__(self, num_units=None, cell_size=None, reuse=None, name="gru_cell"): """Initialize the Block GRU cell. Args: num_units: int, The number of units in the GRU cell. cell_size: int, The old (deprecated) name for `num_units`. reuse: (optional) boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. name: String, the name of the layer. Layers with the same name will share weights, but to avoid mistakes we require reuse=True in such cases. By default this is "lstm_cell", for variable-name compatibility with `tf.compat.v1.nn.rnn_cell.GRUCell`. Raises: ValueError: if both cell_size and num_units are not None; or both are None. """ super(GRUBlockCell, self).__init__(_reuse=reuse, name=name) if (cell_size is None) == (num_units is None): raise ValueError( "Exactly one of num_units or cell_size must be provided.") if num_units is None: num_units = cell_size self._cell_size = num_units # Inputs must be 2-dimensional. self.input_spec = input_spec.InputSpec(ndim=2)
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') self.input_spec = input_spec.InputSpec( min_ndim=2, axes={-1: tensor_shape.dimension_value(input_shape[-1])}) self.kernel = self.add_variable( 'kernel', shape=[tensor_shape.dimension_value(input_shape[-1]), self.units], initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, dtype=self.dtype, trainable=True) self.mask = self.add_variable( name='mask', shape=[tensor_shape.dimension_value(input_shape[-1]), self.units], initializer=init_ops.ones_initializer(), trainable=False, dtype=self.dtype) self.threshold = self.add_variable( name='threshold', shape=[], initializer=init_ops.zeros_initializer(), trainable=False, dtype=self.dtype) # Add masked_weights in the weights namescope so as to make it easier # for the quantization library to add quant ops. self.masked_kernel = math_ops.multiply(self.mask, self.kernel, MASKED_WEIGHT_NAME) ops.add_to_collection(MASK_COLLECTION, self.mask) ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel) ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold) ops.add_to_collection(WEIGHT_COLLECTION, self.kernel) if self.use_bias: self.bias = self.add_variable( 'bias', shape=[ self.units, ], initializer=self.bias_initializer, regularizer=self.bias_regularizer, dtype=self.dtype, trainable=True) else: self.bias = None self.built = True
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) channel_axis = 1 if self.data_format == 'channels_first' else -1 if tensor_shape.dimension_value(input_shape[channel_axis]) is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') input_dim = tensor_shape.dimension_value(input_shape[channel_axis]) kernel_shape = self.kernel_size + (input_dim, self.filters) self.mask = self.add_variable( name='mask', shape=kernel_shape, initializer=init_ops.ones_initializer(), trainable=False, dtype=self.dtype) self.kernel = self.add_variable( name='kernel', shape=kernel_shape, initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, trainable=True, dtype=self.dtype) self.threshold = self.add_variable( name='threshold', shape=[], initializer=init_ops.zeros_initializer(), trainable=False, dtype=self.dtype) # Add masked_weights in the weights namescope so as to make it easier # for the quantization library to add quant ops. self.masked_kernel = math_ops.multiply(self.mask, self.kernel, MASKED_WEIGHT_NAME) ops.add_to_collection(MASK_COLLECTION, self.mask) ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel) ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold) ops.add_to_collection(WEIGHT_COLLECTION, self.kernel) if self.use_bias: self.bias = self.add_variable( name='bias', shape=(self.filters,), initializer=self.bias_initializer, regularizer=self.bias_regularizer, trainable=True, dtype=self.dtype) else: self.bias = None self.input_spec = input_spec.InputSpec( ndim=self.rank + 2, axes={channel_axis: input_dim}) self.built = True
def build(self, input_shape): if isinstance(input_shape, dict): names = sorted(list(input_shape.keys())) self.input_specs = [] self.dense_layers = [] for name in names: shape = input_shape[name] layer = core.Dense(units=self.units, use_bias=False, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer, name=name) layer.build(shape) self.input_specs.append( input_spec.InputSpec(shape=shape, name=name)) self.dense_layers.append(layer) elif isinstance(input_shape, (tuple, list)) and all( isinstance(shape, tensor_shape.TensorShape) for shape in input_shape): self.dense_layers = [] for shape in input_shape: layer = core.Dense(units=self.units, use_bias=False, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer) layer.build(shape) self.dense_layers.append(layer) else: # input_shape can be a single TensorShape or a tuple of ints. layer = core.Dense(units=self.units, use_bias=False, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer) layer.build(input_shape) self.dense_layers = [layer] if self.use_bias: self.bias = self.add_weight('bias', shape=self.units, initializer=self.bias_initializer, regularizer=self.bias_regularizer, dtype=self.dtype, trainable=True) else: self.bias = None self.built = True
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) # TODO(sibyl-vie3Poto): Allow higher dimension inputs. Currently the input is expected # to have shape [batch_size, dimension]. if input_shape.rank != 2: raise ValueError( 'The rank of the input tensor should be 2. Got {} instead.'.format( input_shape.ndims)) if input_shape.dims[1].value is None: raise ValueError( 'The last dimension of the inputs to `RandomFourierFeatures` ' 'should be defined. Found `None`.') self.input_spec = input_spec.InputSpec( ndim=2, axes={1: input_shape.dims[1].value}) input_dim = input_shape.dims[1].value kernel_initializer = _get_random_features_initializer( self.kernel_initializer, shape=(input_dim, self.output_dim)) unscaled_kernel = self.add_weight( name='unscaled_random_features', shape=(input_dim, self.output_dim), dtype=dtypes.float32, initializer=kernel_initializer, trainable=False) self.bias = self.add_weight( name='random_features_bias', shape=(self.output_dim,), dtype=dtypes.float32, initializer=init_ops.random_uniform_initializer( minval=0.0, maxval=2 * np.pi, dtype=dtypes.float32), trainable=False) if self.scale is None: self.scale = _get_default_scale(self.kernel_initializer, input_dim) scale = self.add_weight( name='random_features_scale', shape=(1,), dtype=dtypes.float32, initializer=init_ops.constant_initializer(self.scale), trainable=True, constraint='NonNeg') self.kernel = (1.0 / scale) * unscaled_kernel super(RandomFourierFeatures, self).build(input_shape)
def __init__(self, num_units, forget_bias=1.0, cell_clip=None, use_peephole=False, dtype=None, reuse=None, name="lstm_cell"): """Initialize the basic LSTM cell. Args: num_units: int, The number of units in the LSTM cell. forget_bias: float, The bias added to forget gates (see above). cell_clip: An optional `float`. Defaults to `-1` (no clipping). use_peephole: Whether to use peephole connections or not. dtype: the variable dtype of this layer. Default to tf.float32. reuse: (optional) boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. name: String, the name of the layer. Layers with the same name will share weights, but to avoid mistakes we require reuse=True in such cases. By default this is "lstm_cell", for variable-name compatibility with `tf.compat.v1.nn.rnn_cell.LSTMCell`. When restoring from CudnnLSTM-trained checkpoints, must use CudnnCompatibleLSTMBlockCell instead. """ super(LSTMBlockCell, self).__init__(_reuse=reuse, dtype=dtype, name=name) self._num_units = num_units self._forget_bias = forget_bias self._use_peephole = use_peephole self._cell_clip = cell_clip if cell_clip is not None else -1 self._names = { "W": "kernel", "b": "bias", "wci": "w_i_diag", "wcf": "w_f_diag", "wco": "w_o_diag", "scope": "lstm_cell" } # Inputs must be 2-dimensional. self.input_spec = input_spec.InputSpec(ndim=2)
def build(self, input_shape): if len(input_shape) < 5: raise ValueError('Inputs to `DepthwiseConv3D` should have rank 5. Received input shape:', str(input_shape)) input_shape = tensor_shape.TensorShape(input_shape) channel_axis = self._get_channel_axis() if input_shape.dims[channel_axis].value is None: raise ValueError('The channel dimension of the inputs to `DepthwiseConv3D` should be defined. Found `None`.') self.input_dim = int(input_shape[channel_axis]) depthwise_kernel_shape = (self.kernel_size[0], self.kernel_size[1], self.kernel_size[2], self.input_dim, self.depth_multiplier) self.depthwise_kernel = self.add_weight(shape=depthwise_kernel_shape, initializer=self.depthwise_initializer, name='depthwise_kernel', regularizer=self.depthwise_regularizer, constraint=self.depthwise_constraint) if self.use_bias: self.bias = self.add_weight(shape=(self.input_dim * self.depth_multiplier), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = input_spec.InputSpec(ndim=5, axes={channel_axis: self.input_dim}) self.built = True
def __init__(self, rank, filters, kernel_size, strides=1, padding='valid', data_format='channels_last', dilation_rate=1, activation=None, use_bias=True, kernel_initializer=None, bias_initializer=init_ops.zeros_initializer(), kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, trainable=True, name=None, **kwargs): super(_MaskedConv, self).__init__(trainable=trainable, name=name, activity_regularizer=activity_regularizer, **kwargs) self.rank = rank self.filters = filters self.kernel_size = utils.normalize_tuple(kernel_size, rank, 'kernel_size') self.strides = utils.normalize_tuple(strides, rank, 'strides') self.padding = utils.normalize_padding(padding) self.data_format = utils.normalize_data_format(data_format) self.dilation_rate = utils.normalize_tuple(dilation_rate, rank, 'dilation_rate') self.activation = activation self.use_bias = use_bias self.kernel_initializer = kernel_initializer self.bias_initializer = bias_initializer self.kernel_regularizer = kernel_regularizer self.bias_regularizer = bias_regularizer self.input_spec = input_spec.InputSpec(ndim=self.rank + 2)
def __init__(self, num_units, levels, activation=None, reuse=None, name=None, dtype=None, **kwargs): super(OnLSTMCell, self).__init__(_reuse=reuse, name=name, dtype=dtype, **kwargs) self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._levels = levels self.chunk_size = self._num_units / self._levels assert self._num_units % self._levels == 0 if activation: self._activation = activations.get(activation) else: self._activation = math_ops.tanh
def __init__(self): super(CustomerLayer, self).__init__() self.input_spec = input_spec.InputSpec(shape=(None, 3))
def __init__(self): super(CustomerLayer, self).__init__() self.input_spec = input_spec.InputSpec(axes={-1: 2})
def __init__(self): super(CustomerLayer, self).__init__() self.input_spec = input_spec.InputSpec(dtype='float32')
def __init__(self): super(CustomerLayer, self).__init__() self.input_spec = input_spec.InputSpec(max_ndim=2)
def build(self, input_shape): """Create variables of the Cudnn RNN. It can be called manually before `__call__()` or automatically through `__call__()`. In the former case, subsequent `__call__()`s will skip creating variables. Args: input_shape: network input tensor shape, a python list or a TensorShape object with 3 dimensions. Raises: ValueError: if input_shape has wrong dimension or unknown 3rd dimension. """ if self.built: return input_shape = tensor_shape.TensorShape(input_shape) if input_shape.ndims != 3: raise ValueError("Expecting input_shape with 3 dims, got %d" % input_shape.ndims) if input_shape[-1].value is None: raise ValueError("The last dimension of the inputs to `CudnnRNN` " "should be defined. Found `None`.") self._input_size = input_shape[-1].value self.input_spec = input_spec.InputSpec(ndim=3, axes={-1: self._input_size}) self._set_scope(None) # Not using base class `add_variable()` since the it calls # `tf.compat.v1.get_variable()` with a callable initializer whereas here # with a tensor. The difference is mandated to support forward-compatibility # with Cudnn. with vs.variable_scope(self._scope, reuse=self.built, custom_getter=self._update_trainable_weights): if self._kernel_initializer is None: self._kernel_initializer = init_ops.glorot_uniform_initializer( seed=self._seed, dtype=self._plain_dtype) if self._bias_initializer is None: self._bias_initializer = init_ops.constant_initializer( 0.0, dtype=self._plain_dtype) weights = [ self._kernel_initializer(sp, dtype=self._plain_dtype) for sp in self.canonical_weight_shapes ] biases = [ self._bias_initializer(sp, dtype=self._plain_dtype) for sp in self.canonical_bias_shapes ] opaque_params_t = self._canonical_to_opaque(weights, biases) if vs.get_variable_scope().partitioner is not None: logging.warn( "Partitioner is not supported for Cudnn RNN layer variables, using " "it will create forward-compatibility issues with future " "CUDA/CuDNN generations.") # Initialize opaque params with a tensor with unknown shape, thus couldn't # use self.add_variable(name, shape, initializer, ...) self.kernel = vs.get_variable("opaque_kernel", dtype=self._plain_dtype, initializer=opaque_params_t, validate_shape=False) # Create saveable in the outer scope of the cudnn subgraph, such that # alternative subgraph with platform-independent rnn cells can load the # checkpoints directly. if not (self.built or vs.get_variable_scope().reuse is True): self._create_saveable() self.built = True
def test_model(self, strategy_fn, use_operator=False, use_regularizer=False, policy_name='mixed_float16', get_config=False, save_format=None, use_input_spec=False): self._skip_if_strategy_unsupported(strategy_fn) self._skip_if_save_format_unsupported(save_format) regularizer = (mp_test_util.IdentityRegularizer() if use_regularizer else None) with strategy_fn().scope(): # Pass loss_scale=None, as this test will fail if the DynamicLossScale # skips applying gradients for a step with policy.policy_scope( policy.Policy(policy_name, loss_scale=None)): layer = mp_test_util.MultiplyLayer(assert_type=dtypes.float16, use_operator=use_operator, regularizer=regularizer, input_shape=(1, )) if use_input_spec: layer.input_spec = input_spec.InputSpec(shape=(2, 1)) model = testing_utils.get_model_from_layers( [layer], input_shape=(1, ), input_dtype=dtypes.float16) if get_config: config = model.get_config() model = model.__class__.from_config( config, custom_objects={ 'MultiplyLayer': mp_test_util.MultiplyLayer }) (layer, ) = ( layer for layer in model.layers if isinstance(layer, mp_test_util.MultiplyLayer)) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) # Learning rate is small enough that if applied to a float16 variable, # the variable will not change. So this tests the learning rate not # applied to a float16 value, but instead the float32 variable. opt = gradient_descent.SGD(2**-14) model.compile(opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((2, 1)) y = np.ones((2, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2) model.fit(dataset) # Variable starts at 1, and should have gradient of 2 ** -14 subtracted # from it. expected = 1 - 2**-14 if use_regularizer: # Regularizer adds another 2 ** -14 to the gradient. expected -= 2**-14 self.assertEqual(backend.eval(layer.v), expected) if save_format: with generic_utils.CustomObjectScope({ 'MultiplyLayer': mp_test_util.MultiplyLayer, 'loss_fn': loss_fn }): self._test_saving(model, dataset, save_format, use_regularizer)
def test_defined_shape(self): spec = input_spec.InputSpec(shape=[1, None, 2, 3]) self.assertAllEqual([1, None, 2, 3], input_spec.to_tensor_shape(spec).as_list())