def __call__(self, inputs, *args, **kwargs): with vs.variable_scope(self._scope, reuse=self.built or self._reuse) as scope: with ops.name_scope(scope.original_name_scope): if not self.built: input_shapes = [ x.get_shape() for x in nest.flatten(inputs) ] if len(input_shapes) == 1: self.build(input_shapes[0]) else: self.build(input_shapes) outputs = self.call(inputs, *args, **kwargs) # # Apply activity regularization. # # Note that it should be applied every time the layer creates a new # # output, since it is output-specific. # if hasattr(self, 'activity_regularizer') and self.activity_regularizer: # output_list = _to_list(outputs) # for output in output_list: # with ops.name_scope('ActivityRegularizer'): # activity_regularization = self.activity_regularizer(output) # self.add_loss(activity_regularization) # _add_elements_to_collection( # activity_regularization, ops.GraphKeys.REGULARIZATION_LOSSES) # Update global default collections. _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) self.built = True return outputs
def add_variable(self, name, shape, dtype=None, trainable=True, initializer=None, regularizer=None): if dtype is None: dtype = self.dtype existing_variables = set(tf_variables.global_variables()) with vs.variable_scope(self._scope, reuse=self.built or self._reuse) as scope: with ops.name_scope(scope.original_name_scope): full_name = get_tensor_scope() + name variable = vs.get_variable(name, shape=shape, initializer=initializer, dtype=dtypes.as_dtype(dtype), trainable=trainable and self.trainable) if variable in existing_variables: # Work only if the layer is built return variable if regularizer: raise NotImplementedError() if trainable: self._trainable_weights.append(variable) else: self._non_trainable_weights.append(variable) return variable
def batch_norm(inputs, decay=0.999, center=True, scale=False, epsilon=0.001, activation_fn=None, param_initializers=None, param_regularizers=None, updates_collections=ops.GraphKeys.UPDATE_OPS, is_training=True, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, batch_weights=None, fused=False, data_format=DATA_FORMAT_NHWC, zero_debias_moving_mean=False, scope=None, renorm=False, renorm_clipping=None, renorm_decay=0.99): scope = _default_scope(scope, 'BATCH_NORM', 'BatchNorm') if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): raise ValueError('data_format has to be either NCHW or NHWC.') axis = 1 if data_format == DATA_FORMAT_NCHW else -1 with vs.variable_scope(scope, reuse=reuse) as sc: if not param_initializers: param_initializers = {} beta_initializer = param_initializers.get('beta', init_ops.zeros_initializer()) gamma_initializer = param_initializers.get('gamma', init_ops.ones_initializer()) moving_mean_initializer = param_initializers.get('moving_mean', init_ops.zeros_initializer()) moving_variance_initializer = param_initializers.get('moving_variance', init_ops.ones_initializer()) if not param_regularizers: param_regularizers = {} beta_regularizer = param_regularizers.get('beta') gamma_regularizer = param_regularizers.get('gamma') return layers.batch_norm( inputs=inputs, axis=axis, momentum=decay, epsilon=epsilon, center=center, scale=scale, beta_initializer=beta_initializer, gamma_initializer=gamma_initializer, moving_mean_initializer=moving_mean_initializer, moving_variance_initializer=moving_variance_initializer, beta_regularizer=beta_regularizer, gamma_regularizer=gamma_regularizer, trainable=trainable, renorm=renorm, renorm_clipping=renorm_clipping, renorm_momentum=renorm_decay, fused=fused, training=is_training)
def fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): scope = _default_scope(scope, 'FULLY_CONNECTED', 'fully_connected') with vs.variable_scope(scope, reuse=reuse) as sc: return layers.dense( inputs=inputs, units=num_outputs, activation=activation_fn, use_bias=True if biases_initializer is not None else False, kernel_initializer=weights_initializer, bias_initializer=biases_initializer, bias_regularizer=biases_regularizer, activity_regularizer=None, trainable=trainable, reuse=reuse)
def __call__(self, inputs, *args, **kwargs): with vs.variable_scope(self._scope, reuse=self.built or self._reuse) as scope: with ops.name_scope(scope.original_name_scope): if not self.built: input_shapes = [x.get_shape() for x in nest.flatten(inputs)] if len(input_shapes) == 1: self.build(input_shapes[0]) else: self.build(input_shapes) outputs = self.call(inputs, *args, **kwargs) # # Apply activity regularization. # # Note that it should be applied every time the layer creates a new # # output, since it is output-specific. # if hasattr(self, 'activity_regularizer') and self.activity_regularizer: # output_list = _to_list(outputs) # for output in output_list: # with ops.name_scope('ActivityRegularizer'): # activity_regularization = self.activity_regularizer(output) # self.add_loss(activity_regularization) # _add_elements_to_collection( # activity_regularization, ops.GraphKeys.REGULARIZATION_LOSSES) # Update global default collections. _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) self.built = True return outputs
def __call__(self, inputs, *args, **kwargs): with vs.variable_scope(self.name, reuse=self.built or self._reuse) as scope: if not self.built: input_shapes = [x.get_shape() for x in nest.flatten(inputs)] if len(input_shapes) == 1: self.build(input_shapes[0]) else: self.build(input_shapes) outputs = self.call(inputs, *args, **kwargs) # Update global default collections. _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) return outputs
def convolution( inputs, num_outputs, kernel_size, stride=1, padding='SAME', data_format=None, rate=1, activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None, ): scope = _default_scope(scope, 'Conv') if data_format not in [None, 'NHWC', 'NCHW']: raise ValueError('Invalid data_format: %r' % (data_format,)) data_format = 'channels_first' if data_format == 'NCHW' else 'channels_last' input_rank = inputs.get_shape().ndims with vs.variable_scope(scope, reuse=reuse) as sc: if input_rank == 4: return layers.conv2d( inputs=inputs, filters=num_outputs, kernel_size=kernel_size, strides=stride, padding=padding, data_format=data_format, dilation_rate=rate, activation=activation_fn, use_bias=True if biases_initializer is not None else False, kernel_initializer=weights_initializer, bias_initializer=biases_initializer, bias_regularizer=biases_regularizer, activity_regularizer=None, trainable=trainable, reuse=reuse, )
def convolution(inputs, num_outputs, kernel_size, stride=1, padding='SAME', data_format=None, rate=1, activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): scope = _default_scope(scope, 'CONVOLUTION', 'Conv') if data_format not in [None, 'NHWC', 'NCHW']: raise ValueError('Invalid data_format: %r' % (data_format,)) data_format = 'channels_first' if data_format == 'NCHW' else 'channels_last' input_rank = inputs.get_shape().ndims with vs.variable_scope(scope, reuse=reuse) as sc: if input_rank == 4: return layers.conv2d( inputs=inputs, filters=num_outputs, kernel_size=kernel_size, strides=stride, padding=padding, data_format=data_format, dilation_rate=rate, activation=activation_fn, use_bias=True if biases_initializer is not None else False, kernel_initializer=weights_initializer, bias_initializer=biases_initializer, bias_regularizer=biases_regularizer, activity_regularizer=None, trainable=trainable, reuse=reuse)