def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten. Returns: 2D Tensor [samples, num_units]. """ self._declare_dependencies() input_shape = get_shape(incoming) incoming = validate_dtype(incoming) assert len( input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' n_inputs = total_tensor_depth(tensor_shape=input_shape) regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True) self._w = variable(name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer, initializer=getters.get_initializer( self.weights_init), trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 2: inference = tf.reshape(tensor=inference, shape=[-1, n_inputs]) inference = tf.matmul(a=inference, b=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype, initializer=getters.get_initializer( self.bias_init), trainable=self.trainable, restore=self.restore) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.nn.bias_add(value=inference, bias=self._b) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) if self._dropout: inference = self._dropout(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten. Returns: 2D Tensor [samples, num_units]. """ self._declare_dependencies() input_shape = get_shape(incoming) assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' n_inputs = total_tensor_depth(tensor_shape=input_shape) regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True) initializer = getters.get_initializer(self.weights_init) self._w = variable(name='w', shape=[n_inputs, self.num_units], regularizer=regularizer, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) self._b = variable(name='b', shape=[self.num_units], initializer=getters.get_initializer(self.bias_init), trainable=self.trainable, restore=self.restore) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) # Weight and bias for the transform gate self._w_t = variable(name='w_t', shape=[n_inputs, self.num_units], regularizer=None, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name) self._b_t = variable(name='b_t', shape=[self.num_units], initializer=tf.constant_initializer(-1), trainable=self.trainable, restore=self.restore) track(self._b_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name) # If input is not 2d, flatten it. if len(input_shape) > 2: incoming = tf.reshape(tensor=incoming, shape=[-1, n_inputs]) H = getters.get_activation(self.activation)(tf.matmul(a=incoming, b=self._w) + self._b) T = tf.sigmoid(tf.matmul(a=incoming, b=self._w_t) + self._b_t) if self._transform_dropout: T = self._transform_dropout(T) C = tf.subtract(x=1.0, y=T) inference = tf.add(x=tf.multiply(x=H, y=T), y=tf.multiply(x=incoming, y=C)) track(inference, tf.GraphKeys.ACTIVATIONS) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _linear(args, output_size, bias, bias_start=0.0, weights_init=None, trainable=True, restore=True, scope=None): """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. Args: args: a 2D Tensor or a list of 2D, batch x n, Tensors. output_size: int, second dimension of W[i]. bias: boolean, whether to add a bias term or not. bias_start: starting value to initialize the bias; 0 by default. scope: VariableScope for the created subgraph; defaults to "Linear". Returns: A 2D Tensor with shape [batch x output_size] equal to sum_i(args[i] * W[i]), where W[i]s are newly created matrices. Raises: ValueError: if some of the arguments has unspecified or wrong shape. """ if args is None or (nest.is_sequence(args) and not args): raise ValueError('`args` must be specified') if not nest.is_sequence(args): args = [args] # Calculate the total size of arguments on dimension 1. total_arg_size = 0 shapes = [a.get_shape().as_list() for a in args] for shape in shapes: if len(shape) != 2: raise ValueError('Linear is expecting 2D arguments: %s' % str(shapes)) if not shape[1]: raise ValueError('Linear expects shape[1] of arguments: %s' % str(shapes)) else: total_arg_size += shape[1] # Now the computation. with get_variable_scope(scope or 'Linear'): _w = variable(name='w', shape=[total_arg_size, output_size], initializer=weights_init, trainable=trainable, restore=restore) if len(args) == 1: res = tf.matmul(a=args[0], b=_w) else: res = tf.matmul(a=array_ops.concat(values=args, axis=1), b=_w) if not bias: return res _b = variable(name='b', shape=[output_size], initializer=tf.constant_initializer(bias_start), trainable=trainable, restore=restore) return res + _b
def _build(self, incoming, *args, **kwargs): input_shape = get_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=self.gamma, stddev=self.stddev) self._beta = variable(name='beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(self.beta), trainable=self.trainable, restore=self.restore) self._gamma = variable(name='gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=self.trainable, restore=self.restore) track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name) track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if not self.restore: track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta) track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma) axis = list(xrange(input_ndim - 1)) moving_mean = variable(name='moving_mean', shape=input_shape[-1:], initializer=tf.zeros_initializer(), trainable=False, restore=self.restore) moving_variance = variable(name='moving_variance', shape=input_shape[-1:], initializer=tf.constant_initializer(1.), trainable=False, restore=self.restore) def update_mean_var(): mean, variance = tf.nn.moments(x=incoming, axes=axis) update_moving_mean = moving_averages.assign_moving_average( variable=moving_mean, value=mean, decay=self.decay, zero_debias=False) update_moving_variance = moving_averages.assign_moving_average( variable=moving_variance, value=variance, decay=self.decay, zero_debias=False) with tf.control_dependencies([update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) # Retrieve variable managing training mode if Modes.is_train(self.mode): mean, var = update_mean_var() else: mean, var = moving_mean, moving_variance incoming = tf.nn.batch_normalization(x=incoming, mean=mean, variance=var, offset=self._beta, scale=self._gamma, variance_epsilon=self.epsilon) incoming.set_shape(input_shape) track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name) return incoming
def _build(self, incoming, *args, **kwargs): """ Args: 2-D Tensor [samples, ids]. Returns: 3-D Tensor [samples, embedded_ids, features]. """ input_shape = get_shape(incoming) assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D' weights_init = getters.get_initializer(self.weights_init) self._w = variable('w', shape=[self.input_dim, self.output_dim], initializer=weights_init, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.cast(x=incoming, dtype=tf.int32) inference = tf.nn.embedding_lookup( params=self._w, ids=inference, validate_indices=self.validate_indices) # Embedding doesn't support masking, so we save sequence length prior to the lookup. # Expand dim to 3d. shape = [-1] + inference.get_shape().as_list()[1:3] + [1] inference.seq_length = retrieve_seq_length_op( tf.reshape(incoming, shape)) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: 2-D Tensor [samples, ids]. Returns: 3-D Tensor [samples, embedded_ids, features]. """ input_shape = get_shape(incoming) assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D' weights_init = getters.get_initializer(self.weights_init) self._w = variable('w', shape=[self.input_dim, self.output_dim], initializer=weights_init, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.cast(x=incoming, dtype=tf.int32) inference = tf.nn.embedding_lookup( params=self._w, ids=inference, validate_indices=self.validate_indices) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def create_global_counter(collection, name, graph=None): """Create global counter tensor in graph. Args: collection: the counter's collection. name: the counter's name. graph: The graph in which to create the global counter tensor. If missing, use default graph. Returns: Global step tensor. Raises: ValueError: if global counter tensor is already defined. """ graph = graph or tf.get_default_graph() if get_global_counter(collection, name, graph) is not None: raise ValueError("`{}` already exists.".format(collection)) # Create in proper graph and base name_scope. with graph.as_default() as g, g.name_scope(None): return variable( collection, shape=[], dtype=tf.int64, initializer=getters.get_initializer('zeros', dtype=tf.int64), trainable=False, collections=[tf.GraphKeys.GLOBAL_VARIABLES, collection])
def _build(self, incoming, *args, **kwargs): """ Args: incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten. Returns: 2D Tensor [samples, num_units]. """ self._declare_dependencies() input_shape = get_shape(incoming) incoming = validate_dtype(incoming) assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D' n_inputs = total_tensor_depth(tensor_shape=input_shape) regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True) self._w = variable( name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer, initializer=getters.get_initializer(self.weights_init), trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 2: inference = tf.reshape(tensor=inference, shape=[-1, n_inputs]) inference = tf.matmul(a=inference, b=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype, initializer=getters.get_initializer(self.bias_init), trainable=self.trainable, restore=self.restore) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.nn.bias_add(value=inference, bias=self._b) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) if self._dropout: inference = self._dropout(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: incoming: 1-D Tensor [samples]. If not 2D, input will be flatten. Returns: 1-D Tensor [samples]. """ input_shape = get_shape(incoming) n_inputs = int(np.prod(a=input_shape[1:])) initializer = tf.constant_initializer(value=np.random.randn()) self._w = variable(name='w', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 1: inference = tf.reshape(tensor=inference, shape=[-1]) inference = tf.multiply(x=inference, y=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) inference = tf.add(inference, self._b) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _prelu(x, name): with get_name_scope(name): if channel_shared: w_shape = (1,) else: w_shape = get_shape(x)[-1:] w_init = getters.get_initializer(weights_init) alphas = variable(shape=w_shape, initializer=w_init, restore=restore, name="alphas") x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5 x.alphas = alphas return x
def _prelu(x, name): with get_name_scope(name): if channel_shared: w_shape = (1,) else: w_shape = get_shape(x)[-1:] W_init = getters.get_initializer(weights_init) alphas = variable(shape=w_shape, initializer=W_init, restore=restore, name="alphas") x = tf.nn.relu(features=x) + tf.multiply(x=alphas, y=(x - tf.abs(x))) * 0.5 x.alphas = alphas return x
def _build(self, incoming, *args, **kwargs): """ Args: incoming: 1-D Tensor [samples]. If not 2D, input will be flatten. Returns: 1-D Tensor [samples]. """ input_shape = get_shape(incoming) n_inputs = total_tensor_depth(tensor_shape=input_shape) initializer = tf.constant_initializer(value=np.random.randn()) self._w = variable(name='w', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = incoming # If input is not 2d, flatten it. if len(input_shape) > 1: inference = tf.reshape(tensor=inference, shape=[-1]) inference = tf.multiply(x=inference, y=self._w) self._b = None if self.bias: self._b = variable(name='b', shape=[n_inputs], dtype=incoming.dtype, initializer=initializer, trainable=self.trainable, restore=self.restore) inference = tf.add(inference, self._b) track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if self.activation: inference = getters.get_activation(self.activation, collect=True)(inference) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): """ Args: 2-D Tensor [samples, ids]. Returns: 3-D Tensor [samples, embedded_ids, features]. """ input_shape = get_shape(incoming) assert len(input_shape) == 2, 'Incoming Tensor shape must be 2-D' weights_init = getters.get_initializer(self.weights_init) self._w = variable('w', shape=[self.input_dim, self.output_dim], initializer=weights_init, trainable=self.trainable, restore=self.restore) track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name) inference = tf.cast(x=incoming, dtype=tf.int32) inference = tf.nn.embedding_lookup(params=self._w, ids=inference, validate_indices=self.validate_indices) track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name) return inference
def _build(self, incoming, *args, **kwargs): input_shape = get_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=self.gamma, stddev=self.stddev) self._beta = variable(name='beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(self.beta), trainable=self.trainable, restore=self.restore) self._gamma = variable(name='gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=self.trainable, restore=self.restore) track(self._beta, tf.GraphKeys.LAYER_VARIABLES, self.module_name) track(self._gamma, tf.GraphKeys.LAYER_VARIABLES, self.module_name) if not self.restore: track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._beta) track(tf.GraphKeys.EXCL_RESTORE_VARIABLES, self._gamma) axis = list(xrange(input_ndim - 1)) moving_mean = variable(name='moving_mean', shape=input_shape[-1:], initializer=tf.zeros_initializer(), trainable=False, restore=self.restore) moving_variance = variable(name='moving_variance', shape=input_shape[-1:], initializer=tf.constant_initializer(1.), trainable=False, restore=self.restore) def update_mean_var(): mean, variance = tf.nn.moments(x=incoming, axes=axis) update_moving_mean = moving_averages.assign_moving_average( variable=moving_mean, value=mean, decay=self.decay, zero_debias=False) update_moving_variance = moving_averages.assign_moving_average( variable=moving_variance, value=variance, decay=self.decay, zero_debias=False) with tf.control_dependencies( [update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) # Retrieve variable managing training mode if Modes.is_train(self.mode): mean, var = update_mean_var() else: mean, var = moving_mean, moving_variance incoming = tf.nn.batch_normalization(x=incoming, mean=mean, variance=var, offset=self._beta, scale=self._gamma, variance_epsilon=self.epsilon) incoming.set_shape(input_shape) track(incoming, tf.GraphKeys.LAYER_TENSOR, self.module_name) return incoming