def _dnn_logits(self, features, is_training=False): net = layers.input_from_feature_columns( features, self._get_dnn_feature_columns(), weight_collections=[self._dnn_weight_collection]) for layer_id, num_hidden_units in enumerate(self._dnn_hidden_units): net = layers.legacy_fully_connected( net, num_hidden_units, activation_fn=self._dnn_activation_fn, weight_collections=[self._dnn_weight_collection], bias_collections=[self._dnn_weight_collection], name="hiddenlayer_%d" % layer_id) if self._dnn_dropout is not None and is_training: net = layers.dropout( net, keep_prob=(1.0 - self._dnn_dropout)) self._add_hidden_layer_summary(net, "hiddenlayer_%d" % layer_id) logit = layers.legacy_fully_connected( net, self._num_label_columns(), weight_collections=[self._dnn_weight_collection], bias_collections=[self._dnn_weight_collection], name="dnn_logit") self._add_hidden_layer_summary(logit, "dnn_logit") return logit
def dnn(tensor_in, hidden_units, activation=nn.relu, dropout=None): """Creates fully connected deep neural network subgraph. Args: tensor_in: tensor or placeholder for input features. hidden_units: list of counts of hidden units in each layer. activation: activation function between layers. Can be None. dropout: if not None, will add a dropout layer with given probability. Returns: A tensor which would be a deep neural network. """ with vs.variable_scope('dnn'): for i, n_units in enumerate(hidden_units): with vs.variable_scope('layer%d' % i): # Weight initializer was set to None to replicate the behavior of # rnn_cell.linear. Using fully_connected's default initializer gets # slightly worse quality results on unit tests. tensor_in = layers.legacy_fully_connected( tensor_in, n_units, weight_init=None, weight_collections=['dnn_weights'], bias_collections=['dnn_biases']) if activation is not None: tensor_in = activation(tensor_in) if dropout is not None: is_training = array_ops_.squeeze(ops.get_collection('IS_TRAINING')) tensor_in = control_flow_ops.cond( is_training, lambda: dropout_ops.dropout(tensor_in, prob=(1.0 - dropout)), lambda: tensor_in) return tensor_in
def _propagate(dim_indices, conf, cell, c_prev, m_prev, new_output, new_state, first_call): """Propagates through all the cells in dim_indices dimensions. """ if len(dim_indices) == 0: return # Because of the way RNNCells are implemented, we take the last dimension # (H_{N-1}) out and feed it as the state of the RNN cell # (in `last_dim_output`). # The input of the cell (H_0 to H_{N-2}) are concatenated into `cell_inputs` if conf.num_dims > 1: ls_cell_inputs = [None] * (conf.num_dims - 1) for d in conf.dims[:-1]: ls_cell_inputs[d.idx] = new_output[d.idx] if new_output[ d.idx] is not None else m_prev[d.idx] cell_inputs = array_ops.concat_v2(ls_cell_inputs, 1) else: cell_inputs = array_ops.zeros([m_prev[0].get_shape().as_list()[0], 0], m_prev[0].dtype) last_dim_output = new_output[-1] if new_output[-1] is not None else m_prev[ -1] for i in dim_indices: d = conf.dims[i] if d.non_recurrent_fn: linear_args = array_ops.concat_v2( [cell_inputs, last_dim_output], 1) if conf.num_dims > 1 else last_dim_output with vs.variable_scope('non_recurrent' if conf.tied else 'non_recurrent/cell_{}'.format(i)): if conf.tied and not (first_call and i == dim_indices[0]): vs.get_variable_scope().reuse_variables() new_output[d.idx] = layers.legacy_fully_connected( linear_args, num_output_units=conf.num_units, activation_fn=d.non_recurrent_fn, weight_init=vs.get_variable_scope().initializer or layers.initializers.xavier_initializer) else: if c_prev[i] is not None: cell_state = array_ops.concat_v2([c_prev[i], last_dim_output], 1) else: # for GRU/RNN, the state is just the previous output cell_state = last_dim_output with vs.variable_scope('recurrent' if conf. tied else 'recurrent/cell_{}'.format(i)): if conf.tied and not (first_call and i == dim_indices[0]): vs.get_variable_scope().reuse_variables() new_output[d.idx], new_state[d.idx] = cell( cell_inputs, cell_state)
def _propagate(dim_indices, conf, cells, c_prev, m_prev, new_output, new_state, first_call): """Propagates through all the cells in dim_indices dimensions. """ if len(dim_indices) == 0: return # Because of the way RNNCells are implemented, we take the last dimension # (H_{N-1}) out and feed it as the state of the RNN cell # (in `last_dim_output`). # The input of the cell (H_0 to H_{N-2}) are concatenated into `cell_inputs` if conf.num_dims > 1: ls_cell_inputs = [None] * (conf.num_dims - 1) for d in conf.dims[:-1]: ls_cell_inputs[d.idx] = new_output[d.idx] if new_output[ d.idx] is not None else m_prev[d.idx] cell_inputs = array_ops.concat(ls_cell_inputs, 1) else: cell_inputs = array_ops.zeros([m_prev[0].get_shape().as_list()[0], 0], m_prev[0].dtype) last_dim_output = new_output[-1] if new_output[-1] is not None else m_prev[-1] for i in dim_indices: d = conf.dims[i] if d.non_recurrent_fn: linear_args = array_ops.concat( [cell_inputs, last_dim_output], 1) if conf.num_dims > 1 else last_dim_output with vs.variable_scope('non_recurrent' if conf.tied else 'non_recurrent/cell_{}'.format(i)): if conf.tied and not (first_call and i == dim_indices[0]): vs.get_variable_scope().reuse_variables() new_output[d.idx] = layers.legacy_fully_connected( linear_args, num_output_units=conf.num_units, activation_fn=d.non_recurrent_fn, weight_init=vs.get_variable_scope().initializer or layers.initializers.xavier_initializer) else: if c_prev[i] is not None: cell_state = array_ops.concat([c_prev[i], last_dim_output], 1) else: # for GRU/RNN, the state is just the previous output cell_state = last_dim_output with vs.variable_scope('recurrent' if conf.tied else 'recurrent/cell_{}'.format(i)): if conf.tied and not (first_call and i == dim_indices[0]): vs.get_variable_scope().reuse_variables() cell = cells[i] new_output[d.idx], new_state[d.idx] = cell(cell_inputs, cell_state)
def _dnn_logits(self, features): net = layers.input_from_feature_columns( features, self._get_dnn_feature_columns(), weight_collections=[self._dnn_weight_collection]) for layer_id, num_hidden_units in enumerate(self._dnn_hidden_units): net = layers.legacy_fully_connected( net, num_hidden_units, activation_fn=self._dnn_activation_fn, weight_collections=[self._dnn_weight_collection], bias_collections=[self._dnn_weight_collection], name="hiddenlayer_%d" % layer_id) self._add_hidden_layer_summary(net, "hiddenlayer_%d" % layer_id) logit = layers.legacy_fully_connected( net, self._num_label_columns(), weight_collections=[self._dnn_weight_collection], bias_collections=[self._dnn_weight_collection], name="dnn_logit") self._add_hidden_layer_summary(logit, "dnn_logit") return logit