def _dnn_logits(self, features, is_training=False):
net = layers.input_from_feature_columns(
features,
self._get_dnn_feature_columns(),
weight_collections=[self._dnn_weight_collection])
for layer_id, num_hidden_units in enumerate(self._dnn_hidden_units):
net = layers.legacy_fully_connected(
net,
num_hidden_units,
activation_fn=self._dnn_activation_fn,
weight_collections=[self._dnn_weight_collection],
bias_collections=[self._dnn_weight_collection],
name="hiddenlayer_%d" % layer_id)
if self._dnn_dropout is not None and is_training:
net = layers.dropout(
net,
keep_prob=(1.0 - self._dnn_dropout))
self._add_hidden_layer_summary(net, "hiddenlayer_%d" % layer_id)
logit = layers.legacy_fully_connected(
net,
self._num_label_columns(),
weight_collections=[self._dnn_weight_collection],
bias_collections=[self._dnn_weight_collection],
name="dnn_logit")
self._add_hidden_layer_summary(logit, "dnn_logit")
return logit
def _dnn_logits(self, features, is_training=False):
net = layers.input_from_feature_columns(
features,
self._get_dnn_feature_columns(),
weight_collections=[self._dnn_weight_collection])
for layer_id, num_hidden_units in enumerate(self._dnn_hidden_units):
net = layers.legacy_fully_connected(
net,
num_hidden_units,
activation_fn=self._dnn_activation_fn,
weight_collections=[self._dnn_weight_collection],
bias_collections=[self._dnn_weight_collection],
name="hiddenlayer_%d" % layer_id)
if self._dnn_dropout is not None and is_training:
net = layers.dropout(
net,
keep_prob=(1.0 - self._dnn_dropout))
self._add_hidden_layer_summary(net, "hiddenlayer_%d" % layer_id)
logit = layers.legacy_fully_connected(
net,
self._num_label_columns(),
weight_collections=[self._dnn_weight_collection],
bias_collections=[self._dnn_weight_collection],
name="dnn_logit")
self._add_hidden_layer_summary(logit, "dnn_logit")
return logit
def dnn(tensor_in, hidden_units, activation=nn.relu, dropout=None):
"""Creates fully connected deep neural network subgraph.
Args:
tensor_in: tensor or placeholder for input features.
hidden_units: list of counts of hidden units in each layer.
activation: activation function between layers. Can be None.
dropout: if not None, will add a dropout layer with given probability.
Returns:
A tensor which would be a deep neural network.
"""
with vs.variable_scope('dnn'):
for i, n_units in enumerate(hidden_units):
with vs.variable_scope('layer%d' % i):
# Weight initializer was set to None to replicate the behavior of
# rnn_cell.linear. Using fully_connected's default initializer gets
# slightly worse quality results on unit tests.
tensor_in = layers.legacy_fully_connected(
tensor_in,
n_units,
weight_init=None,
weight_collections=['dnn_weights'],
bias_collections=['dnn_biases'])
if activation is not None:
tensor_in = activation(tensor_in)
if dropout is not None:
is_training = array_ops_.squeeze(ops.get_collection('IS_TRAINING'))
tensor_in = control_flow_ops.cond(
is_training,
lambda: dropout_ops.dropout(tensor_in, prob=(1.0 - dropout)),
lambda: tensor_in)
return tensor_in
def _propagate(dim_indices, conf, cell, c_prev, m_prev, new_output, new_state,
first_call):
"""Propagates through all the cells in dim_indices dimensions.
"""
if len(dim_indices) == 0:
return
# Because of the way RNNCells are implemented, we take the last dimension
# (H_{N-1}) out and feed it as the state of the RNN cell
# (in `last_dim_output`).
# The input of the cell (H_0 to H_{N-2}) are concatenated into `cell_inputs`
if conf.num_dims > 1:
ls_cell_inputs = [None] * (conf.num_dims - 1)
for d in conf.dims[:-1]:
ls_cell_inputs[d.idx] = new_output[d.idx] if new_output[
d.idx] is not None else m_prev[d.idx]
cell_inputs = array_ops.concat_v2(ls_cell_inputs, 1)
else:
cell_inputs = array_ops.zeros([m_prev[0].get_shape().as_list()[0], 0],
m_prev[0].dtype)
last_dim_output = new_output[-1] if new_output[-1] is not None else m_prev[
-1]
for i in dim_indices:
d = conf.dims[i]
if d.non_recurrent_fn:
linear_args = array_ops.concat_v2(
[cell_inputs, last_dim_output],
1) if conf.num_dims > 1 else last_dim_output
with vs.variable_scope('non_recurrent' if conf.tied else
'non_recurrent/cell_{}'.format(i)):
if conf.tied and not (first_call and i == dim_indices[0]):
vs.get_variable_scope().reuse_variables()
new_output[d.idx] = layers.legacy_fully_connected(
linear_args,
num_output_units=conf.num_units,
activation_fn=d.non_recurrent_fn,
weight_init=vs.get_variable_scope().initializer
or layers.initializers.xavier_initializer)
else:
if c_prev[i] is not None:
cell_state = array_ops.concat_v2([c_prev[i], last_dim_output],
1)
else:
# for GRU/RNN, the state is just the previous output
cell_state = last_dim_output
with vs.variable_scope('recurrent' if conf.
tied else 'recurrent/cell_{}'.format(i)):
if conf.tied and not (first_call and i == dim_indices[0]):
vs.get_variable_scope().reuse_variables()
new_output[d.idx], new_state[d.idx] = cell(
cell_inputs, cell_state)
def _propagate(dim_indices, conf, cells, c_prev, m_prev, new_output, new_state,
first_call):
"""Propagates through all the cells in dim_indices dimensions.
"""
if len(dim_indices) == 0:
return
# Because of the way RNNCells are implemented, we take the last dimension
# (H_{N-1}) out and feed it as the state of the RNN cell
# (in `last_dim_output`).
# The input of the cell (H_0 to H_{N-2}) are concatenated into `cell_inputs`
if conf.num_dims > 1:
ls_cell_inputs = [None] * (conf.num_dims - 1)
for d in conf.dims[:-1]:
ls_cell_inputs[d.idx] = new_output[d.idx] if new_output[
d.idx] is not None else m_prev[d.idx]
cell_inputs = array_ops.concat(ls_cell_inputs, 1)
else:
cell_inputs = array_ops.zeros([m_prev[0].get_shape().as_list()[0], 0],
m_prev[0].dtype)
last_dim_output = new_output[-1] if new_output[-1] is not None else m_prev[-1]
for i in dim_indices:
d = conf.dims[i]
if d.non_recurrent_fn:
linear_args = array_ops.concat(
[cell_inputs, last_dim_output],
1) if conf.num_dims > 1 else last_dim_output
with vs.variable_scope('non_recurrent' if conf.tied else
'non_recurrent/cell_{}'.format(i)):
if conf.tied and not (first_call and i == dim_indices[0]):
vs.get_variable_scope().reuse_variables()
new_output[d.idx] = layers.legacy_fully_connected(
linear_args,
num_output_units=conf.num_units,
activation_fn=d.non_recurrent_fn,
weight_init=vs.get_variable_scope().initializer or
layers.initializers.xavier_initializer)
else:
if c_prev[i] is not None:
cell_state = array_ops.concat([c_prev[i], last_dim_output], 1)
else:
# for GRU/RNN, the state is just the previous output
cell_state = last_dim_output
with vs.variable_scope('recurrent' if conf.tied else
'recurrent/cell_{}'.format(i)):
if conf.tied and not (first_call and i == dim_indices[0]):
vs.get_variable_scope().reuse_variables()
cell = cells[i]
new_output[d.idx], new_state[d.idx] = cell(cell_inputs, cell_state)
def _dnn_logits(self, features):
net = layers.input_from_feature_columns(
features,
self._get_dnn_feature_columns(),
weight_collections=[self._dnn_weight_collection])
for layer_id, num_hidden_units in enumerate(self._dnn_hidden_units):
net = layers.legacy_fully_connected(
net,
num_hidden_units,
activation_fn=self._dnn_activation_fn,
weight_collections=[self._dnn_weight_collection],
bias_collections=[self._dnn_weight_collection],
name="hiddenlayer_%d" % layer_id)
self._add_hidden_layer_summary(net, "hiddenlayer_%d" % layer_id)
logit = layers.legacy_fully_connected(
net,
self._num_label_columns(),
weight_collections=[self._dnn_weight_collection],
bias_collections=[self._dnn_weight_collection],
name="dnn_logit")
self._add_hidden_layer_summary(logit, "dnn_logit")
return logit
