def _dynamic_rnn_model_fn(features, labels, mode):
"""The model to be passed to an `Estimator`."""
with ops.name_scope(name):
sequence_length = features.get(sequence_length_key)
sequence_input = build_sequence_input(features,
sequence_feature_columns,
context_feature_columns)
dropout = (dropout_keep_probabilities
if mode == model_fn.ModeKeys.TRAIN
else None)
# This class promises to use the cell type selected by that function.
cell = rnn_common.construct_rnn_cell(num_units, cell_type, dropout)
initial_state = dict_to_state_tuple(features, cell)
rnn_activations, final_state = construct_rnn(
initial_state,
sequence_input,
cell,
target_column.num_label_columns,
dtype=dtype,
parallel_iterations=parallel_iterations,
swap_memory=swap_memory)
loss = None # Created below for modes TRAIN and EVAL.
if prediction_type == rnn_common.PredictionType.MULTIPLE_VALUE:
prediction_dict = rnn_common.multi_value_predictions(
rnn_activations, target_column, problem_type, predict_probabilities)
if mode != model_fn.ModeKeys.INFER:
loss = _multi_value_loss(
rnn_activations, labels, sequence_length, target_column, features)
elif prediction_type == rnn_common.PredictionType.SINGLE_VALUE:
prediction_dict = _single_value_predictions(
rnn_activations, sequence_length, target_column,
problem_type, predict_probabilities)
if mode != model_fn.ModeKeys.INFER:
loss = _single_value_loss(
rnn_activations, labels, sequence_length, target_column, features)
state_dict = state_tuple_to_dict(final_state)
prediction_dict.update(state_dict)
eval_metric_ops = None
if mode != model_fn.ModeKeys.INFER:
eval_metric_ops = rnn_common.get_eval_metric_ops(
problem_type, prediction_type, sequence_length, prediction_dict,
labels)
train_op = None
if mode == model_fn.ModeKeys.TRAIN:
train_op = optimizers.optimize_loss(
loss=loss,
global_step=None, # Get it internally.
learning_rate=learning_rate,
optimizer=optimizer,
clip_gradients=gradient_clipping_norm,
summaries=optimizers.OPTIMIZER_SUMMARIES)
output_alternatives = _get_output_alternatives(prediction_type,
problem_type,
prediction_dict)
return model_fn.ModelFnOps(mode=mode,
predictions=prediction_dict,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
output_alternatives=output_alternatives)
def _dynamic_rnn_model_fn(features, labels, mode):
"""The model to be passed to an `Estimator`."""
with ops.name_scope(name):
sequence_length = features.get(sequence_length_key)
sequence_input = build_sequence_input(features,
sequence_feature_columns,
context_feature_columns)
dropout = (dropout_keep_probabilities
if mode == model_fn.ModeKeys.TRAIN
else None)
# This class promises to use the cell type selected by that function.
cell = rnn_common.construct_rnn_cell(num_units, cell_type, dropout)
initial_state = dict_to_state_tuple(features, cell)
rnn_activations, final_state = construct_rnn(
initial_state,
sequence_input,
cell,
target_column.num_label_columns,
dtype=dtype,
parallel_iterations=parallel_iterations,
swap_memory=swap_memory)
loss = None # Created below for modes TRAIN and EVAL.
if prediction_type == rnn_common.PredictionType.MULTIPLE_VALUE:
prediction_dict = rnn_common.multi_value_predictions(
rnn_activations, target_column, problem_type, predict_probabilities)
if mode != model_fn.ModeKeys.INFER:
loss = _multi_value_loss(
rnn_activations, labels, sequence_length, target_column, features)
elif prediction_type == rnn_common.PredictionType.SINGLE_VALUE:
prediction_dict = _single_value_predictions(
rnn_activations, sequence_length, target_column,
problem_type, predict_probabilities)
if mode != model_fn.ModeKeys.INFER:
loss = _single_value_loss(
rnn_activations, labels, sequence_length, target_column, features)
state_dict = state_tuple_to_dict(final_state)
prediction_dict.update(state_dict)
eval_metric_ops = None
if mode != model_fn.ModeKeys.INFER:
eval_metric_ops = rnn_common.get_eval_metric_ops(
problem_type, prediction_type, sequence_length, prediction_dict,
labels)
train_op = None
if mode == model_fn.ModeKeys.TRAIN:
train_op = optimizers.optimize_loss(
loss=loss,
global_step=None, # Get it internally.
learning_rate=learning_rate,
optimizer=optimizer,
clip_gradients=gradient_clipping_norm,
summaries=optimizers.OPTIMIZER_SUMMARIES)
output_alternatives = _get_output_alternatives(prediction_type,
problem_type,
prediction_dict)
return model_fn.ModelFnOps(mode=mode,
predictions=prediction_dict,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
output_alternatives=output_alternatives)
def _rnn_model_fn(features, labels, mode):
"""The model to be passed to an `Estimator`."""
with ops.name_scope(name):
dropout = (dropout_keep_probabilities
if mode == model_fn.ModeKeys.TRAIN else None)
cell = rnn_common.construct_rnn_cell(num_units, cell_type, dropout)
batch = _read_batch(
cell=cell,
features=features,
labels=labels,
mode=mode,
num_unroll=num_unroll,
batch_size=batch_size,
sequence_feature_columns=sequence_feature_columns,
context_feature_columns=context_feature_columns,
num_threads=num_threads,
queue_capacity=queue_capacity,
seed=seed)
sequence_features = batch.sequences
context_features = batch.context
if mode != model_fn.ModeKeys.INFER:
labels = sequence_features.pop(rnn_common.RNNKeys.LABELS_KEY)
inputs = _prepare_inputs_for_rnn(sequence_features,
context_features,
sequence_feature_columns,
num_unroll)
state_name = _get_state_names(cell)
rnn_activations, final_state = construct_state_saving_rnn(
cell=cell,
inputs=inputs,
num_label_columns=target_column.num_label_columns,
state_saver=batch,
state_name=state_name)
loss = None # Created below for modes TRAIN and EVAL.
prediction_dict = rnn_common.multi_value_predictions(
rnn_activations, target_column, problem_type,
predict_probabilities)
if mode != model_fn.ModeKeys.INFER:
loss = _multi_value_loss(rnn_activations, labels, batch.length,
target_column, features)
eval_metric_ops = None
if mode != model_fn.ModeKeys.INFER:
eval_metric_ops = rnn_common.get_eval_metric_ops(
problem_type, rnn_common.PredictionType.MULTIPLE_VALUE,
batch.length, prediction_dict, labels)
state_dict = state_tuple_to_dict(final_state)
prediction_dict.update(state_dict)
train_op = None
if mode == model_fn.ModeKeys.TRAIN:
train_op = optimizers.optimize_loss(
loss=loss,
global_step=None, # Get it internally.
learning_rate=learning_rate,
optimizer=optimizer,
clip_gradients=gradient_clipping_norm,
summaries=optimizers.OPTIMIZER_SUMMARIES)
return model_fn.ModelFnOps(mode=mode,
predictions=prediction_dict,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops)
def _rnn_model_fn(features, labels, mode):
"""The model to be passed to an `Estimator`."""
with ops.name_scope(name):
dropout = (dropout_keep_probabilities
if mode == model_fn.ModeKeys.TRAIN
else None)
cell = rnn_common.construct_rnn_cell(num_units, cell_type, dropout)
batch = _read_batch(
cell=cell,
features=features,
labels=labels,
mode=mode,
num_unroll=num_unroll,
batch_size=batch_size,
sequence_feature_columns=sequence_feature_columns,
context_feature_columns=context_feature_columns,
num_threads=num_threads,
queue_capacity=queue_capacity,
seed=seed)
sequence_features = batch.sequences
context_features = batch.context
if mode != model_fn.ModeKeys.INFER:
labels = sequence_features.pop(rnn_common.RNNKeys.LABELS_KEY)
inputs = _prepare_inputs_for_rnn(sequence_features, context_features,
sequence_feature_columns, num_unroll)
state_name = _get_state_names(cell)
rnn_activations, final_state = construct_state_saving_rnn(
cell=cell,
inputs=inputs,
num_label_columns=target_column.num_label_columns,
state_saver=batch,
state_name=state_name)
loss = None # Created below for modes TRAIN and EVAL.
prediction_dict = rnn_common.multi_value_predictions(
rnn_activations, target_column, problem_type, predict_probabilities)
if mode != model_fn.ModeKeys.INFER:
loss = _multi_value_loss(rnn_activations, labels, batch.length,
target_column, features)
eval_metric_ops = None
if mode != model_fn.ModeKeys.INFER:
eval_metric_ops = rnn_common.get_eval_metric_ops(
problem_type, rnn_common.PredictionType.MULTIPLE_VALUE,
batch.length, prediction_dict, labels)
state_dict = state_tuple_to_dict(final_state)
prediction_dict.update(state_dict)
train_op = None
if mode == model_fn.ModeKeys.TRAIN:
train_op = optimizers.optimize_loss(
loss=loss,
global_step=None, # Get it internally.
learning_rate=learning_rate,
optimizer=optimizer,
clip_gradients=gradient_clipping_norm,
summaries=optimizers.OPTIMIZER_SUMMARIES)
return model_fn.ModelFnOps(mode=mode,
predictions=prediction_dict,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops)
def _to_dnn_input_layer(self,
transformed_input_tensor,
weight_collections=None,
trainable=True,
output_rank=2):
"""Returns a Tensor as an input to the first layer of neural network.
Args:
transformed_input_tensor: A tensor that has undergone the transformations
in `insert_transformed_feature`. Rank should be >= `output_rank`.
unused_weight_collections: Unused. One hot encodings are not variable.
unused_trainable: Unused. One hot encodings are not trainable.
output_rank: the desired rank of the output `Tensor`.
Returns:
A outputs Tensor of RNN to be fed into the first layer of neural network.
Raises:
"""
sparse_id_column = self.sparse_id_column.id_tensor(transformed_input_tensor)
# pylint: disable=protected-access
sparse_id_column = layers._inner_flatten(sparse_id_column, output_rank)
batch_size = sparse_id_column.dense_shape[0]
dense_id_tensor = sparse_ops.sparse_to_dense(sparse_id_column.indices,
[batch_size,
self.max_sequence_length],
sparse_id_column.values,
default_value=0)
# dense_id_tensor = gen_array_ops.reshape(dense_id_tensor, [-1, self.max_sequence_length])
if self.shared_embedding_name is not None:
shared_embedding_collection_name = (
"SHARED_EMBEDDING_COLLECTION_" + self.shared_embedding_name.upper())
graph = ops.get_default_graph()
shared_embedding_collection = (
graph.get_collection_ref(shared_embedding_collection_name))
shape = [self.length, self.embedding_dimension]
if shared_embedding_collection:
if len(shared_embedding_collection) > 1:
raise ValueError(
"Collection %s can only contain one "
"(partitioned) variable." % shared_embedding_collection_name)
else:
embeddings = shared_embedding_collection[0]
if embeddings.get_shape() != shape:
raise ValueError(
"The embedding variable with name {} already "
"exists, but its shape does not match required "
"embedding shape here. Please make sure to use "
"different shared_embedding_name for different "
"shared embeddings.".format(args.shared_embedding_name))
else:
embeddings = contrib_variables.model_variable(
name=self.shared_embedding_name,
shape=shape,
dtype=dtypes.float32,
initializer=self.initializer,
trainable=(trainable and self.trainable),
collections=weight_collections)
graph.add_to_collection(shared_embedding_collection_name, embeddings)
else:
embeddings = contrib_variables.model_variable(
name="weights",
shape=[self.length, self.embedding_dimension],
dtype=dtypes.float32,
initializer=self.initializer,
trainable=(trainable and self.trainable),
collections=weight_collections)
if _is_variable(embeddings):
embeddings = [embeddings]
else:
embeddings = embeddings._get_variable_list() # pylint: disable=protected-access
embedding_inputs = embedding_lookup(
embeddings,
dense_id_tensor,
max_norm=self.max_norm)
dropout = (self.dropout_keep_probabilities
if self.mode == model_fn.ModeKeys.TRAIN
else None)
sequence_length = self._sequence_length(dense_id_tensor)
if bidirectional_rnn:
cell_fw = rnn_common.construct_rnn_cell(self.num_units, self.cell_type, dropout)
cell_bw = rnn_common.construct_rnn_cell(self.num_units, self.cell_type, dropout)
_rnn_outputs, _ = rnn.bidirectional_dynamic_rnn(cell_fw,
cell_bw,
embedding_inputs,
sequence_length=sequence_length,
dtype=dtypes.float32)
rnn_outputs = array_ops.concat(_rnn_outputs, axis=2)
else:
cell = rnn_common.construct_rnn_cell(self.num_units, self.cell_type, dropout)
rnn_outputs, _ = rnn.dynamic_rnn(cell,
embedding_inputs,
sequence_length=sequence_length,
dtype=dtypes.float32)
return self._extract_last_relevent(rnn_outputs, sequence_length)
def _get_dense_tensor(self,
inputs,
weight_collections=None,
trainable=None):
#Get sparse IDs and weights.
sparse_tensors = self.categorical_column._get_sparse_tensors( #pylint: disable=protected-access
inputs,
weight_collections=weight_collections,
trainable=trainable)
sparse_ids = sparse_tensors.id_tensor
batch_size = sparse_ids.dense_shape[0]
dense_tensor_ids = sparse_ops.sparse_to_dense(
sparse_ids.indices, [batch_size, self.max_sequence_length],
sparse_ids.values,
default_value=0)
# Create embedding weight, and restore from checkpoint if necessary.
embedding_weights = variable_scope.get_variable(
name='embedding_weights',
shape=(self.categorical_column._num_buckets,
self.embedding_dimension), # pylint: disable=protected-access
dtype=dtypes.float32,
initializer=self.initializer,
trainable=self.trainable and trainable,
collections=weight_collections)
if self.ckpt_to_load_from is not None:
to_restore = embedding_weights
if isinstance(to_restore, variables.PartitionedVariable):
to_restore = to_restore._get_variable_list() # pylint: disable=protected-access
checkpoint_utils.init_from_checkpoint(
self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore})
#dense_tensor_ids = utils.tf_print(dense_tensor_ids, "dense:")
embedding_inputs = embedding_lookup(embedding_weights,
dense_tensor_ids,
max_norm=self.max_norm)
dropout = (self.dropout_keep_probabilities
if self.mode == model_fn_lib.ModeKeys.TRAIN else None)
sequence_lengths = self._sequence_lengths(sparse_ids)
if self.bidirectional_rnn:
cell_fw = rnn_common.construct_rnn_cell(self.num_units,
self.cell_type, dropout)
cell_bw = rnn_common.construct_rnn_cell(self.num_units,
self.cell_type, dropout)
with ops.name_scope('RNN'):
rnn_outputs, final_states = rnn.bidirectional_dynamic_rnn(
cell_fw,
cell_bw,
embedding_inputs,
sequence_length=sequence_lengths,
dtype=dtypes.float32)
#outputs = layers.fully_connected(
# inputs=array_ops.concat(rnn_outputs, 2),
# num_outputs=self.num_units,
# activation_fn=self.activation_fn,
# trainable=True)
return array_ops.concat(final_states, 1)
else:
cell = rnn_common.construct_rnn_cell(self.num_units,
self.cell_type, dropout)
with ops.name_scope('RNN'):
rnn_outputs, final_state = rnn.dynamic_rnn(
cell,
embedding_inputs,
sequence_length=sequence_lengths,
dtype=dtypes.float32)
#rnn_outputs = utils.tf_print(rnn_outputs, "rnn_output:")
#rnn_last_outputs = utils.tf_print(rnn_last_outputs, "rnn_last:")
#outputs = layers.fully_connected(
# inputs=rnn_outputs,
# num_outputs=self.num_units,
# activation_fn=self.activation_fn,
# trainable=True)
return final_state.h
