def _sparse_intersect_indices(sp_tensor, required_sp_tensor):
"""Filters timestamps in sp_tensor to those present in required_sp_tensor."""
# We extend both sp_tensor and required_sp_tensor with each others indices
# so that they have the same indices.
# E.g. their dense representation of one batch entry could be:
# [dummy, dummy, 1 ]
dummy_value = 'n/a'
dummy_required_sp_tensor = _extend_with_dummy(
sp_tensor, required_sp_tensor, dummy_value)
dummy_sp_tensor = _extend_with_dummy(required_sp_tensor, sp_tensor,
dummy_value)
# We get rid to dummy values both for indices in the required_sp_tensor and
# the sp_tensor.
# First get rid of indices with dummy values in dummy_required_sp_tensor.
in_required = tf.sparse_retain(
dummy_sp_tensor,
tf.logical_not(tf.equal(dummy_required_sp_tensor.values, dummy_value)))
# Remove empty timesteps so that the timesteps align with the original
# required_sp_tensor.
# Then remove the indices with dummy values.
in_required = tf.sparse_retain(
_remove_empty_timesteps(in_required),
tf.logical_not(tf.equal(in_required.values, dummy_value)))
if sp_tensor.values.dtype != tf.string:
in_required = tf.SparseTensor(
indices=in_required.indices, dense_shape=in_required.dense_shape,
values=tf.strings.to_number(
in_required.values, out_type=sp_tensor.values.dtype))
return in_required
def sparse_dropout(x, keep_prob, noise_shape):
"""Dropout for sparse tensors."""
random_tensor = keep_prob
random_tensor += tf.random_uniform(noise_shape)
dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool)
pre_out = tf.sparse_retain(x, dropout_mask)
return pre_out * (1. / keep_prob)
def sparse_dropout(x, keep_prob, noise_shape):
""" Dropout for sparse tensors.
Parameters
----------
x : tf.sparse.SparseTensor
SparseTensor as input.
keep_prob : float
keep_prob
noise_shape : tuple
Returns
-------
tf.sparse.SparseTensor : sparse tensor after applying dropout.
Notes
-----
Recent tf.nn.dropout will use `rate` instead of `keep_prob`.
See Also
--------
tf.compat.v1.nn.dropout : Computes dropout. (deprecated arguments)
"""
random_tensor = keep_prob
random_tensor += tf.random_uniform(noise_shape)
dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool)
pre_out = tf.sparse_retain(x, dropout_mask)
return pre_out * (1. / keep_prob)
def dropout_sparse(x, keep_prob, num_nonzero_elems):
"""Dropout for sparse tensors. Currently fails for very large sparse tensors (>1M elements)
"""
noise_shape = [num_nonzero_elems]
random_tensor = keep_prob
random_tensor += tf.random_uniform(noise_shape)
dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool)
pre_out = tf.sparse_retain(x, dropout_mask)
return pre_out * (1. / keep_prob)
def _dropout_sparse(self, X, keep_prob, n_nonzero_elems):
"""
Dropout for sparse tensors.
"""
noise_shape = [n_nonzero_elems]
random_tensor = keep_prob
random_tensor += tf.random_uniform(noise_shape)
dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool)
pre_out = tf.sparse_retain(X, dropout_mask)
return pre_out * tf.div(1., keep_prob)
def construct_input(sequence_feature_map, categorical_values,
categorical_seq_feature, feature_value, mode, normalize,
momentum, min_value, max_value, input_keep_prob):
"""Returns a function to build the model.
Args:
sequence_feature_map: A dictionary of (Sparse)Tensors of dense shape
[batch_size, max_sequence_length, None] keyed by the feature name.
categorical_values: Potential values of the categorical_seq_feature.
categorical_seq_feature: Name of feature of observation code.
feature_value: Name of feature of observation value.
mode: The execution mode, as defined in tf.estimator.ModeKeys.
normalize: Whether to normalize each lab test.
momentum: For the batch normalization mean and variance will be updated as
momentum*old_value + (1-momentum) * new_value.
min_value: Observation values smaller than this will be capped to min_value.
max_value: Observation values larger than this will be capped to max_value.
input_keep_prob: Keep probability for input observation values.
Returns:
- diff_delta_time: Tensor of shape [batch_size, max_seq_length, 1]
with the
- obs_values: A dense representation of the observation_values with
obs_values[b, t, :] has at most one non-zero value at the
position of the corresponding lab test from obs_code_ids with
the value of the lab result. A padded Tensor of shape
[batch_size, max_sequence_length, vocab_size] of type float32
of possibly normalized observation values.
- indicator: A one-hot encoding of whether a value in obs_values comes from
observation_values or is just filled in to be 0. A Tensor of
shape [batch_size, max_sequence_length, vocab_size] and type
float32.
"""
with tf.variable_scope('input'):
sequence_feature_map = {
k: tf.sparse_reorder(s) if isinstance(s, tf.SparseTensor) else s
for k, s in sequence_feature_map.items()
}
# Filter out invalid values.
# For invalid observation values we do this through a sparse retain.
# This makes sure that the invalid values will not be considered in the
# normalization.
observation_values = sequence_feature_map[feature_value]
observation_code_sparse = sequence_feature_map[categorical_seq_feature]
# Future work: Create a flag for the missing value indicator.
valid_values = tf.abs(observation_values.values - 9999999.0) > TOLERANCE
# apply input dropout
if input_keep_prob < 1.0:
random_tensor = input_keep_prob
random_tensor += tf.random_uniform(tf.shape(observation_values.values))
# 0. if [input_keep_prob, 1.0) and 1. if [1.0, 1.0 + input_keep_prob)
dropout_mask = tf.floor(random_tensor)
if mode == tf.estimator.ModeKeys.TRAIN:
valid_values = tf.to_float(valid_values) * dropout_mask
valid_values = valid_values > 0.5
sequence_feature_map[feature_value] = tf.sparse_retain(
observation_values, valid_values)
sequence_feature_map[categorical_seq_feature] = tf.sparse_retain(
observation_code_sparse, valid_values)
# 1. Construct the sequence of observation values to feed into the RNN
# and their indicator.
# We assign each observation code an id from 0 to vocab_size-1. At each
# timestep we will lookup the id for the observation code and take the value
# of the lab test and a construct a vector with all zeros but the id-th
# position is set to the lab test value.
obs_code = sequence_feature_map[categorical_seq_feature]
obs_code_dense_ids = contrib_lookup.index_table_from_tensor(
tuple(categorical_values), num_oov_buckets=0,
name='vocab_lookup').lookup(obs_code.values)
obs_code_sparse = tf.SparseTensor(
values=obs_code_dense_ids,
indices=obs_code.indices,
dense_shape=obs_code.dense_shape)
obs_code_sparse = tf.sparse_reorder(obs_code_sparse)
observation_values = sequence_feature_map[feature_value]
observation_values = tf.sparse_reorder(observation_values)
vocab_size = len(categorical_values)
obs_values, indicator = combine_observation_code_and_values(
obs_code_sparse, observation_values, vocab_size, mode, normalize,
momentum, min_value, max_value)
# 2. We compute the diff_delta_time as additional sequence feature.
# Note, the LSTM is very sensitive to how you encode time.
delta_time = sequence_feature_map['deltaTime']
diff_delta_time = tf.concat(
[delta_time[:, :1, :], delta_time[:, :-1, :]], axis=1) - delta_time
diff_delta_time = tf.to_float(diff_delta_time) / (60.0 * 60.0)
return (diff_delta_time, obs_values, indicator)
def _process(examples):
"""Supplies input to our model.
This function supplies input to our model after parsing.
Args:
examples: The dictionary from key to (Sparse)Tensors with context
and sequence features.
Returns:
A tuple consisting of 1) a dictionary of tensors whose keys are
the feature names, and 2) a tensor of target labels if the mode
is not INFER (and None, otherwise).
"""
# Combine into a single dictionary.
feature_map = {}
# Add age if requested.
if include_age:
age_in_seconds = (
examples[CONTEXT_KEY_PREFIX + 'timestamp'] -
examples.pop(CONTEXT_KEY_PREFIX + 'Patient.birthDate'))
age_in_years = tf.to_float(age_in_seconds) / (60 * 60 * 24 * 365.0)
feature_map[CONTEXT_KEY_PREFIX + AGE_KEY] = age_in_years
sequence_length = examples.pop(CONTEXT_KEY_PREFIX + 'sequenceLength')
# Cross the requested features.
for cross in time_crossed_features:
# The features may be missing at different rates - we take the union
# of the indices supplying defaults.
extended_features = dict()
dense_shape = tf.concat(
[[tf.to_int64(tf.shape(sequence_length)[0])],
[tf.reduce_max(sequence_length)],
tf.constant([1], dtype=tf.int64)],
axis=0)
for i, feature in enumerate(cross):
sp_tensor = examples[SEQUENCE_KEY_PREFIX + feature]
additional_indices = []
covered_indices = sp_tensor.indices
for j, other_feature in enumerate(cross):
if i != j:
additional_indices.append(
tf.sets.set_difference(
tf.sparse_reorder(
tf.SparseTensor(
indices=examples[
SEQUENCE_KEY_PREFIX +
other_feature].indices,
values=tf.zeros([
tf.shape(examples[
SEQUENCE_KEY_PREFIX +
other_feature].indices)[0]
],
dtype=tf.int32),
dense_shape=dense_shape)),
tf.sparse_reorder(
tf.SparseTensor(
indices=covered_indices,
values=tf.zeros(
[tf.shape(covered_indices)[0]],
dtype=tf.int32),
dense_shape=dense_shape))).indices)
covered_indices = tf.concat([sp_tensor.indices] +
additional_indices,
axis=0)
additional_indices = tf.concat(additional_indices, axis=0)
# Supply defaults for all other indices.
default = tf.tile(tf.constant(['n/a']),
multiples=[tf.shape(additional_indices)[0]])
string_value = sp_tensor.values
if string_value.dtype != tf.string:
string_value = tf.as_string(string_value)
extended_features[feature] = tf.sparse_reorder(
tf.SparseTensor(indices=tf.concat(
[sp_tensor.indices, additional_indices], axis=0),
values=tf.concat([string_value, default],
axis=0),
dense_shape=dense_shape))
new_values = tf.strings.join(
[extended_features[f].values for f in cross], separator='-')
crossed_sp_tensor = tf.sparse_reorder(
tf.SparseTensor(
indices=extended_features[cross[0]].indices,
values=new_values,
dense_shape=extended_features[cross[0]].dense_shape))
examples[SEQUENCE_KEY_PREFIX + '_'.join(cross)] = crossed_sp_tensor
# Remove unwanted features that are used in the cross but should not be
# considered outside the cross.
for cross in time_crossed_features:
for feature in cross:
if (feature not in sequence_features
and SEQUENCE_KEY_PREFIX + feature in examples):
del examples[SEQUENCE_KEY_PREFIX + feature]
# Flatten sparse tensor to compute event age. This dense tensor also
# contains padded values. These will not be used when gathering elements
# from the dense tensor since each sparse feature won't have a value
# defined for the padding.
padded_event_age = (
# Broadcast current time along sequence dimension.
tf.expand_dims(examples.pop(CONTEXT_KEY_PREFIX + 'timestamp'), 1)
# Subtract time of events.
- examples.pop(SEQUENCE_KEY_PREFIX + 'eventId'))
for i in range(len(time_windows) - 1):
max_age = time_windows[i]
min_age = time_windows[i + 1]
padded_in_time_window = tf.logical_and(padded_event_age <= max_age,
padded_event_age > min_age)
for k, v in examples.iteritems():
if k.startswith(CONTEXT_KEY_PREFIX):
continue
# For each sparse feature entry, look up whether it is in the time
# window or not.
in_time_window = tf.gather_nd(padded_in_time_window,
v.indices[:, 0:2])
v = tf.sparse_retain(v, in_time_window)
sp_tensor = tf.sparse_reshape(v, [v.dense_shape[0], -1])
if dedup:
sp_tensor = _dedup_tensor(sp_tensor)
feature_map[k + '-til-%d' % min_age] = sp_tensor
for k, v in examples.iteritems():
if k.startswith(CONTEXT_KEY_PREFIX):
feature_map[k] = v
return feature_map
