def test_readme_example(self):
data = tf.random.uniform((128, 128), 0, 10, dtype=tf.int32)
histogram = tf.bincount(data, minlength=10, maxlength=10)
cdf = tf.cumsum(histogram, exclusive=False)
cdf = tf.pad(cdf, [[1, 0]])
cdf = tf.reshape(cdf, [1, 1, -1])
data = tf.cast(data, tf.int16)
encoded = range_coding_ops.range_encode(data, cdf, precision=14)
decoded = range_coding_ops.range_decode(encoded,
tf.shape(data),
cdf,
precision=14)
with self.cached_session() as sess:
self.assertAllEqual(*sess.run((data, decoded)))
def _example_index_to_sparse_index(example_indices, batch_size):
"""Creates a sparse index tensor from a list of example indices.
For example, this would do the transformation:
[0, 0, 0, 1, 3, 3] -> [[0,0], [0,1], [0,2], [1,0], [3,0], [3,1]]
The second column of the output tensor is the running count of the occurrences
of that example index.
Args:
example_indices: A sorted 1D Tensor with example indices.
batch_size: The batch_size. Could be larger than max(example_indices) if the
last examples of the batch do not have the feature present.
Returns:
The sparse index tensor.
The maxmium length of a row in this tensor.
"""
binned_counts = tf.bincount(example_indices, minlength=batch_size)
max_len = tf.to_int64(tf.reduce_max(binned_counts))
return tf.where(tf.sequence_mask(binned_counts)), max_len
def _compute_word_overlap(context_ids, context_len, question_ids, question_len,
reduce_type, weighted, vocab_df):
"""Compute word overlap between question and context ids.
Args:
context_ids: <int32> [batch_size, num_contexts, max_context_len]
context_len: <int32> [batch_size, num_contexts]
question_ids: <int32> [batch_size, max_question_len]
question_len: <int32> [batch_size]
reduce_type: String for reduce type when computing overlap. Choices are: max
- Allows at most one match per question word. sum - Sums over all matches
for each question word.
weighted: Boolean indicate whether or not weight the overlap by IDF.
vocab_df: Tensor of shape [vocab_size] for word frequency. Computes this at
the document-level if not given.
Returns:
overlap: <float32> [batch_size, num_contexts]
Raises:
Exception: If invalid reduce_type is provided.
"""
# <float> [batch_size, num_contexts, question_len, context_len]
overlap = tf.to_float(
_word_overlap_helper(question_ids=question_ids,
context_ids=context_ids))
# <float> [batch_size, question_len]
question_mask = tf.sequence_mask(question_len,
tf.shape(question_ids)[1],
dtype=tf.float32)
# <float> [batch_size, num_contexts, context_len]
context_mask = tf.sequence_mask(context_len,
tf.shape(context_ids)[2],
dtype=tf.float32)
overlap *= tf.expand_dims(tf.expand_dims(question_mask, 1), -1)
overlap *= tf.expand_dims(context_mask, 2)
if weighted:
if vocab_df is None:
# Use document-level IDF computed with respect to the current batch.
flat_context_ids = tf.to_int32(tf.reshape(context_ids, [-1]))
# <float> [number of unique words]
vocab_df = tf.bincount(flat_context_ids,
minlength=tf.reduce_max(question_ids) + 1,
dtype=tf.float32)
# Replace all zeros with ones.
vocab_df = tf.where(tf.equal(vocab_df, 0),
x=tf.ones_like(vocab_df),
y=vocab_df)
# <float>[batch_size, question_len] expanded to
# <float> [batch_size, 1, question_len, 1]
question_df = tf.gather(vocab_df, question_ids)
question_df = tf.expand_dims(tf.expand_dims(question_df, 1), -1)
# <float> [batch_size, num_contexts, question_len, context_len]
overlap = tf.divide(tf.to_float(overlap), question_df)
if reduce_type == "max":
# <float> [batch_size, num_contexts]
overlap = tf.reduce_sum(tf.reduce_max(overlap, axis=[3]), axis=[2])
elif reduce_type == "sum":
# <float> [batch_size, num_contexts]
overlap = tf.reduce_sum(overlap, axis=[2, 3])
else:
raise Exception("Reduce type %s is invalid." % reduce_type)
return overlap
def expected_calibration_error(y_true, y_pred, nbins=20):
"""Calculates Expected Calibration Error (ECE).
ECE is a scalar summary statistic of calibration error. It is the
sample-weighted average of the difference between the predicted and true
probabilities of a positive detection across uniformly-spaced model
confidences [0, 1]. See referenced paper for a thorough explanation.
Reference:
Guo, et. al, "On Calibration of Modern Neural Networks"
Page 2, Expected Calibration Error (ECE).
https://arxiv.org/pdf/1706.04599.pdf
This function creates three local variables, `bin_counts`, `bin_true_sum`, and
`bin_preds_sum` that are used to compute ECE. For estimation of the metric
over a stream of data, the function creates an `update_op` operation that
updates these variables and returns the ECE.
Args:
y_true: 1-D tf.int64 Tensor of binarized ground truth, corresponding to each
prediction in y_pred.
y_pred: 1-D tf.float32 tensor of model confidence scores in range
[0.0, 1.0].
nbins: int specifying the number of uniformly-spaced bins into which y_pred
will be bucketed.
Returns:
value_op: A value metric op that returns ece.
update_op: An operation that increments the `bin_counts`, `bin_true_sum`,
and `bin_preds_sum` variables appropriately and whose value matches `ece`.
Raises:
InvalidArgumentError: if y_pred is not in [0.0, 1.0].
"""
bin_counts = metrics_impl.metric_variable([nbins],
tf.float32,
name='bin_counts')
bin_true_sum = metrics_impl.metric_variable([nbins],
tf.float32,
name='true_sum')
bin_preds_sum = metrics_impl.metric_variable([nbins],
tf.float32,
name='preds_sum')
with tf.control_dependencies([
tf.assert_greater_equal(y_pred, 0.0),
tf.assert_less_equal(y_pred, 1.0),
]):
bin_ids = tf.histogram_fixed_width_bins(y_pred, [0.0, 1.0],
nbins=nbins)
with tf.control_dependencies([bin_ids]):
update_bin_counts_op = tf.assign_add(
bin_counts,
tf.cast(tf.bincount(bin_ids, minlength=nbins), dtype=tf.float32))
update_bin_true_sum_op = tf.assign_add(
bin_true_sum,
tf.cast(tf.bincount(bin_ids, weights=y_true, minlength=nbins),
dtype=tf.float32))
update_bin_preds_sum_op = tf.assign_add(
bin_preds_sum,
tf.cast(tf.bincount(bin_ids, weights=y_pred, minlength=nbins),
dtype=tf.float32))
ece_update_op = _ece_from_bins(update_bin_counts_op,
update_bin_true_sum_op,
update_bin_preds_sum_op,
name='update_op')
ece = _ece_from_bins(bin_counts, bin_true_sum, bin_preds_sum, name='value')
return ece, ece_update_op
def __init__(self, atom_types, **kwargs):
self.atom_types = atom_types
self.build_forces = kwargs.get('build_forces', False)
self.precision = kwargs.get('precision', _tf.float32)
self.atomic_contributions = {}
self.atom_indices = {}
self.feature_types = {'error_weights': self.precision}
self.feature_shapes = {
'error_weights': _tf.TensorShape([
None,
])
}
for t in self.atom_types:
self.feature_types['%s_indices' % t] = _tf.int32
self.feature_shapes['%s_indices' % t] = _tf.TensorShape([None, 1])
self.label_types = {'energy': self.precision}
self.label_shapes = {
'energy': _tf.TensorShape([
None,
])
}
if self.build_forces:
self.label_types['forces'] = self.precision
self.label_shapes['forces'] = _tf.TensorShape([None, None, 3])
# The individual atomic contributions and the data iterator is set up
# in this abstact method that has to be implemented for each potential
# type
self.configureAtomicContributions(**kwargs)
# Convenience handle for backwards compatibility
self.ANNs = self.atomic_contributions
self.target = self.labels['energy']
if self.build_forces:
self.target_forces = self.labels['forces']
for t in self.atom_types:
self.atom_indices[t] = self.features['%s_indices' % t]
with _tf.name_scope('Energy_Calculation'):
self.E_predict = _tf.scatter_nd(
_tf.concat([self.atom_indices[t] for t in self.atom_types], 0),
_tf.concat([
_tf.reshape(self.atomic_contributions[t].output, [-1])
for t in self.atom_types
], 0),
_tf.shape(self.target),
name='E_prediction')
with _tf.name_scope('Atom_Counting'):
self.num_atoms = _tf.reduce_sum(
[_tf.bincount(self.atom_indices[t]) for t in self.atom_types],
axis=0,
name='NumberOfAtoms')
with _tf.name_scope('MSE'):
self.error_weights = self.features['error_weights']
self.mse = _tf.reduce_mean(
(self.target - self.E_predict)**2 * self.error_weights)
self.mse_summ = _tf.summary.scalar('MSE',
self.mse,
family='performance')
with _tf.name_scope('RMSE'):
self.error_weights = self.features['error_weights']
self.rmse = _tf.sqrt(
_tf.reduce_mean(
(self.target - self.E_predict)**2 * self.error_weights))
self.rmse_summ = _tf.summary.scalar('RMSE',
self.rmse,
family='performance')
if self.build_forces:
with _tf.name_scope('Force_Calculation'):
self.gradients = {}
for t in self.atom_types:
self.gradients[t] = _tf.gradients(
self.atomic_contributions[t].output,
self.atomic_contributions[t].input,
name='%s_gradients' % t)[0]
self.F_predict = _tf.negative(
_tf.scatter_nd(
_tf.concat(
[self.atom_indices[t] for t in self.atom_types],
0),
_tf.concat([
_tf.einsum(
'ijkl,ij->ikl',
self.atomic_contributions[t].derivatives_input,
self.gradients[t],
name='%s_einsum' % t) for t in self.atom_types
], 0),
_tf.shape(self.target_forces),
name='F_prediction'))
with _tf.name_scope('MSE_Forces'):
# Following Behler. Int. J. Quant. Chem. 2015 115, 1032-1050
# equation (21). !! Not the same !! for varying number of atoms
self.mse_forces = _tf.reduce_mean(
_tf.reduce_mean((self.target_forces - self.F_predict)**2,
axis=[1, 2]) * self.error_weights)
self.mse_forces_summ = _tf.summary.scalar('MSE_Forces',
self.mse_forces,
family='performance')
with _tf.name_scope('RMSE_Forces'):
# Following Behler. Int. J. Quant. Chem. 2015 115, 1032-1050
# equation (21). !! Not the same !! for varying number of atoms
self.rmse_forces = _tf.sqrt(
_tf.reduce_mean(
_tf.reduce_mean(
(self.target_forces - self.F_predict)**2,
axis=[1, 2]) * self.error_weights))
self.rmse_forces_summ = _tf.summary.scalar(
'RMSE_Forces', self.rmse_forces, family='performance')
self.variables = _tf.get_collection(
_tf.GraphKeys.MODEL_VARIABLES,
scope=_tf.get_default_graph().get_name_scope())
self.saver = _tf.train.Saver(self.variables,
max_to_keep=None,
save_relative_paths=True)