def testNCELoss(self):
# A simple test to verify the numerics.
def _SigmoidCrossEntropyWithLogits(logits, targets):
# logits, targets: float arrays of the same shape.
assert logits.shape == targets.shape
pred = 1. / (1. + np.exp(-logits))
eps = 0.0001
pred = np.minimum(np.maximum(pred, eps), 1 - eps)
return -targets * np.log(pred) - (1. - targets) * np.log(1. - pred)
np.random.seed(0)
num_classes = 5
batch_size = 3
labels = [0, 1, 2]
(weights, biases, hidden_acts, sampled_vals, exp_logits,
exp_labels) = self._GenerateTestData(num_classes=num_classes,
dim=10,
batch_size=batch_size,
num_true=1,
labels=labels,
sampled=[1, 0, 2, 3],
subtract_log_q=True)
exp_nce_loss = np.sum(
_SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1)
with self.test_session():
got_nce_loss = nn_impl.nce_loss(
weights=constant_op.constant(weights),
biases=constant_op.constant(biases),
labels=constant_op.constant(labels, shape=(batch_size, 1)),
inputs=constant_op.constant(hidden_acts),
num_sampled=4,
num_classes=num_classes,
num_true=1,
sampled_values=sampled_vals,
partition_strategy="div")
self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4)
# Test with sharded weights and sharded biases.
weight_shards, bias_shards = self._ShardTestEmbeddings(
weights, biases, num_shards=3)
got_nce_loss = nn_impl.nce_loss(
weights=[
constant_op.constant(shard) for shard in weight_shards
],
biases=[constant_op.constant(shard) for shard in bias_shards],
labels=constant_op.constant(labels, shape=(batch_size, 1)),
inputs=constant_op.constant(hidden_acts),
num_sampled=4,
num_classes=num_classes,
num_true=1,
sampled_values=sampled_vals,
partition_strategy="div")
self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4)
def testNCELoss(self):
# A simple test to verify the numerics.
def _SigmoidCrossEntropyWithLogits(logits, targets):
# logits, targets: float arrays of the same shape.
assert logits.shape == targets.shape
pred = 1. / (1. + np.exp(-logits))
eps = 0.0001
pred = np.minimum(np.maximum(pred, eps), 1 - eps)
return -targets * np.log(pred) - (1. - targets) * np.log(1. - pred)
np.random.seed(0)
num_classes = 5
batch_size = 3
labels = [0, 1, 2]
(weights, biases, hidden_acts, sampled_vals, exp_logits,
exp_labels) = self._GenerateTestData(
num_classes=num_classes,
dim=10,
batch_size=batch_size,
num_true=1,
labels=labels,
sampled=[1, 0, 2, 3],
subtract_log_q=True)
exp_nce_loss = np.sum(
_SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1)
with self.test_session():
got_nce_loss = nn_impl.nce_loss(
weights=constant_op.constant(weights),
biases=constant_op.constant(biases),
labels=constant_op.constant(labels, shape=(batch_size, 1)),
inputs=constant_op.constant(hidden_acts),
num_sampled=4,
num_classes=num_classes,
num_true=1,
sampled_values=sampled_vals,
partition_strategy="div")
self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4)
# Test with sharded weights and sharded biases.
weight_shards, bias_shards = self._ShardTestEmbeddings(
weights, biases, num_shards=3)
got_nce_loss = nn_impl.nce_loss(
weights=[constant_op.constant(shard) for shard in weight_shards],
biases=[constant_op.constant(shard) for shard in bias_shards],
labels=constant_op.constant(labels, shape=(batch_size, 1)),
inputs=constant_op.constant(hidden_acts),
num_sampled=4,
num_classes=num_classes,
num_true=1,
sampled_values=sampled_vals,
partition_strategy="div")
self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4)
def testNCELoss(self):
# A simple test to verify the numerics.
def _SigmoidCrossEntropyWithLogits(logits, targets):
# logits, targets: float arrays of the same shape.
assert logits.shape == targets.shape
pred = 1. / (1. + np.exp(-logits))
eps = 0.0001
pred = np.minimum(np.maximum(pred, eps), 1 - eps)
return -targets * np.log(pred) - (1. - targets) * np.log(1. - pred)
weights, biases, hidden_acts, sharded_weights = self._GenerateTestInputs(
)
labels = [0, 1, 2]
true_w, true_b = weights[labels], biases[labels]
sampled = [1, 0, 2, 3]
num_sampled = len(sampled)
true_exp = np.empty([self._batch_size, 1], dtype=np.float32)
true_exp.fill(0.5)
sampled_exp = np.empty([num_sampled], dtype=np.float32)
sampled_exp.fill(0.5)
sampled_w, sampled_b = weights[sampled], biases[sampled]
test_sampled_vals = (sampled, true_exp, sampled_exp)
with self.test_session():
logits_np, labels_np = self._ComputeSampledLogitsNP(
true_w,
true_b,
sampled_w,
sampled_b,
hidden_acts,
true_expected=true_exp,
sampled_expected=sampled_exp)
nce_loss_np = np.sum(
_SigmoidCrossEntropyWithLogits(logits_np, labels_np), 1)
labels_tf = constant_op.constant(labels,
shape=(self._batch_size, 1))
weights_tf = constant_op.constant(weights)
biases_tf = constant_op.constant(biases)
inputs_tf = constant_op.constant(hidden_acts)
nce_loss_tf = nn_impl.nce_loss(weights_tf,
biases_tf,
labels_tf,
inputs_tf,
num_sampled=1,
num_classes=self._num_classes,
num_true=1,
sampled_values=test_sampled_vals)
self.assertAllClose(nce_loss_np, nce_loss_tf.eval(), 1e-4)
# Test with sharded weights
nce_loss_tf = nn_impl.nce_loss(
[constant_op.constant(shard) for shard in sharded_weights],
biases_tf,
labels_tf,
inputs_tf,
num_sampled=1,
num_classes=self._num_classes,
num_true=1,
sampled_values=test_sampled_vals)
self.assertAllClose(nce_loss_np, nce_loss_tf.eval(), 1e-4)
def testNCELoss(self):
# A simple test to verify the numerics.
def _SigmoidCrossEntropyWithLogits(logits, targets):
# logits, targets: float arrays of the same shape.
assert logits.shape == targets.shape
pred = 1. / (1. + np.exp(-logits))
eps = 0.0001
pred = np.minimum(np.maximum(pred, eps), 1 - eps)
return -targets * np.log(pred) - (1. - targets) * np.log(1. - pred)
weights, biases, hidden_acts, sharded_weights, sharded_biases = (
self._GenerateTestInputs())
labels = [0, 1, 2]
true_w, true_b = weights[labels], biases[labels]
sampled = [1, 0, 2, 3]
num_sampled = len(sampled)
true_exp = np.empty([self._batch_size, 1], dtype=np.float32)
true_exp.fill(0.5)
sampled_exp = np.empty([num_sampled], dtype=np.float32)
sampled_exp.fill(0.5)
sampled_w, sampled_b = weights[sampled], biases[sampled]
test_sampled_vals = (sampled, true_exp, sampled_exp)
with self.test_session():
logits_np, labels_np = self._ComputeSampledLogitsNP(
true_w,
true_b,
sampled_w,
sampled_b,
hidden_acts,
true_expected=true_exp,
sampled_expected=sampled_exp)
nce_loss_np = np.sum(
_SigmoidCrossEntropyWithLogits(logits_np, labels_np), 1)
labels_tf = constant_op.constant(labels, shape=(self._batch_size, 1))
weights_tf = constant_op.constant(weights)
biases_tf = constant_op.constant(biases)
inputs_tf = constant_op.constant(hidden_acts)
nce_loss_tf = nn_impl.nce_loss(
weights_tf,
biases_tf,
labels_tf,
inputs_tf,
num_sampled=num_sampled,
num_classes=self._num_classes,
num_true=1,
sampled_values=test_sampled_vals,
partition_strategy="div")
self.assertAllClose(nce_loss_np, nce_loss_tf.eval(), 1e-4)
# Test with sharded weights and sharded biases.
nce_loss_tf = nn_impl.nce_loss(
sharded_weights,
sharded_biases,
labels_tf,
inputs_tf,
num_sampled=num_sampled,
num_classes=self._num_classes,
num_true=1,
sampled_values=test_sampled_vals,
partition_strategy="div")
self.assertAllClose(nce_loss_np, nce_loss_tf.eval(), 1e-4)
