def Mask(self, seq_ids, weights, actual_seq_len):
p = self.params
(src_ids, tgt_ids, tgt_labels,
tgt_weights) = ops.mass(seq_ids,
weights,
actual_seq_len,
mask_id=p.mask_id,
mask_ratio=p.mask_ratio,
mask_minlen=p.mask_minlen,
span_len=p.span_len,
random_start_prob=p.random_start_prob,
keep_prob=p.keep_prob,
rand_prob=p.rand_prob,
mask_prob=p.mask_prob,
mask_target=p.mask_target,
vocab_size=p.vocab_size,
first_unreserved_id=p.first_unreserved_id)
mass_out = py_utils.NestedMap()
mass_out.src = py_utils.NestedMap()
mass_out.src.ids = src_ids
mass_out.tgt = py_utils.NestedMap()
mass_out.tgt.ids = tgt_ids
mass_out.tgt.labels = tgt_labels
mass_out.tgt.weights = tgt_weights
return mass_out
def Mask(self, seq_ids, weights, actual_seq_len):
p = self.params
if p.mask_ratio == 0.:
tf.logging.info(
'ATTENTION! mask ratio is set to 0, no mask is applied')
src_ids = seq_ids
tgt_ids = tf.pad(seq_ids, [[0, 0], [1, 0]],
constant_values=1)[:, :-1]
tgt_labels = seq_ids
tgt_weights = weights
else:
(src_ids, tgt_ids, tgt_labels,
tgt_weights) = ops.mass(seq_ids,
weights,
actual_seq_len,
mask_id=p.mask_id,
mask_ratio=p.mask_ratio,
mask_minlen=p.mask_minlen,
span_len=p.span_len,
random_start_prob=p.random_start_prob,
keep_prob=p.keep_prob,
rand_prob=p.rand_prob,
mask_prob=p.mask_prob,
mask_target=p.mask_target,
vocab_size=p.vocab_size,
first_unreserved_id=p.first_unreserved_id)
mass_out = py_utils.NestedMap()
mass_out.src = py_utils.NestedMap()
mass_out.src.ids = src_ids
mass_out.tgt = py_utils.NestedMap()
mass_out.tgt.ids = tgt_ids
mass_out.tgt.labels = tgt_labels
mass_out.tgt.weights = tgt_weights
return mass_out
def testZeroLengthSeq(self):
ids = np.array([[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
dtype=np.int32)
weights = np.array([[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
dtype=np.float32)
actual_seq_len = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=np.int32)
g = tf.Graph()
with g.as_default():
(src_ids, tgt_ids, tgt_labels, tgt_weights) = ops.mass(
ids,
weights,
actual_seq_len,
mask_id=3,
mask_ratio=0.5,
mask_minlen=0,
span_len=8,
random_start_prob=0,
keep_prob=0,
rand_prob=0,
mask_prob=1,
mask_target=True,
vocab_size=9)
with self.session(graph=g) as sess:
(src_ids, tgt_ids, tgt_labels,
tgt_weights) = sess.run([src_ids, tgt_ids, tgt_labels, tgt_weights])
self.assertAllEqual(
src_ids,
np.array([[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
dtype=np.int32))
self.assertAllEqual(
tgt_ids,
np.array([[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
dtype=np.int32))
self.assertAllEqual(
tgt_labels,
np.array([[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
dtype=np.int32))
self.assertAllEqual(
tgt_weights,
np.array([[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]],
dtype=np.float32))
def testSpanLen1(self):
ids = np.array(
[[4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0, 0, 0],
[4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0, 0, 0],
[4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.int32)
weights = np.array(
[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.float32)
actual_seq_len = np.array([14, 10, 14, 10], dtype=np.int32)
g = tf.Graph()
with g.as_default():
(src_ids, tgt_ids, tgt_labels, tgt_weights) = ops.mass(
ids,
weights,
actual_seq_len,
mask_id=3,
mask_ratio=0.5,
mask_minlen=1,
span_len=1,
keep_prob=0,
rand_prob=0,
mask_prob=1,
mask_target=True,
vocab_size=9)
with self.session(graph=g) as sess:
(src_ids, tgt_ids, tgt_labels, tgt_weights) = sess.run([
src_ids,
tgt_ids,
tgt_labels,
tgt_weights,
])
result = MassOutput(src_ids, tgt_ids, tgt_labels, tgt_weights)
expected_output1 = MassOutput(
np.array([
[4, 3, 3, 7, 8, 9, 3, 3, 3, 13, 14, 3, 16, 3, 0, 0, 0, 0, 0, 0],
[3, 3, 6, 3, 3, 9, 10, 11, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[4, 5, 3, 3, 8, 9, 10, 3, 3, 13, 3, 15, 3, 3, 0, 0, 0, 0, 0, 0],
[3, 5, 6, 3, 8, 3, 10, 3, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
],
dtype=np.int32),
np.array([[
3, 4, 5, 3, 3, 3, 9, 10, 11, 3, 3, 14, 3, 16, 0, 0, 0, 0, 0, 0
], [
1, 4, 3, 6, 7, 3, 3, 3, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
], [
3, 3, 5, 6, 3, 3, 3, 10, 11, 3, 13, 3, 15, 16, 0, 0, 0, 0, 0, 0
], [BOS, 3, 3, 6, 3, 8, 3, 10, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
],
dtype=np.int32),
np.array([[
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0,
0, 0
], [
4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
],
[
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0,
0, 0, 0, 0, 0
],
[
4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0
]],
dtype=np.int32),
np.array(
[[0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.float32))
expected_output2 = MassOutput(
np.array([[
4, 5, 6, 3, 3, 3, 10, 3, 12, 13, 3, 3, 16, 3, 0, 0, 0, 0, 0, 0
], [
3, 5, 3, 7, 3, 9, 3, 3, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
], [
3, 5, 3, 3, 8, 9, 3, 3, 12, 3, 3, 15, 16, 2, 0, 0, 0, 0, 0, 0
], [3, 3, 3, 7, 8, 9, 3, 11, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.int32),
np.array([[
3, 3, 3, 6, 7, 8, 3, 10, 3, 3, 13, 14, 3, 16, 0, 0, 0, 0, 0, 0
], [1, 3, 5, 3, 7, 3, 9, 10, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
], [
1, 3, 5, 6, 3, 3, 9, 10, 3, 12, 13, 3, 3, 3, 0, 0, 0, 0, 0, 0
], [1, 4, 5, 3, 3, 3, 9, 3, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
],
dtype=np.int32),
np.array([[
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 2, 0, 0, 0, 0, 0,
0
], [4, 5, 6, 7, 8, 9, 10, 11, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 2, 0, 0,
0, 0, 0, 0
],
[
4, 5, 6, 7, 8, 9, 10, 11, 12, 2, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0
]],
dtype=np.int32),
np.array(
[[0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0],
[1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.float32))
match_idx = FindResultFromList(result,
[expected_output1, expected_output2])
self.assertIsNotNone(match_idx,
'{} is not a valid result'.format(result))
def testFixedStart(self):
ids = np.array(
[[4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0, 0, 0],
[4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0, 0, 0],
[4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.int32)
weights = np.array(
[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.float32)
actual_seq_len = np.array([14, 10, 14, 10], dtype=np.int32)
g = tf.Graph()
with g.as_default():
(src_ids, tgt_ids, tgt_labels, tgt_weights) = ops.mass(
ids,
weights,
actual_seq_len,
mask_id=3,
mask_ratio=0.5,
mask_minlen=1,
span_len=8,
random_start_prob=0,
keep_prob=0,
rand_prob=0,
mask_prob=1,
mask_target=True,
vocab_size=9)
with self.session(graph=g) as sess:
(src_ids, tgt_ids, tgt_labels,
tgt_weights) = sess.run([src_ids, tgt_ids, tgt_labels, tgt_weights])
self.assertAllEqual(
src_ids,
np.array([[
3, 3, 3, 3, 3, 3, 3, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0, 0,
0
], [
3, 3, 3, 3, 3, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
], [4, 5, 6, 7, 8, 9, 10, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0
], [4, 5, 6, 7, 8, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.int32))
self.assertAllEqual(
tgt_ids,
np.array([
[BOS, 4, 5, 6, 7, 8, 9, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0],
[BOS, 4, 5, 6, 7, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[
3, 3, 3, 3, 3, 3, 3, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0,
0, 0
],
[3, 3, 3, 3, 3, 8, 9, 10, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
],
dtype=np.int32))
self.assertAllEqual(
tgt_labels,
np.array([[
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0, 0, 0, 0,
0, 0
], [
4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
],
[
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, EOS, 0,
0, 0, 0, 0, 0
],
[
4, 5, 6, 7, 8, 9, 10, 11, 12, EOS, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0
]],
dtype=np.int32))
self.assertAllEqual(
tgt_weights,
np.array(
[[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.float32))