def _register_alignment(self, features, labels, alignment):
labels["alignment"] = text.alignment_matrix_from_pharaoh(
alignment,
self.features_inputter.get_length(features, ignore_special_tokens=True),
self.labels_inputter.get_length(labels, ignore_special_tokens=True),
)
return features, labels
def make_features(self, element=None, features=None, training=None):
if training and self.alignment_file is not None:
element, alignment = element
else:
alignment = None
features, labels = super(SequenceToSequenceInputter, self).make_features(
element=element, features=features, training=training)
if alignment is not None:
labels["alignment"] = text.alignment_matrix_from_pharaoh(
alignment,
self.features_inputter.get_length(features, ignore_special_tokens=True),
self.labels_inputter.get_length(labels, ignore_special_tokens=True))
return features, labels
def make_features(self, element=None, features=None, training=None):
if training and self.alignment_file is not None:
element, alignment = element
else:
alignment = None
features, labels = super(SequenceToSequenceInputter, self).make_features(
element=element, features=features, training=training)
if alignment is not None:
labels["alignment"] = text.alignment_matrix_from_pharaoh(
alignment,
self.features_inputter.get_length(features),
self.labels_inputter.get_length(labels))
_shift_target_sequence(labels)
if "noisy_ids" in labels:
_shift_target_sequence(labels, prefix="noisy_")
return features, labels
def _testPharaohAlignments(self, line, lengths, expected_matrix):
matrix = text.alignment_matrix_from_pharaoh(tf.constant(line),
lengths[0],
lengths[1],
dtype=tf.int32)
self.assertListEqual(expected_matrix, self.evaluate(matrix).tolist())