def call(self, inputs, training=False):
input_image_feature, patch_type_ids = inputs
input_image_feature = self.dropout_input(input_image_feature,
training=training)
patch_embeddings = tf.einsum("abc,cd->abd", input_image_feature,
self.image_projector)
input_shape = layers.get_shape_list(patch_embeddings)
batch_size = input_shape[0]
seq_length = input_shape[1]
width = input_shape[2]
# This vocab will be small so we always do one-hot here, since it is always
# faster for a small vocabulary.
flat_token_type_ids = tf.reshape(patch_type_ids, [-1])
one_hot_ids = tf.one_hot(flat_token_type_ids,
depth=self.patch_type_vocab_size)
type_embeddings = tf.matmul(one_hot_ids, self.patch_type_embeddings)
type_embeddings = tf.reshape(type_embeddings,
[batch_size, seq_length, width])
position_embeddings = tf.gather(self.patch_position_embeddings,
tf.range(0, seq_length))
position_embeddings = tf.expand_dims(position_embeddings, 0)
embeddings = patch_embeddings + type_embeddings + position_embeddings
embeddings = self.LayerNorm(embeddings, name="ImageEmbLayerNorm")
embeddings = self.dropout_output(embeddings, training=training)
return embeddings
def call(self, inputs, training=False):
input_ids, input_mask, segment_ids, masked_image_feature, image_mask = inputs
token_embedding_output = self.embeddings([input_ids, segment_ids],
training=training)
from_shape = layers.get_shape_list(masked_image_feature)
batch_size = from_shape[0]
image_seq_length = from_shape[1]
input_patch_type_ids = tf.ones(shape=[batch_size, image_seq_length],
dtype=tf.int32)
image_embedding_output = self.image_embeddings(
[masked_image_feature, input_patch_type_ids], training=training)
embedding_output = tf.concat(
[token_embedding_output, image_embedding_output], axis=1)
attention_mask = layers.get_attn_mask_imagebert(
input_ids, input_mask, masked_image_feature, image_mask)
encoder_outputs = self.encoder([embedding_output, attention_mask],
training=training)
pooled_output = self.pooler(encoder_outputs[0][-1][:, 0])
outputs = (encoder_outputs[0][-1], pooled_output)
return outputs
def call(self, inputs,
input_mask=None,
segment_ids=None,
history_answer_marker=None,
training=False):
if isinstance(inputs, (tuple, list)):
input_ids = inputs[0]
input_mask = inputs[1] if len(inputs) > 1 else input_mask
segment_ids = inputs[2] if len(inputs) > 2 else segment_ids
history_answer_marker = inputs[3] if len(inputs) > 3 else history_answer_marker
else:
input_ids = inputs
input_shape = layers.get_shape_list(input_ids)
batch_size = input_shape[0]
seq_length = input_shape[1]
if input_mask is None:
input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32)
if segment_ids is None:
segment_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32)
if history_answer_marker is None:
history_answer_marker = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32)
with tf.variable_scope("embeddings"):
# Perform embedding lookup on the word ids.
(embedding_output, embedding_table) = embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=self.config.hidden_size,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=False)
# Add positional embeddings and token type embeddings, then layer
# normalize and perform dropout.
embedding_output = embedding_postprocessor(
input_tensor=embedding_output,
use_token_type=True,
token_type_ids=segment_ids,
token_type_vocab_size=self.config.type_vocab_size,
token_type_embedding_name="token_type_embeddings",
use_position_embeddings=True,
position_embedding_name="position_embeddings",
initializer_range=self.config.initializer_range,
max_position_embeddings=self.config.max_position_embeddings,
use_history_answer_embedding=True,
history_answer_marker=history_answer_marker,
history_answer_embedding_vocab_size=2,
history_answer_embedding_name='history_answer_embedding',
dropout_prob=self.config.hidden_dropout_prob)
attention_mask = layers.get_attn_mask_bert(input_ids, input_mask)
encoder_outputs = self.encoder([embedding_output, attention_mask], training=training)
pooled_output = self.pooler(encoder_outputs[0][-1][:, 0])
outputs = (encoder_outputs[0][-1], pooled_output)
return outputs
def call(self, inputs, input_mask=None, segment_ids=None, training=False):
if isinstance(inputs, (tuple, list)):
input_ids = inputs[0]
input_mask = inputs[1] if len(inputs) > 1 else input_mask
segment_ids = inputs[2] if len(inputs) > 2 else segment_ids
else:
input_ids = inputs
input_shape = layers.get_shape_list(input_ids)
batch_size = input_shape[0]
seq_length = input_shape[1]
if input_mask is None:
input_mask = tf.ones(shape=[batch_size, seq_length],
dtype=tf.int32)
if segment_ids is None:
segment_ids = tf.zeros(shape=[batch_size, seq_length],
dtype=tf.int32)
embedding_output = self.embeddings([input_ids, segment_ids],
training=training)
attention_mask = layers.get_attn_mask_bert(input_ids, input_mask)
encoder_outputs = self.encoder([embedding_output, attention_mask],
training=training)
pooled_output = self.pooler(encoder_outputs[0][-1][:, 0])
outputs = (encoder_outputs[0][-1], pooled_output)
return outputs
def call(self, inputs, training=False):
input_video_feature, video_token_type_ids = inputs
clip_embeddings = tf.einsum("abc,cd->abd", input_video_feature,
self.video_embeddings)
input_shape = layers.get_shape_list(clip_embeddings)
batch_size = input_shape[0]
seq_length = input_shape[1]
width = input_shape[2]
# This vocab will be small so we always do one-hot here, since it is always
# faster for a small vocabulary.
flat_token_type_ids = tf.reshape(video_token_type_ids, [-1])
one_hot_ids = tf.one_hot(flat_token_type_ids, depth=self.clip_size)
clip_type_embeddings = tf.matmul(one_hot_ids,
self.clip_type_embeddings)
clip_type_embeddings = tf.reshape(clip_type_embeddings,
[batch_size, seq_length, width])
clip_embeddings += clip_type_embeddings
position_embeddings = tf.gather(self.clip_position_embeddings,
tf.range(0, seq_length))
position_embeddings = tf.expand_dims(position_embeddings, 0)
clip_embeddings += position_embeddings
output = self.LayerNorm(clip_embeddings, name="LayerNorm")
output = self.dropout(output, training=training)
return output
def call(self, inputs, masked_lm_positions=None, **kwargs):
"""
Args:
inputs : [input_ids, input_mask, segment_ids]
masked_lm_positions: masked_lm_positions
Returns:
sequence_output, pooled_output
Examples::
hit-roberta-base-zh
hit-roberta-large-zh
pai-roberta-base-zh
pai-roberta-large-zh
model = model_zoo.get_pretrained_model('hit-roberta-base-zh')
outputs = model([input_ids, input_mask, segment_ids], mode=mode)
"""
training = kwargs['mode'] == tf.estimator.ModeKeys.TRAIN
if kwargs.get("output_features", True) == True:
outputs = self.bert(inputs, training=training)
sequence_output = outputs[0]
pooled_output = outputs[1]
return sequence_output, pooled_output
else:
outputs = self.bert(inputs, training=training)
sequence_output = outputs[0]
pooled_output = outputs[1]
input_shape = layers.get_shape_list(sequence_output)
batch_size = input_shape[0]
seq_length = input_shape[1]
if masked_lm_positions is None:
masked_lm_positions = tf.ones(shape=[batch_size, seq_length],
dtype=tf.int64)
mlm_logits = self.mlm(sequence_output, masked_lm_positions)
nsp_logits = self.nsp(pooled_output)
return mlm_logits, nsp_logits, pooled_output
def matching_embedding_margin_loss(emb1, emb2):
margin = 0.3
batch_size = layers.get_shape_list(emb1)[0]
emb1_norm = tf.maximum(1e-12, tf.norm(emb1, axis=1))
tf.summary.scalar('emb1_norm_mean', tf.reduce_mean(emb1_norm))
emb1_rep = tf.div(tf.transpose(emb1), emb1_norm)
emb2_norm = tf.maximum(1e-12, tf.norm(emb2, axis=1))
tf.summary.scalar('emb2_norm_mean', tf.reduce_mean(emb2_norm))
emb2_rep = tf.div(tf.transpose(emb2), emb2_norm)
dis = tf.matmul(tf.transpose(emb1_rep), emb2_rep)
tf.summary.scalar('dis_mean', tf.reduce_mean(dis))
positive_distance = tf.reshape(tf.diag_part(dis), [batch_size, 1])
tf.summary.scalar('positive_distance_mean',
tf.reduce_mean(positive_distance))
term1 = tf.reduce_mean(tf.maximum(
0., -positive_distance + (dis - margin * tf.eye(batch_size)) + margin),
axis=1)
loss = tf.reduce_mean(term1)
return loss
def call(self,
input_ids,
input_mask=None,
segment_ids=None,
masked_lm_positions=None,
image_feature=None,
image_mask=None,
masked_patch_positions=None,
**kwargs):
"""
Examples::
model = model_zoo.get_pretrained_model('icbu-imagebert-small-en')
mlm_logits, nsp_logits, mpm_logits, target_raw_patch_features = \
model(input_ids,
input_mask=input_mask,
segment_ids=token_type_ids,
image_feature=image_feature,
image_mask=image_mask,
masked_lm_positions=lm_positions,
masked_patch_positions=masked_patch_positions,
output_features=False,
mode=mode)
"""
training = kwargs['mode'] == tf.estimator.ModeKeys.TRAIN
input_shape = layers.get_shape_list(input_ids)
batch_size = input_shape[0]
seq_length = input_shape[1]
if input_mask is None:
input_mask = tf.ones(shape=[batch_size, seq_length],
dtype=tf.int32)
if segment_ids is None:
segment_ids = tf.zeros(shape=[batch_size, seq_length],
dtype=tf.int32)
if image_mask is None:
image_mask = tf.ones(
shape=[batch_size, self.config.max_patch_position_embeddings],
dtype=tf.int32)
if masked_lm_positions is None:
masked_lm_positions = tf.ones(shape=[batch_size, seq_length],
dtype=tf.int64)
if masked_patch_positions is None:
masked_patch_positions = tf.ones(
shape=[batch_size, self.config.masked_image_token_num],
dtype=tf.int64)
if image_feature is None:
image_feature = tf.constant(
[[1.0] * 131072, [1.0] * 131072, [1.0] * 131072],
dtype=tf.float32)
image_feature = tf.reshape(image_feature, [
-1, self.config.max_patch_position_embeddings,
self.config.patch_feature_size
])
if kwargs['mode'] == tf.estimator.ModeKeys.PREDICT:
masked_image_feature = image_feature
else:
masked_image_feature = self.mask_patch_features(
image_feature, masked_patch_positions)
inputs = [
input_ids, input_mask, segment_ids, masked_image_feature,
image_mask
]
if kwargs.get("output_features", True) == True:
outputs = self.bert(inputs, training=training)
sequence_output = outputs[0]
pooled_output = outputs[1]
return sequence_output, pooled_output
else:
outputs = self.bert(inputs, training=training)
sequence_output = outputs[0]
text_sequence_output = sequence_output[:, :seq_length, :]
image_sequence_output = sequence_output[:, seq_length:, :]
pooled_output = outputs[1]
mlm_logits = self.mlm(text_sequence_output, masked_lm_positions)
nsp_logits = self.nsp(pooled_output)
mpm_logits = self.mpm(image_sequence_output,
masked_patch_positions)
target_raw_patch_features = layers.gather_indexes(
image_feature, masked_patch_positions)
return mlm_logits, nsp_logits, mpm_logits, target_raw_patch_features, pooled_output
def call(self,
input_ids,
input_mask=None,
segment_ids=None,
masked_lm_positions=None,
video_feature=None,
video_mask=None,
masked_clip_positions=None,
**kwargs):
training = kwargs['mode'] == tf.estimator.ModeKeys.TRAIN
input_shape = layers.get_shape_list(input_ids)
batch_size = input_shape[0]
seq_length = input_shape[1]
if input_mask is None:
input_mask = tf.ones(shape=[batch_size, seq_length],
dtype=tf.int32)
if segment_ids is None:
segment_ids = tf.zeros(shape=[batch_size, seq_length],
dtype=tf.int32)
if video_mask is None:
#video_mask = tf.constant([[0] * 10, [0] * 10, [0] * 10], dtype=tf.int32)
video_mask = tf.ones(shape=[batch_size, 10], dtype=tf.int32)
if masked_lm_positions is None:
masked_lm_positions = tf.ones(shape=[batch_size, seq_length],
dtype=tf.int64)
if masked_clip_positions is None:
masked_clip_positions = tf.ones(shape=[batch_size, 4],
dtype=tf.int64)
if video_feature is None:
video_feature = tf.constant(
[[1.0] * 15360, [1.0] * 15360, [1.0] * 15360],
dtype=tf.float32)
video_feature = tf.reshape(video_feature, [-1, 10, 1536])
masked_video_feature = self.mask_clip_features(video_feature,
masked_clip_positions)
inputs = [
input_ids, input_mask, segment_ids, masked_video_feature,
video_mask
]
if kwargs.get("output_features", True) == True:
outputs = self.bert(inputs, training=training)
sequence_output = outputs[0]
pooled_output = outputs[1]
return sequence_output, pooled_output
else:
outputs = self.bert(inputs, training=training)
sequence_output = outputs[0]
pooled_output = outputs[1]
mlm_logits = self.mlm(sequence_output, masked_lm_positions)
nsp_logits = self.nsp(pooled_output)
mvc_logits = self.mvc(sequence_output, masked_clip_positions,
seq_length)
target_raw_clip_features = layers.gather_indexes(
video_feature, masked_clip_positions)
return mlm_logits, nsp_logits, mvc_logits, target_raw_clip_features, pooled_output