def __init__(self, model_proto, is_training):
super(FinetuneCC, self).__init__(model_proto, is_training)
if not isinstance(model_proto, model_pb2.FinetuneCC):
raise ValueError('Options has to be an FinetuneCC proto.')
options = model_proto
self._token_to_id_func = token_to_id.TokenToIdLayer(
options.bert_vocab_file, options.bert_unk_token_id)
self._bert_config = BertConfig.from_json_file(options.bert_config_file)
self._slim_fc_scope = hyperparams.build_hyperparams(
options.fc_hyperparams, is_training)()
if options.rationale_model:
self._field_label = InputFields.rationale_label
self._field_choices = InputFields.rationale_choices_with_question
self._field_choices_tag = InputFields.rationale_choices_with_question_tag
self._field_choices_len = InputFields.rationale_choices_with_question_len
else:
self._field_label = InputFields.answer_label
self._field_choices = InputFields.answer_choices_with_question
self._field_choices_tag = InputFields.answer_choices_with_question_tag
self._field_choices_len = InputFields.answer_choices_with_question_len
def __init__(self, model_proto, is_training):
super(VBertFtFrcnnAdvM2, self).__init__(model_proto, is_training)
if not isinstance(model_proto, model_pb2.VBertFtFrcnnAdvM2):
raise ValueError('Options has to be an VBertFtFrcnnAdvM2 proto.')
options = model_proto
self._bert_config = bert_modeling.BertConfig.from_json_file(
options.bert_config_file)
self._bert2_config = bert_modeling.BertConfig.from_json_file(
options.bert2_config_file)
self._slim_fc_scope = hyperparams.build_hyperparams(
options.fc_hyperparams, is_training)()
if options.rationale_model:
self._field_label = InputFields.rationale_label
self._field_choices = InputFields.mixed_rationale_choices
self._field_choices_tag = InputFields.mixed_rationale_choices_tag
self._field_choices_len = InputFields.mixed_rationale_choices_len
else:
self._field_label = InputFields.answer_label
self._field_choices = InputFields.mixed_answer_choices
self._field_choices_tag = InputFields.mixed_answer_choices_tag
self._field_choices_len = InputFields.mixed_answer_choices_len
def predict(self, inputs, **kwargs):
"""Predicts the resulting tensors.
Args:
inputs: A dictionary of input tensors keyed by names.
Returns:
predictions: A dictionary of prediction tensors keyed by name.
"""
is_training = self._is_training
options = self._model_proto
token_to_id_layer = token_to_id.TokenToIdLayer(
options.bert_vocab_file, options.bert_unk_token_id)
bert_config = BertConfig.from_json_file(options.bert_config_file)
slim_fc_scope = hyperparams.build_hyperparams(options.fc_hyperparams,
is_training)()
# Prediction.
answer_logits = self._predict_logits(
inputs[self._field_answer_choices],
inputs[self._field_answer_choices_len], token_to_id_layer,
bert_config, slim_fc_scope, options.dropout_keep_prob, is_training)
# Restore from checkpoint.
assignment_map, _ = get_assignment_map_from_checkpoint(
tf.global_variables(), options.bert_checkpoint_file)
tf.compat.v1.train.init_from_checkpoint(options.bert_checkpoint_file,
assignment_map)
return {
FIELD_ANSWER_PREDICTION: answer_logits,
}
def predict(self, inputs, **kwargs):
"""Predicts the resulting tensors.
Args:
inputs: A dictionary of input tensors keyed by names.
Returns:
predictions: A dictionary of prediction tensors keyed by name.
"""
is_training = self._is_training
options = self._model_proto
fc_scope_fn = hyperparams.build_hyperparams(options.fc_hyperparams,
is_training)
with slim.arg_scope(fc_scope_fn()):
return self._predict(inputs, **kwargs)
def __init__(self, model_proto, is_training):
super(VBertFtFrcnnV2, self).__init__(model_proto, is_training)
if not isinstance(model_proto, model_pb2.VBertFtFrcnnV2):
raise ValueError('Options has to be an VBertFtFrcnn proto.')
options = model_proto
self._bert_config = bert_modeling.BertConfig.from_json_file(
options.bert_config_file)
self._slim_fc_scope = hyperparams.build_hyperparams(
options.fc_hyperparams, is_training)()
if options.rationale_model:
assert False
else:
self._field_label = 'answer_label'
self._field_choice = 'answer_choice'
def predict(self, inputs, **kwargs):
"""Predicts the resulting tensors.
Args:
inputs: A dictionary of input tensors keyed by names.
Returns:
predictions: A dictionary of prediction tensors keyed by name.
"""
is_training = self._is_training
options = self._model_proto
token_to_id_layer = token_to_id.TokenToIdLayer(
options.bert_vocab_file, options.bert_unk_token_id)
bert_config = BertConfig.from_json_file(options.bert_config_file)
slim_fc_scope = hyperparams.build_hyperparams(options.fc_hyperparams,
is_training)()
# Predict object embedding vectors.
(num_objects, object_bboxes, object_labels, object_scores,
object_features, max_num_objects) = _trim_to_max_num_objects(
inputs[InputFields.num_detections],
inputs[InputFields.detection_boxes],
inputs[InputFields.detection_classes],
inputs[InputFields.detection_scores],
inputs[InputFields.detection_features],
max_num_objects=options.max_num_objects)
object_features = _predict_object_embeddings(
object_features,
bert_config.hidden_size,
slim_fc_scope,
keep_prob=options.dropout_keep_prob,
is_training=is_training)
# Gather text inputs.
(answer_choices, answer_choices_tag,
answer_choices_len) = (inputs[self._field_answer_choices],
inputs[self._field_answer_choices_tag],
inputs[self._field_answer_choices_len])
batch_size = answer_choices.shape[0]
answer_choices_tag = _assign_invalid_tags(answer_choices_tag,
max_num_objects)
# Convert tokens into token ids.
answer_choices_token_ids = token_to_id_layer(answer_choices)
answer_choices_token_ids = tf.reshape(answer_choices_token_ids,
[batch_size * NUM_CHOICES, -1])
answer_choices_mask = tf.sequence_mask(
answer_choices_len, maxlen=tf.shape(answer_choices)[-1])
answer_choices_mask = tf.reshape(answer_choices_mask,
[batch_size * NUM_CHOICES, -1])
# Create tag features sequence.
answer_choices_tag = tf.reshape(answer_choices_tag,
[batch_size * NUM_CHOICES, -1])
answer_choices_tag_embeddings = _ground_tag_using_object_features(
object_features, answer_choices_tag)
(tiled_object_masks, tiled_object_ids,
tiled_object_features) = _tile_objects(
num_objects, token_to_id_layer(object_labels), object_features)
# Create Bert model.
input_ids = tf.concat([answer_choices_token_ids, tiled_object_ids], -1)
input_tag_embeddings = tf.concat(
[answer_choices_tag_embeddings, tiled_object_features], 1)
input_mask = tf.concat([answer_choices_mask, tiled_object_masks], -1)
output = self._bert_model(
input_ids,
input_tag_embeddings,
input_mask,
bert_config,
bert_checkpoint_file=options.bert_checkpoint_file,
is_training=is_training)
# Classification layer.
with slim.arg_scope(slim_fc_scope):
output = slim.fully_connected(output,
num_outputs=1,
activation_fn=None,
scope='logits')
output = tf.reshape(output, [batch_size, NUM_CHOICES])
return {FIELD_ANSWER_PREDICTION: output}
def predict(self, inputs, **kwargs):
"""Predicts the resulting tensors.
Args:
inputs: A dictionary of input tensors keyed by names.
Returns:
predictions: A dictionary of prediction tensors keyed by name.
"""
is_training = self._is_training
options = self._model_proto
fc_scope_fn = hyperparams.build_hyperparams(options.fc_hyperparams,
is_training)
(answer_choices, answer_choices_tag, answer_choices_len,
answer_label) = (inputs[InputFields.answer_choices_with_question],
inputs[InputFields.answer_choices_with_question_tag],
inputs[InputFields.answer_choices_with_question_len],
inputs[InputFields.answer_label])
batch_size = answer_choices.shape[0]
# Trim lengths of the object arrays to `max_num_objects`.
(num_objects, object_bboxes, object_labels, object_scores,
object_features, max_num_objects) = _trim_to_max_num_objects(
inputs[InputFields.num_objects],
inputs[InputFields.object_bboxes],
inputs[InputFields.object_labels],
inputs[InputFields.object_scores],
inputs[InputFields.object_features],
max_num_objects=options.max_num_objects)
answer_choices_tag = _assign_invalid_tags(answer_choices_tag,
max_num_objects)
# Merge class label embeddings to the Fast-RCNN features.
object_features = _project_object_features(
object_features,
output_dims=options.visual_feature_dims,
dropout_keep_prob=options.dropout_keep_prob,
is_training=is_training)
object_feature_dims = object_features.shape[-1]
# Convert tokens into token ids.
token_to_id_layer = token_to_id.TokenToIdLayer(
options.bert_vocab_file, options.bert_unk_token_id)
answer_choices_token_ids = token_to_id_layer(answer_choices)
answer_choices_token_ids = tf.reshape(answer_choices_token_ids,
[batch_size * NUM_CHOICES, -1])
answer_choices_mask = tf.sequence_mask(
answer_choices_len, maxlen=tf.shape(answer_choices)[-1])
answer_choices_mask = tf.reshape(answer_choices_mask,
[batch_size * NUM_CHOICES, -1])
# Create tag features sequence.
answer_choices_tag = tf.reshape(answer_choices_tag,
[batch_size * NUM_CHOICES, -1])
answer_choices_tag_features = _ground_tag_using_object_features(
object_features, answer_choices_tag)
# Convert class labels into token ids, tile object features.
object_mask = tf.sequence_mask(num_objects,
maxlen=tf.shape(object_labels)[-1])
object_mask = tf.gather(tf.expand_dims(object_mask, 1),
[0] * NUM_CHOICES,
axis=1)
object_mask = tf.reshape(object_mask, [batch_size * NUM_CHOICES, -1])
object_label_token_ids = token_to_id_layer(object_labels)
object_label_token_ids = tf.gather(tf.expand_dims(
object_label_token_ids, 1), [0] * NUM_CHOICES,
axis=1)
object_label_token_ids = tf.reshape(object_label_token_ids,
[batch_size * NUM_CHOICES, -1])
object_features = tf.gather(tf.expand_dims(object_features, 1),
[0] * NUM_CHOICES,
axis=1)
object_features = tf.reshape(
object_features,
[batch_size * NUM_CHOICES, -1, object_feature_dims])
# Create Bert model.
input_ids = tf.concat(
[answer_choices_token_ids, object_label_token_ids], -1)
input_tag_features = tf.concat(
[answer_choices_tag_features, object_features], 1)
input_mask = tf.concat([answer_choices_mask, object_mask], -1)
final_features = self._bert_model(input_ids, input_tag_features,
input_mask)
# Classification layer.
with slim.arg_scope(fc_scope_fn()):
output = tf.contrib.layers.fully_connected(final_features,
num_outputs=1,
activation_fn=None)
output = tf.reshape(output, [batch_size, NUM_CHOICES])
return {FIELD_ANSWER_PREDICTION: output}
def predict(self, inputs, **kwargs):
"""Predicts the resulting tensors.
Args:
inputs: A dictionary of input tensors keyed by names.
Returns:
predictions: A dictionary of prediction tensors keyed by name.
"""
options = self._model_proto
is_training = self._is_training
token_to_id_layer = token_to_id.TokenToIdLayer(options.vocab_file,
options.unk_token_id)
fc_scope_fn = hyperparams.build_hyperparams(options.fc_hyperparams,
is_training)
# Extract input fields.
(question, question_len, answer_choices,
answer_choices_len) = (inputs[InputFields.question],
inputs[InputFields.question_len],
inputs[InputFields.answer_choices],
inputs[InputFields.answer_choices_len])
batch_size = answer_choices.shape[0]
# Convert question tokens into token ids.
question_token_ids = token_to_id_layer(question)
# Convert answer choice tokens into token ids.
answer_choices_token_ids = token_to_id_layer(answer_choices)
answer_choices_token_ids = tf.reshape(answer_choices_token_ids,
[batch_size * NUM_CHOICES, -1])
answer_choices_len = tf.reshape(answer_choices_len,
[batch_size * NUM_CHOICES])
# Convert word ids to embedding vectors.
glove_embedding_array = create_embedding_matrix(options.glove_file,
options.vocab_file)
embedding = tf.get_variable('word/embedding',
initializer=glove_embedding_array,
trainable=True)
question_embs = tf.nn.embedding_lookup(embedding,
question_token_ids,
max_norm=None)
answer_choices_embs = tf.nn.embedding_lookup(embedding,
answer_choices_token_ids,
max_norm=None)
# Tile the question embeddings.
question_embs = tf.gather(tf.expand_dims(question_embs, 1),
[0] * NUM_CHOICES,
axis=1)
question_embs = tf.reshape(
question_embs, [batch_size * NUM_CHOICES, -1, question_embs.shape[-1]])
question_len = tf.gather(tf.expand_dims(question_len, 1), [0] * NUM_CHOICES,
axis=1)
question_len = tf.reshape(question_len, [batch_size * NUM_CHOICES])
# Encode the sequence using BiLSTM model.
with tf.variable_scope('question_encoder'):
_, question_features = rnn.RNN(question_embs,
question_len,
options.rnn_config,
is_training=is_training)
with tf.variable_scope('answer_choice_encoder'):
_, answer_features = rnn.RNN(answer_choices_embs,
answer_choices_len,
options.rnn_config,
is_training=is_training)
final_features = tf.concat(
[answer_features, answer_features * question_features], axis=-1)
# MLP.
with slim.arg_scope(fc_scope_fn()):
with tf.variable_scope('classification'):
with tf.variable_scope('hidden'):
output = tf.contrib.layers.fully_connected(final_features,
num_outputs=1024,
activation_fn=tf.nn.relu)
output = tf.contrib.layers.dropout(
output,
keep_prob=options.dropout_keep_prob,
is_training=is_training)
with tf.variable_scope('output'):
output = tf.contrib.layers.fully_connected(output,
num_outputs=1,
activation_fn=None)
output = tf.reshape(output, [batch_size, NUM_CHOICES])
return {
FIELD_ANSWER_PREDICTION: output,
}
