def __init__(self, model_proto, is_training): super(FinetuneCC, self).__init__(model_proto, is_training) if not isinstance(model_proto, model_pb2.FinetuneCC): raise ValueError('Options has to be an FinetuneCC proto.') options = model_proto self._token_to_id_func = token_to_id.TokenToIdLayer( options.bert_vocab_file, options.bert_unk_token_id) self._bert_config = BertConfig.from_json_file(options.bert_config_file) self._slim_fc_scope = hyperparams.build_hyperparams( options.fc_hyperparams, is_training)() if options.rationale_model: self._field_label = InputFields.rationale_label self._field_choices = InputFields.rationale_choices_with_question self._field_choices_tag = InputFields.rationale_choices_with_question_tag self._field_choices_len = InputFields.rationale_choices_with_question_len else: self._field_label = InputFields.answer_label self._field_choices = InputFields.answer_choices_with_question self._field_choices_tag = InputFields.answer_choices_with_question_tag self._field_choices_len = InputFields.answer_choices_with_question_len
def __init__(self, model_proto, is_training): super(VBertFtFrcnnAdvM2, self).__init__(model_proto, is_training) if not isinstance(model_proto, model_pb2.VBertFtFrcnnAdvM2): raise ValueError('Options has to be an VBertFtFrcnnAdvM2 proto.') options = model_proto self._bert_config = bert_modeling.BertConfig.from_json_file( options.bert_config_file) self._bert2_config = bert_modeling.BertConfig.from_json_file( options.bert2_config_file) self._slim_fc_scope = hyperparams.build_hyperparams( options.fc_hyperparams, is_training)() if options.rationale_model: self._field_label = InputFields.rationale_label self._field_choices = InputFields.mixed_rationale_choices self._field_choices_tag = InputFields.mixed_rationale_choices_tag self._field_choices_len = InputFields.mixed_rationale_choices_len else: self._field_label = InputFields.answer_label self._field_choices = InputFields.mixed_answer_choices self._field_choices_tag = InputFields.mixed_answer_choices_tag self._field_choices_len = InputFields.mixed_answer_choices_len
def predict(self, inputs, **kwargs): """Predicts the resulting tensors. Args: inputs: A dictionary of input tensors keyed by names. Returns: predictions: A dictionary of prediction tensors keyed by name. """ is_training = self._is_training options = self._model_proto token_to_id_layer = token_to_id.TokenToIdLayer( options.bert_vocab_file, options.bert_unk_token_id) bert_config = BertConfig.from_json_file(options.bert_config_file) slim_fc_scope = hyperparams.build_hyperparams(options.fc_hyperparams, is_training)() # Prediction. answer_logits = self._predict_logits( inputs[self._field_answer_choices], inputs[self._field_answer_choices_len], token_to_id_layer, bert_config, slim_fc_scope, options.dropout_keep_prob, is_training) # Restore from checkpoint. assignment_map, _ = get_assignment_map_from_checkpoint( tf.global_variables(), options.bert_checkpoint_file) tf.compat.v1.train.init_from_checkpoint(options.bert_checkpoint_file, assignment_map) return { FIELD_ANSWER_PREDICTION: answer_logits, }
def predict(self, inputs, **kwargs): """Predicts the resulting tensors. Args: inputs: A dictionary of input tensors keyed by names. Returns: predictions: A dictionary of prediction tensors keyed by name. """ is_training = self._is_training options = self._model_proto fc_scope_fn = hyperparams.build_hyperparams(options.fc_hyperparams, is_training) with slim.arg_scope(fc_scope_fn()): return self._predict(inputs, **kwargs)
def __init__(self, model_proto, is_training): super(VBertFtFrcnnV2, self).__init__(model_proto, is_training) if not isinstance(model_proto, model_pb2.VBertFtFrcnnV2): raise ValueError('Options has to be an VBertFtFrcnn proto.') options = model_proto self._bert_config = bert_modeling.BertConfig.from_json_file( options.bert_config_file) self._slim_fc_scope = hyperparams.build_hyperparams( options.fc_hyperparams, is_training)() if options.rationale_model: assert False else: self._field_label = 'answer_label' self._field_choice = 'answer_choice'
def predict(self, inputs, **kwargs): """Predicts the resulting tensors. Args: inputs: A dictionary of input tensors keyed by names. Returns: predictions: A dictionary of prediction tensors keyed by name. """ is_training = self._is_training options = self._model_proto token_to_id_layer = token_to_id.TokenToIdLayer( options.bert_vocab_file, options.bert_unk_token_id) bert_config = BertConfig.from_json_file(options.bert_config_file) slim_fc_scope = hyperparams.build_hyperparams(options.fc_hyperparams, is_training)() # Predict object embedding vectors. (num_objects, object_bboxes, object_labels, object_scores, object_features, max_num_objects) = _trim_to_max_num_objects( inputs[InputFields.num_detections], inputs[InputFields.detection_boxes], inputs[InputFields.detection_classes], inputs[InputFields.detection_scores], inputs[InputFields.detection_features], max_num_objects=options.max_num_objects) object_features = _predict_object_embeddings( object_features, bert_config.hidden_size, slim_fc_scope, keep_prob=options.dropout_keep_prob, is_training=is_training) # Gather text inputs. (answer_choices, answer_choices_tag, answer_choices_len) = (inputs[self._field_answer_choices], inputs[self._field_answer_choices_tag], inputs[self._field_answer_choices_len]) batch_size = answer_choices.shape[0] answer_choices_tag = _assign_invalid_tags(answer_choices_tag, max_num_objects) # Convert tokens into token ids. answer_choices_token_ids = token_to_id_layer(answer_choices) answer_choices_token_ids = tf.reshape(answer_choices_token_ids, [batch_size * NUM_CHOICES, -1]) answer_choices_mask = tf.sequence_mask( answer_choices_len, maxlen=tf.shape(answer_choices)[-1]) answer_choices_mask = tf.reshape(answer_choices_mask, [batch_size * NUM_CHOICES, -1]) # Create tag features sequence. answer_choices_tag = tf.reshape(answer_choices_tag, [batch_size * NUM_CHOICES, -1]) answer_choices_tag_embeddings = _ground_tag_using_object_features( object_features, answer_choices_tag) (tiled_object_masks, tiled_object_ids, tiled_object_features) = _tile_objects( num_objects, token_to_id_layer(object_labels), object_features) # Create Bert model. input_ids = tf.concat([answer_choices_token_ids, tiled_object_ids], -1) input_tag_embeddings = tf.concat( [answer_choices_tag_embeddings, tiled_object_features], 1) input_mask = tf.concat([answer_choices_mask, tiled_object_masks], -1) output = self._bert_model( input_ids, input_tag_embeddings, input_mask, bert_config, bert_checkpoint_file=options.bert_checkpoint_file, is_training=is_training) # Classification layer. with slim.arg_scope(slim_fc_scope): output = slim.fully_connected(output, num_outputs=1, activation_fn=None, scope='logits') output = tf.reshape(output, [batch_size, NUM_CHOICES]) return {FIELD_ANSWER_PREDICTION: output}
def predict(self, inputs, **kwargs): """Predicts the resulting tensors. Args: inputs: A dictionary of input tensors keyed by names. Returns: predictions: A dictionary of prediction tensors keyed by name. """ is_training = self._is_training options = self._model_proto fc_scope_fn = hyperparams.build_hyperparams(options.fc_hyperparams, is_training) (answer_choices, answer_choices_tag, answer_choices_len, answer_label) = (inputs[InputFields.answer_choices_with_question], inputs[InputFields.answer_choices_with_question_tag], inputs[InputFields.answer_choices_with_question_len], inputs[InputFields.answer_label]) batch_size = answer_choices.shape[0] # Trim lengths of the object arrays to `max_num_objects`. (num_objects, object_bboxes, object_labels, object_scores, object_features, max_num_objects) = _trim_to_max_num_objects( inputs[InputFields.num_objects], inputs[InputFields.object_bboxes], inputs[InputFields.object_labels], inputs[InputFields.object_scores], inputs[InputFields.object_features], max_num_objects=options.max_num_objects) answer_choices_tag = _assign_invalid_tags(answer_choices_tag, max_num_objects) # Merge class label embeddings to the Fast-RCNN features. object_features = _project_object_features( object_features, output_dims=options.visual_feature_dims, dropout_keep_prob=options.dropout_keep_prob, is_training=is_training) object_feature_dims = object_features.shape[-1] # Convert tokens into token ids. token_to_id_layer = token_to_id.TokenToIdLayer( options.bert_vocab_file, options.bert_unk_token_id) answer_choices_token_ids = token_to_id_layer(answer_choices) answer_choices_token_ids = tf.reshape(answer_choices_token_ids, [batch_size * NUM_CHOICES, -1]) answer_choices_mask = tf.sequence_mask( answer_choices_len, maxlen=tf.shape(answer_choices)[-1]) answer_choices_mask = tf.reshape(answer_choices_mask, [batch_size * NUM_CHOICES, -1]) # Create tag features sequence. answer_choices_tag = tf.reshape(answer_choices_tag, [batch_size * NUM_CHOICES, -1]) answer_choices_tag_features = _ground_tag_using_object_features( object_features, answer_choices_tag) # Convert class labels into token ids, tile object features. object_mask = tf.sequence_mask(num_objects, maxlen=tf.shape(object_labels)[-1]) object_mask = tf.gather(tf.expand_dims(object_mask, 1), [0] * NUM_CHOICES, axis=1) object_mask = tf.reshape(object_mask, [batch_size * NUM_CHOICES, -1]) object_label_token_ids = token_to_id_layer(object_labels) object_label_token_ids = tf.gather(tf.expand_dims( object_label_token_ids, 1), [0] * NUM_CHOICES, axis=1) object_label_token_ids = tf.reshape(object_label_token_ids, [batch_size * NUM_CHOICES, -1]) object_features = tf.gather(tf.expand_dims(object_features, 1), [0] * NUM_CHOICES, axis=1) object_features = tf.reshape( object_features, [batch_size * NUM_CHOICES, -1, object_feature_dims]) # Create Bert model. input_ids = tf.concat( [answer_choices_token_ids, object_label_token_ids], -1) input_tag_features = tf.concat( [answer_choices_tag_features, object_features], 1) input_mask = tf.concat([answer_choices_mask, object_mask], -1) final_features = self._bert_model(input_ids, input_tag_features, input_mask) # Classification layer. with slim.arg_scope(fc_scope_fn()): output = tf.contrib.layers.fully_connected(final_features, num_outputs=1, activation_fn=None) output = tf.reshape(output, [batch_size, NUM_CHOICES]) return {FIELD_ANSWER_PREDICTION: output}
def predict(self, inputs, **kwargs): """Predicts the resulting tensors. Args: inputs: A dictionary of input tensors keyed by names. Returns: predictions: A dictionary of prediction tensors keyed by name. """ options = self._model_proto is_training = self._is_training token_to_id_layer = token_to_id.TokenToIdLayer(options.vocab_file, options.unk_token_id) fc_scope_fn = hyperparams.build_hyperparams(options.fc_hyperparams, is_training) # Extract input fields. (question, question_len, answer_choices, answer_choices_len) = (inputs[InputFields.question], inputs[InputFields.question_len], inputs[InputFields.answer_choices], inputs[InputFields.answer_choices_len]) batch_size = answer_choices.shape[0] # Convert question tokens into token ids. question_token_ids = token_to_id_layer(question) # Convert answer choice tokens into token ids. answer_choices_token_ids = token_to_id_layer(answer_choices) answer_choices_token_ids = tf.reshape(answer_choices_token_ids, [batch_size * NUM_CHOICES, -1]) answer_choices_len = tf.reshape(answer_choices_len, [batch_size * NUM_CHOICES]) # Convert word ids to embedding vectors. glove_embedding_array = create_embedding_matrix(options.glove_file, options.vocab_file) embedding = tf.get_variable('word/embedding', initializer=glove_embedding_array, trainable=True) question_embs = tf.nn.embedding_lookup(embedding, question_token_ids, max_norm=None) answer_choices_embs = tf.nn.embedding_lookup(embedding, answer_choices_token_ids, max_norm=None) # Tile the question embeddings. question_embs = tf.gather(tf.expand_dims(question_embs, 1), [0] * NUM_CHOICES, axis=1) question_embs = tf.reshape( question_embs, [batch_size * NUM_CHOICES, -1, question_embs.shape[-1]]) question_len = tf.gather(tf.expand_dims(question_len, 1), [0] * NUM_CHOICES, axis=1) question_len = tf.reshape(question_len, [batch_size * NUM_CHOICES]) # Encode the sequence using BiLSTM model. with tf.variable_scope('question_encoder'): _, question_features = rnn.RNN(question_embs, question_len, options.rnn_config, is_training=is_training) with tf.variable_scope('answer_choice_encoder'): _, answer_features = rnn.RNN(answer_choices_embs, answer_choices_len, options.rnn_config, is_training=is_training) final_features = tf.concat( [answer_features, answer_features * question_features], axis=-1) # MLP. with slim.arg_scope(fc_scope_fn()): with tf.variable_scope('classification'): with tf.variable_scope('hidden'): output = tf.contrib.layers.fully_connected(final_features, num_outputs=1024, activation_fn=tf.nn.relu) output = tf.contrib.layers.dropout( output, keep_prob=options.dropout_keep_prob, is_training=is_training) with tf.variable_scope('output'): output = tf.contrib.layers.fully_connected(output, num_outputs=1, activation_fn=None) output = tf.reshape(output, [batch_size, NUM_CHOICES]) return { FIELD_ANSWER_PREDICTION: output, }