def bert_encoding(net, params, mode, type_id): config = modeling.BertConfig(vocab_size=params.vocab_size, hidden_size=params.hidden_size, num_hidden_layers=8, num_attention_heads=8, intermediate_size=3072) input_shape = modeling.get_shape_list(net, expected_rank=2) batch_size = input_shape[0] seq_length = input_shape[1] model = modeling.BertModel(config=config, is_training=(mode == TRAIN), input_ids=net, token_type_ids=tf.fill(input_shape, type_id)) tvars = tf.trainable_variables() use_tpu = False """ init_checkpoint = params.bert_checkpoint if init_checkpoint: (assignment_map, initialized_variable_names ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) if use_tpu: def tpu_scaffold(): tf.train.init_from_checkpoint(init_checkpoint, assignment_map) return tf.train.Scaffold() scaffold_fn = tpu_scaffold else: tf.train.init_from_checkpoint(init_checkpoint, assignment_map) """ return model.get_sequence_output()
def build_model(self): # Placeholders for input, output BERT_VOCAB = '../chinese_L-12_H-768_A-12/vocab.txt' BERT_INIT_CHKPNT = '../chinese_L-12_H-768_A-12/bert_model.ckpt' BERT_CONFIG = '../chinese_L-12_H-768_A-12/bert_config.json' tokenization.validate_case_matches_checkpoint(True, '') bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG) tokenizer = tokenization.FullTokenizer(vocab_file=BERT_VOCAB, do_lower_case=True) bert_config = modeling.BertConfig( vocab_size=self.vocab_size, hidden_size=self.size_layer, num_hidden_layers=self.num_layers, num_attention_heads=self.size_layer // 4, intermediate_size=self.size_layer * 2, ) self.input_ids = tf.placeholder(tf.int32, [None, self.seq_len]) self.input_mask = tf.placeholder(tf.int32, [None, self.seq_len]) self.segment_ids = tf.placeholder(tf.int32, [None, self.seq_len]) self.label_ids = tf.placeholder(tf.int32, [None]) self.is_training = tf.placeholder(tf.bool) use_one_hot_embeddings = False self.loss, self.logits, probabilities, model, self.accuracy = create_model( bert_config, self.is_training, self.input_ids, self.input_mask, self.segment_ids, self.label_ids, self.num_classes, use_one_hot_embeddings, ) global_step = tf.Variable(0, trainable=False, name='Global_Step') self.optimizer = tf.contrib.layers.optimize_loss( self.loss, global_step=global_step, learning_rate=self.learning_rate, optimizer='Adam', clip_gradients=3.0, ) tf.summary.scalar("loss", self.loss) self.summary_op = tf.summary.merge_all() self.saver = tf.train.Saver(tf.global_variables())
def create_model(self): input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length], self.vocab_size) input_mask = None if self.use_input_mask: input_mask = BertModelTest.ids_tensor( [self.batch_size, self.seq_length], vocab_size=2) token_type_ids = None if self.use_token_type_ids: token_type_ids = BertModelTest.ids_tensor( [self.batch_size, self.seq_length], self.type_vocab_size) config = modeling.BertConfig( vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, intermediate_size=self.intermediate_size, hidden_act=self.hidden_act, hidden_dropout_prob=self.hidden_dropout_prob, attention_probs_dropout_prob=self.attention_probs_dropout_prob, max_position_embeddings=self.max_position_embeddings, type_vocab_size=self.type_vocab_size, initializer_range=self.initializer_range) model = modeling.BertModel( config=config, is_training=self.is_training, input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids, scope=self.scope) outputs = { "embedding_output": model.get_embedding_output(), "sequence_output": model.get_sequence_output(), "pooled_output": model.get_pooled_output(), "all_encoder_layers": model.get_all_encoder_layers(), } return outputs
def build_model(input_ids, is_training, bert_in_use, max_length=200, input_mask=None, token_type_ids=None, batch_size=32): """ { "attention_probs_dropout_prob": 0.1, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "max_position_embeddings": 512, "num_attention_heads": 12, "num_hidden_layers": 12, "type_vocab_size": 2, "vocab_size": 30522 } """ config = modeling.BertConfig(attention_probs_dropout_prob=0.1, hidden_act="gelu", hidden_dropout_prob=0.1, hidden_size=768, initializer_range=0.02, intermediate_size=3072, max_position_embeddings=512, num_attention_heads=12, num_hidden_layers=bert_in_use, type_vocab_size=2, vocab_size=30522) model = modeling.BertModel(config=config, is_training=is_training, input_ids=input_ids, input_mask=input_mask) #output_layer = model.get_sequence_output() return model
def test_config_to_json_string(self): config = modeling.BertConfig(vocab_size=99, hidden_size=37) obj = json.loads(config.to_json_string()) self.assertEqual(obj["vocab_size"], 99) self.assertEqual(obj["hidden_size"], 37)