def bert_encoding(net, params, mode, type_id):
config = modeling.BertConfig(vocab_size=params.vocab_size, hidden_size=params.hidden_size,
num_hidden_layers=8, num_attention_heads=8, intermediate_size=3072)
input_shape = modeling.get_shape_list(net, expected_rank=2)
batch_size = input_shape[0]
seq_length = input_shape[1]
model = modeling.BertModel(config=config, is_training=(mode == TRAIN), input_ids=net,
token_type_ids=tf.fill(input_shape, type_id))
tvars = tf.trainable_variables()
use_tpu = False
"""
init_checkpoint = params.bert_checkpoint
if init_checkpoint:
(assignment_map, initialized_variable_names
) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)
if use_tpu:
def tpu_scaffold():
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
return tf.train.Scaffold()
scaffold_fn = tpu_scaffold
else:
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
"""
return model.get_sequence_output()
def build_model(self):
# Placeholders for input, output
BERT_VOCAB = '../chinese_L-12_H-768_A-12/vocab.txt'
BERT_INIT_CHKPNT = '../chinese_L-12_H-768_A-12/bert_model.ckpt'
BERT_CONFIG = '../chinese_L-12_H-768_A-12/bert_config.json'
tokenization.validate_case_matches_checkpoint(True, '')
bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)
tokenizer = tokenization.FullTokenizer(vocab_file=BERT_VOCAB,
do_lower_case=True)
bert_config = modeling.BertConfig(
vocab_size=self.vocab_size,
hidden_size=self.size_layer,
num_hidden_layers=self.num_layers,
num_attention_heads=self.size_layer // 4,
intermediate_size=self.size_layer * 2,
)
self.input_ids = tf.placeholder(tf.int32, [None, self.seq_len])
self.input_mask = tf.placeholder(tf.int32, [None, self.seq_len])
self.segment_ids = tf.placeholder(tf.int32, [None, self.seq_len])
self.label_ids = tf.placeholder(tf.int32, [None])
self.is_training = tf.placeholder(tf.bool)
use_one_hot_embeddings = False
self.loss, self.logits, probabilities, model, self.accuracy = create_model(
bert_config,
self.is_training,
self.input_ids,
self.input_mask,
self.segment_ids,
self.label_ids,
self.num_classes,
use_one_hot_embeddings,
)
global_step = tf.Variable(0, trainable=False, name='Global_Step')
self.optimizer = tf.contrib.layers.optimize_loss(
self.loss,
global_step=global_step,
learning_rate=self.learning_rate,
optimizer='Adam',
clip_gradients=3.0,
)
tf.summary.scalar("loss", self.loss)
self.summary_op = tf.summary.merge_all()
self.saver = tf.train.Saver(tf.global_variables())
def create_model(self):
input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length],
self.vocab_size)
input_mask = None
if self.use_input_mask:
input_mask = BertModelTest.ids_tensor(
[self.batch_size, self.seq_length], vocab_size=2)
token_type_ids = None
if self.use_token_type_ids:
token_type_ids = BertModelTest.ids_tensor(
[self.batch_size, self.seq_length], self.type_vocab_size)
config = modeling.BertConfig(
vocab_size=self.vocab_size,
hidden_size=self.hidden_size,
num_hidden_layers=self.num_hidden_layers,
num_attention_heads=self.num_attention_heads,
intermediate_size=self.intermediate_size,
hidden_act=self.hidden_act,
hidden_dropout_prob=self.hidden_dropout_prob,
attention_probs_dropout_prob=self.attention_probs_dropout_prob,
max_position_embeddings=self.max_position_embeddings,
type_vocab_size=self.type_vocab_size,
initializer_range=self.initializer_range)
model = modeling.BertModel(
config=config,
is_training=self.is_training,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=token_type_ids,
scope=self.scope)
outputs = {
"embedding_output": model.get_embedding_output(),
"sequence_output": model.get_sequence_output(),
"pooled_output": model.get_pooled_output(),
"all_encoder_layers": model.get_all_encoder_layers(),
}
return outputs
def build_model(input_ids,
is_training,
bert_in_use,
max_length=200,
input_mask=None,
token_type_ids=None,
batch_size=32):
"""
{
"attention_probs_dropout_prob": 0.1,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"hidden_size": 768,
"initializer_range": 0.02,
"intermediate_size": 3072,
"max_position_embeddings": 512,
"num_attention_heads": 12,
"num_hidden_layers": 12,
"type_vocab_size": 2,
"vocab_size": 30522
}
"""
config = modeling.BertConfig(attention_probs_dropout_prob=0.1,
hidden_act="gelu",
hidden_dropout_prob=0.1,
hidden_size=768,
initializer_range=0.02,
intermediate_size=3072,
max_position_embeddings=512,
num_attention_heads=12,
num_hidden_layers=bert_in_use,
type_vocab_size=2,
vocab_size=30522)
model = modeling.BertModel(config=config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask)
#output_layer = model.get_sequence_output()
return model
def test_config_to_json_string(self):
config = modeling.BertConfig(vocab_size=99, hidden_size=37)
obj = json.loads(config.to_json_string())
self.assertEqual(obj["vocab_size"], 99)
self.assertEqual(obj["hidden_size"], 37)
