def build_embedder(self, input_ids, token_type_ids, hidden_dropout_prob,
attention_probs_dropout_prob, **kargs):
reuse = kargs["reuse"]
with tf.variable_scope(self.config.get("scope", "bert"), reuse=reuse):
with tf.variable_scope("embeddings"):
# Perform embedding lookup on the word ids.
(self.embedding_output_word,
self.embedding_table) = bert_modules.embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=self.config.hidden_size,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=self.config.use_one_hot_embeddings)
if kargs.get("perturbation", None):
self.embedding_output_word += kargs["perturbation"]
tf.logging.info(
" add word pertubation for robust learning ")
# Add positional embeddings and token type embeddings, then layer
# normalize and perform dropout.
self.embedding_output = bert_modules.embedding_postprocessor(
input_tensor=self.embedding_output_word,
use_token_type=True,
token_type_ids=token_type_ids,
token_type_vocab_size=self.config.type_vocab_size,
token_type_embedding_name="token_type_embeddings",
use_position_embeddings=True,
position_embedding_name="position_embeddings",
initializer_range=self.config.initializer_range,
max_position_embeddings=self.config.
max_position_embeddings,
dropout_prob=hidden_dropout_prob)
def build_embedder(self, input_ids, token_type_ids,
hidden_dropout_prob,
attention_probs_dropout_prob,
**kargs):
reuse = kargs["reuse"]
if self.config.get("embedding", "none_factorized") == "none_factorized":
projection_width = self.config.hidden_size
tf.logging.info("==not using embedding factorized==")
else:
projection_width = self.config.get('embedding_size', self.config.hidden_size)
tf.logging.info("==using embedding factorized: embedding size: %s==", str(projection_width))
if self.config.get('embedding_scope', None):
embedding_scope = self.config['embedding_scope']
other_embedding_scope = self.config.get("scope", "bert")
tf.logging.info("==using embedding scope of original model_config.embedding_scope: %s==", embedding_scope)
else:
embedding_scope = self.config.get("scope", "bert")
other_embedding_scope = self.config.get("scope", "bert")
tf.logging.info("==using embedding scope of original model_config.scope: %s==", embedding_scope)
with tf.variable_scope(embedding_scope, reuse=reuse):
with tf.variable_scope("embeddings"):
# Perform embedding lookup on the word ids.
(self.embedding_output_word, self.embedding_table) = bert_modules.embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=projection_width,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=self.config.use_one_hot_embeddings)
if kargs.get("perturbation", None):
self.embedding_output_word += kargs["perturbation"]
tf.logging.info(" add word pertubation for robust learning ")
with tf.variable_scope(other_embedding_scope, reuse=reuse):
with tf.variable_scope("embeddings"):
# Add positional embeddings and token type embeddings, then layer
# normalize and perform dropout.
tf.logging.info("==using segment type embedding ratio: %s==", str(self.config.get("token_type_ratio", 1.0)))
self.embedding_output = bert_modules.embedding_postprocessor(
input_tensor=self.embedding_output_word,
use_token_type=True,
token_type_ids=token_type_ids,
token_type_vocab_size=self.config.type_vocab_size,
token_type_embedding_name="token_type_embeddings",
use_position_embeddings=True,
position_embedding_name="position_embeddings",
initializer_range=self.config.initializer_range,
max_position_embeddings=self.config.max_position_embeddings,
dropout_prob=hidden_dropout_prob,
token_type_ratio=self.config.get("token_type_ratio", 1.0))
def build_embedder(self, input_ids, token_type_ids, rule_type_ids,
hidden_dropout_prob, attention_probs_dropout_prob,
**kargs):
reuse = kargs["reuse"]
with tf.variable_scope(self.config.get("scope", "bert"), reuse=reuse):
with tf.variable_scope("embeddings"):
# Perform embedding lookup on the word ids.
(self.embedding_output_word,
self.embedding_table) = bert_modules.embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=self.config.hidden_size,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=self.config.use_one_hot_embeddings)
if kargs.get("perturbation", None):
self.embedding_output_word += kargs["perturbation"]
tf.logging.info(
" add word pertubation for robust learning ")
# Add positional embeddings and token type embeddings, then layer
# normalize and perform dropout.
# modify token type to rule type
# rule with associated type:
# [0,1] for class-unrelated and class-related
# [0,1,2,3,4] for specific class type
self.embedding_output = bert_modules.embedding_rule_type_postprocessor(
input_tensor=self.embedding_output_word,
use_token_type=True,
token_type_ids=token_type_ids,
rule_type_ids=rule_type_ids,
token_type_vocab_size=self.config.type_vocab_size,
rule_type_size=self.config.rule_type_size,
token_type_embedding_name=
"token_type_embeddings", # rule type to incoroperate mined rule
use_position_embeddings=True,
position_embedding_name="position_embeddings",
rule_type_embedding_name="rule_type_embedding",
use_rule_type_embeddings=True,
initializer_range=self.config.initializer_range,
max_position_embeddings=self.config.
max_position_embeddings,
dropout_prob=hidden_dropout_prob)
def build_embedder(self,
input_ids,
token_type_ids,
hidden_dropout_prob,
attention_probs_dropout_prob,
past=None,
decode_loop_step=None,
**kargs):
reuse = kargs["reuse"]
if self.config.get("embedding",
"none_factorized") == "none_factorized":
projection_width = self.config.hidden_size
tf.logging.info("==not using embedding factorized==")
else:
projection_width = self.config.get('embedding_size',
self.config.hidden_size)
tf.logging.info(
"==using embedding factorized: embedding size: %s==",
str(projection_width))
if self.config.get('embedding_scope', None):
embedding_scope = self.config['embedding_scope']
other_embedding_scope = self.config[
'embedding_scope'] #self.config.get("scope", "bert")
tf.logging.info(
"==using embedding scope of original model_config.embedding_scope: %s, other_embedding_scope:%s ==",
embedding_scope, other_embedding_scope)
else:
embedding_scope = self.config.get("scope", "bert")
other_embedding_scope = self.config.get("scope", "bert")
tf.logging.info(
"==using embedding scope of original model_config.embedding_scope: %s, other_embedding_scope:%s ==",
embedding_scope, other_embedding_scope)
if past is None:
self.past_length = 0
else:
# batch_size_, num_layers_, two_, num_heads_, self.cache_length, features_
if decode_loop_step is None:
# gpu-decode length
past_shape = bert_utils.get_shape_list(past, expected_rank=[6])
self.past_length = past_shape[-2]
else:
self.past_length = decode_loop_step
with tf.variable_scope(embedding_scope, reuse=reuse):
with tf.variable_scope("embeddings"):
# Perform embedding lookup on the word ids.
# (self.embedding_output_word, self.embedding_table) = bert_modules.embedding_lookup(
# input_ids=input_ids,
# vocab_size=self.config.vocab_size,
# embedding_size=projection_width,
# initializer_range=self.config.initializer_range,
# word_embedding_name="word_embeddings",
# use_one_hot_embeddings=self.config.use_one_hot_embeddings)
input_shape = bert_utils.get_shape_list(input_ids,
expected_rank=[2, 3])
print(input_shape, "=====input_shape=====")
if len(input_shape) == 3:
tf.logging.info("****** 3D embedding matmul *******")
(self.embedding_output_word, self.embedding_table
) = bert_modules.gumbel_embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=projection_width,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=self.config.
use_one_hot_embeddings)
elif len(input_shape) == 2:
(self.embedding_output_word,
self.embedding_table) = bert_modules.embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=projection_width,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=self.config.
use_one_hot_embeddings)
else:
(self.embedding_output_word,
self.embedding_table) = bert_modules.embedding_lookup(
input_ids=input_ids,
vocab_size=self.config.vocab_size,
embedding_size=projection_width,
initializer_range=self.config.initializer_range,
word_embedding_name="word_embeddings",
use_one_hot_embeddings=self.config.
use_one_hot_embeddings)
if kargs.get("perturbation", None):
self.embedding_output_word += kargs["perturbation"]
tf.logging.info(
" add word pertubation for robust learning ")
with tf.variable_scope(other_embedding_scope, reuse=reuse):
with tf.variable_scope("embeddings"):
# Add positional embeddings and token type embeddings, then layer
# normalize and perform dropout.
tf.logging.info("==using segment type embedding ratio: %s==",
str(self.config.get("token_type_ratio", 1.0)))
self.embedding_output = bert_seq_modules.embedding_postprocessor(
input_tensor=self.embedding_output_word,
use_token_type=kargs.get('use_token_type', True),
token_type_ids=token_type_ids,
token_type_vocab_size=self.config.type_vocab_size,
token_type_embedding_name="token_type_embeddings",
use_position_embeddings=True,
position_embedding_name="position_embeddings",
initializer_range=self.config.initializer_range,
max_position_embeddings=self.config.
max_position_embeddings,
dropout_prob=hidden_dropout_prob,
token_type_ratio=self.config.get("token_type_ratio", 1.0),
position_offset=self.past_length)