def build_placeholder(self, config):
self.config = config
with self.graph.as_default():
self.model_name = self.config.model_name
self.token_emb_mat = self.config["token_emb_mat"]
self.char_emb_mat = self.config["char_emb_mat"]
self.vocab_size = int(self.config["vocab_size"])
self.char_vocab_size = int(self.config["char_vocab_size"])
self.max_length = int(self.config["max_length"])
self.emb_size = int(self.config["emb_size"])
self.extra_symbol = self.config["extra_symbol"]
self.scope = self.config["scope"]
self.num_classes = int(self.config["num_classes"])
self.batch_size = int(self.config["batch_size"])
self.grad_clipper = float(self.config.get("grad_clipper", 10.0))
self.char_limit = self.config.get("char_limit", 10)
self.char_dim = self.config.get("char_emb_size", 300)
# ---- add EMLO embedding ----
if self.config.apply_elmo:
self.elmo_token_emb_mat = self.config["elmo_token_emb_mat"]
self.elmo_vocab_size = int(self.config["elmo_vocab_size"])
self.elmo_emb_size = int(self.config["elmo_emb_size"])
# ---- place holder -----
self.seq_word_left = tf.placeholder(tf.int32, [None, None], name='seq_word_left')
self.seq_word_right = tf.placeholder(tf.int32, [None, None], name='seq_word_right')
self.labels = tf.placeholder(tf.int32, [None], name='gold_label')
# self.sent1_token_len = tf.placeholder(tf.int32, [None], name='sent1_token_lengths')
# self.sent2_token_len = tf.placeholder(tf.int32, [None], name='sent2_token_lengths')
self.seq_word_ans = tf.placeholder(tf.int32, [None, None, None], name='seq_word_answer')
self.seq_len_word_ans = tf.placeholder(tf.int32, shape=[None, None], name="seq_word_len_answer")
self.seq_word_ans_mask = tf.cast(self.seq_word_ans, tf.bool)
self.seq_word_ans_len = tf.reduce_sum(tf.cast(self.seq_word_ans_mask, tf.int32), -1)
self.seq_len_word_left = tf.placeholder(tf.int32, shape=[None], name="seq_len_word_left")
self.seq_len_word_right = tf.placeholder(tf.int32, shape=[None], name="seq_len_word_right")
self.seq_word_left_mask = tf.cast(self.seq_word_left, tf.bool)
self.seq_word_left_len = tf.reduce_sum(tf.cast(self.seq_word_left_mask, tf.int32), -1)
self.seq_word_right_mask = tf.cast(self.seq_word_right, tf.bool)
self.seq_word_right_len = tf.reduce_sum(tf.cast(self.seq_word_right_mask, tf.int32), -1)
if self.config.with_char:
# self.sent1_char_len = tf.placeholder(tf.int32, [None,None]) # [batch_size, question_len]
# self.sent2_char_len = tf.placeholder(tf.int32, [None,None]) # [batch_size, passage_len]
self.seq_char_left = tf.placeholder(tf.int32, [None, None, None], name="seq_char_left") # [batch_size, question_len, q_char_len]
self.seq_char_right = tf.placeholder(tf.int32, [None, None, None], name="seq_char_right") # [batch_size, passage_len, p_char_len]
self.seq_len_char_left = tf.placeholder(tf.int32, shape=[None], name="seq_len_char_left")
self.seq_len_char_right = tf.placeholder(tf.int32, shape=[None], name="seq_len_char_right")
self.seq_char_left_mask = tf.cast(self.seq_char_left, tf.bool)
self.seq_char_left_len = tf.reduce_sum(tf.cast(self.seq_char_left_mask, tf.int32), -1)
self.seq_char_right_mask = tf.cast(self.seq_char_right, tf.bool)
self.seq_char_right_len = tf.reduce_sum(tf.cast(self.seq_char_right_mask, tf.int32), -1)
self.seq_char_ans = tf.placeholder(tf.int32, [None, None, None, None], name="seq_char_answer")
self.seq_char_ans_mask = tf.cast(self.seq_char_ans, tf.bool)
self.seq_char_ans_len = tf.reduce_sum(tf.cast(self.seq_char_ans_mask, tf.int32), -1)
self.char_mat = integration_func.generate_embedding_mat(self.vocab_size, emb_len=self.emb_size,
init_mat=self.token_emb_mat,
extra_symbol=self.extra_symbol,
scope='gene_char_emb_mat')
self.emb_mat = integration_func.generate_embedding_mat(self.vocab_size, emb_len=self.emb_size,
init_mat=self.token_emb_mat,
extra_symbol=self.extra_symbol,
scope='gene_token_emb_mat')
if self.config.apply_elmo:
self.elmo_mat = integration_func.generate_embedding_mat(self.elmo_vocab_size,
emb_len=self.elmo_emb_size,
init_mat=self.elmo_token_emb_mat,
extra_symbol=self.extra_symbol,
scope='elmo_gene_token_emb_mat')
# ---------------- for dynamic learning rate -------------------
# self.learning_rate = tf.placeholder(tf.float32, [], 'learning_rate')
self.dropout_prob = tf.placeholder(tf.float32, name="dropout_prob")
self.is_training = tf.placeholder(tf.bool, name="is_training")
self.global_step = tf.Variable(0, name="global_step", trainable=False)
#### features
if self.config.use_features:
self.features = tf.placeholder(tf.float32, shape=[None, self.config["num_features"]], name="features")
self.learning_rate = tf.train.exponential_decay(self.config["init_lr"], self.global_step,
self.config["decay_steps"], self.config["decay_rate"])
self.augmentation_dropout = tf.train.exponential_decay(self.config["augmentation_init_dropout"], self.global_step,
self.config["augmentation_dropout_decay_steps"],
self.config["augmentation_dropout_decay_rate"])
self.augmentation_permutation = tf.train.exponential_decay(self.config["augmentation_init_permutation"],
self.global_step,
self.config["augmentation_permutation_decay_steps"],
self.config["augmentation_permutation_decay_rate"])
def build_placeholder(self, config):
self.config = config
with self.graph.as_default():
self.token_emb_mat = self.config["token_emb_mat"]
self.char_emb_mat = self.config["char_emb_mat"]
self.vocab_size = int(self.config["vocab_size"])
self.char_vocab_size = int(self.config["char_vocab_size"])
self.max_length = int(self.config["max_length"])
self.emb_size = int(self.config["emb_size"])
self.extra_symbol = self.config["extra_symbol"]
self.scope = self.config["scope"]
self.num_classes = int(self.config["num_classes"])
self.batch_size = int(self.config["batch_size"])
self.grad_clipper = float(self.config.get("grad_clipper", 10.0))
self.char_limit = self.config.get("char_limit", 10)
self.char_dim = self.config.get("char_emb_size", 300)
self.idf_emb_mat = self.config.get("idf_emb_mat", None)
# ---- place holder -----
self.sent_token = tf.placeholder(tf.int32, [None, None],
name='sent_token')
self.entity_token = tf.placeholder(tf.int32, [None, None],
name='entity_token')
self.gold_label = tf.placeholder(tf.int32, [None],
name='gold_label')
self.entity_token_mask = tf.cast(self.entity_token, tf.bool)
self.entity_token_len = tf.reduce_sum(
tf.cast(self.entity_token_mask, tf.int32), -1)
self.sent_token_mask = tf.cast(self.sent_token, tf.bool)
self.sent_token_len = tf.reduce_sum(
tf.cast(self.sent_token_mask, tf.int32), -1)
if self.config.with_char:
# self.sent1_char_len = tf.placeholder(tf.int32, [None,None]) # [batch_size, question_len]
# self.sent2_char_len = tf.placeholder(tf.int32, [None,None]) # [batch_size, passage_len]
self.sent_char = tf.placeholder(
tf.int32, [None, None, None
]) # [batch_size, question_len, q_char_len]
self.sent_char_mask = tf.cast(self.sent_char, tf.bool)
self.sent_char_len = tf.reduce_sum(
tf.cast(self.sent_char_mask, tf.int32), -1)
self.char_mat = integration_func.generate_embedding_mat(
self.vocab_size,
emb_len=self.emb_size,
init_mat=self.token_emb_mat,
extra_symbol=self.extra_symbol,
scope=self.scope + '_gene_char_emb_mat')
if self.config.with_idf:
print("====apply idf embedding====")
self.idf_mat = integration_func.generate_embedding_mat(
self.vocab_size,
emb_len=1,
init_mat=self.idf_emb_mat,
extra_symbol=self.extra_symbol,
scope=self.scope + '_gene_idf_emb_mat')
self.emb_mat = integration_func.generate_embedding_mat(
self.vocab_size,
emb_len=self.emb_size,
init_mat=self.token_emb_mat,
extra_symbol=self.extra_symbol,
scope=self.scope + '_gene_token_emb_mat')
# ---------------- for dynamic learning rate -------------------
self.learning_rate = tf.placeholder(tf.float32, [],
'learning_rate')
self.dropout_prob = tf.placeholder(tf.float32, name="dropout_prob")
self.is_training = tf.placeholder(tf.bool, name="is_training")
self.global_step = tf.Variable(0,
name="global_step",
trainable=False)
def build_placeholder(self, config):
self.config = config
with self.graph.as_default():
self.token_emb_mat = self.config["token_emb_mat"]
self.char_emb_mat = self.config["char_emb_mat"]
self.vocab_size = int(self.config["vocab_size"])
self.char_vocab_size = int(self.config["char_vocab_size"])
self.max_length = int(self.config["max_length"])
self.emb_size = int(self.config["emb_size"])
self.extra_symbol = self.config["extra_symbol"]
self.scope = self.config["scope"]
self.num_classes = int(self.config["num_classes"])
self.batch_size = int(self.config["batch_size"])
self.grad_clipper = float(self.config.get("grad_clipper", 10.0))
self.char_limit = self.config.get("char_limit", 10)
self.char_dim = self.config.get("char_emb_size", 300)
self.entity_max_len = self.config.get("entity_max_len", 20)
self.class_emb_mat = self.config.get("class_emb_mat", None)
# ---- place holder -----
self.sent_token = tf.placeholder(tf.int32, [None, None], name='sent_token')
self.entity_token = tf.placeholder(tf.int32, [None, None], name='entity_token')
self.gold_label = tf.placeholder(tf.int32, [None], name='gold_label')
self.entity_token_mask = tf.cast(self.entity_token, tf.bool)
self.entity_token_len = tf.reduce_sum(tf.cast(self.entity_token_mask, tf.int32), -1)
self.sent_token_mask = tf.cast(self.sent_token, tf.bool)
self.sent_token_len = tf.reduce_sum(tf.cast(self.sent_token_mask, tf.int32), -1)
if self.config.with_char:
self.sent_char = tf.placeholder(tf.int32, [None, None, None]) # [batch_size, question_len, q_char_len]
self.sent_char_mask = tf.cast(self.sent_char, tf.bool)
self.sent_char_len = tf.reduce_sum(tf.cast(self.sent_char_mask, tf.int32), -1)
self.char_mat = integration_func.generate_embedding_mat(self.vocab_size, emb_len=self.emb_size,
init_mat=self.token_emb_mat,
extra_symbol=self.extra_symbol,
scope=self.scope+'gene_char_emb_mat')
self.emb_mat = integration_func.generate_embedding_mat(self.vocab_size, emb_len=self.emb_size,
init_mat=self.token_emb_mat,
extra_symbol=self.extra_symbol,
scope=self.scope+'gene_token_emb_mat')
with tf.variable_scope(self.scope + '_class_emb'):
try:
print("===pre-trained class emb")
self.memory = tf.get_variable("label_memory",
shape=[self.num_classes, self.emb_size],
dtype=tf.float32,
initializer=tf.constant(self.class_emb_mat),
trainable=True)
except:
print("===random initialize class emb")
self.memory = tf.get_variable("label_memory",
shape=[self.num_classes, self.emb_size],
dtype=tf.float32,
initializer=tf.random_uniform_initializer(-0.001, 0.001),
trainable=True)
# ---------------- for dynamic learning rate -------------------
self.learning_rate = tf.placeholder(tf.float32, [], 'learning_rate')
self.dropout_prob = tf.placeholder(tf.float32, name="dropout_prob")
self.is_training = tf.placeholder(tf.bool, name="is_training")
self.global_step = tf.Variable(0, name="global_step", trainable=False)
