def add_embedding(self):
"""Add embedding layer. that maps from vocabulary to vectors.
inputs: a list of tensors each of which have a size of [batch_size, embed_size]
"""
self.global_step = tf.Variable(0, name='global_step', trainable=False)
vocab_sz = max(self.config.vocab_dict.values())
with tf.variable_scope('embedding') as scp:
self.exclude_reg_scope(scp) ## exclude scope 추가
if self.config.pre_trained:
embed = utils.readEmbedding(
self.config.embed_path) ## embedding 파일 조회
embed_matrix, valid_mask = utils.mkEmbedMatrix(
embed,
dict(self.config.vocab_dict)) ## embedding matrix 생성
embedding = tf.Variable(embed_matrix,
'Embedding') ## embedding 생성
partial_update_embedding = entry_stop_gradients(
embedding, tf.expand_dims(valid_mask, 1)
) ## a tensor have the same value of target, but some entry will have no gradient during backprop
embedding = tf.cond(
self.on_epoch < self.config.partial_update_until_epoch,
lambda: partial_update_embedding, lambda: embedding
) ## https://www.tensorflow.org/api_docs/python/tf/cond
else:
embedding = tf.get_variable('Embedding',
[vocab_sz, self.config.embed_size],
trainable=True) ## embedding 생성
return embedding
def add_loss_op(self, logits, labels):
'''
:param logits: shape(b_sz, c_num) type(float)
:param labels: shape(b_sz,) type(int)
:return:
'''
self.prediction = tf.argmax(logits, axis=-1, output_type=labels.dtype)
self.accuracy = tf.reduce_mean(
tf.cast(tf.equal(self.prediction, labels), tf.float32))
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=labels)
ce_loss = tf.reduce_mean(entry_stop_gradients(loss, self.stop))
# ce_loss = tf.reduce_mean(loss)
exclude_vars = nest.flatten(
[[v for v in tf.trainable_variables(o.name)]
for o in self.EX_REG_SCOPE])
exclude_vars_2 = [
v for v in tf.trainable_variables() if '/bias:' in v.name
]
exclude_vars = exclude_vars + exclude_vars_2
reg_var_list = [
v for v in tf.trainable_variables() if v not in exclude_vars
]
reg_loss = tf.add_n([tf.nn.l2_loss(v) for v in reg_var_list])
self.param_cnt = np.sum(
[np.prod(v.get_shape().as_list()) for v in reg_var_list])
print('===' * 20)
print('total reg parameter count: %.3f M' %
(self.param_cnt / 1000000.))
print('excluded variables from regularization')
print([v.name for v in exclude_vars])
print('===' * 20)
print('regularized variables')
print([
'%s:%.3fM' % (v.name, np.prod(v.get_shape().as_list()) / 1000000.)
for v in reg_var_list
])
print('===' * 20)
'''shape(b_sz,)'''
self.ce_loss = ce_loss
self.w_loss = tf.reduce_mean(tf.multiply(loss, self.ph_sample_weights))
reg = self.config.reg
return self.ce_loss + reg * reg_loss
def add_embedding(self):
"""Add embedding layer. that maps from vocabulary to vectors.
inputs: a list of tensors each of which have a size of [batch_size, embed_size]
"""
if self.config.pre_trained:
embed = helper.readEmbedding(self.config.embed_path +
str(self.config.embed_size))
embed_matrix, valid_mask = helper.mkEmbedMatrix(
embed, self.vocab.word_to_index)
embedding = tf.Variable(embed_matrix, 'Embedding')
embedding = entry_stop_gradients(embedding,
tf.expand_dims(valid_mask, 1))
else:
embedding = tf.get_variable(
'Embedding', [len(self.vocab), self.config.embed_size],
trainable=True)
return embedding
def add_embedding(self):
"""Add embedding layer. that maps from vocabulary to vectors.
inputs: a list of tensors each of which have a size of [batch_size, embed_size]
"""
self.global_step = tf.Variable(0, name='global_step', trainable=False)
vocab_sz = max(self.config.vocab_dict.values())
with tf.variable_scope('embedding') as scp:
self.exclude_reg_scope(scp)
if self.config.pre_trained:
embed = utils.readEmbedding(self.config.embed_path)
embed_matrix, valid_mask = utils.mkEmbedMatrix(
embed, dict(self.config.vocab_dict))
embedding = tf.Variable(embed_matrix, 'Embedding')
partial_update_embedding = entry_stop_gradients(
embedding, tf.expand_dims(valid_mask, 1))
embedding = tf.cond(
self.on_epoch < self.config.partial_update_until_epoch,
lambda: partial_update_embedding, lambda: embedding)
else:
embedding = tf.get_variable('Embedding',
[vocab_sz, self.config.embed_size],
trainable=True)
return embedding