def apply(self, is_train, x, mask=None):
if is_train:
shape = get_shape_tuple(x)
return tf.nn.dropout(x,
rate=self.dropout_rate,
noise_shape=[shape[0], 1, shape[2]])
else:
return x
def words_to_char_ids(words, word_length, cpu=True):
flat_words = tf.reshape(words, [-1])
# Surprisingly, using a py_func here is much faster then the pure tensorflow option
# Presumably because we have to .map in the tensorflow version which is very slow
flat_char_ids = tf.py_func(lambda x: words_to_char_ids_py(x, word_length),
[flat_words], [tf.int32], False)
return tf.reshape(flat_char_ids,
ops.get_shape_tuple(words) + [word_length])
def apply(self, is_train, x, mask=None):
x_shape = get_shape_tuple(x)
dim = x_shape[-1]
time = x_shape[-2]
if mask is not None:
x *= tf.expand_dims(tf.sequence_mask(mask, x_shape[1], tf.float32),
2)
if self.leftpad:
if self.same:
raise ValueError()
x = tf.pad(x, [[0, 0], [self.filter_size - 1, 0], [0, 0]])
n_filters = self.num_filters
if self.activation == "glu":
n_filters *= 2
if len(x_shape) != 3:
x = tf.reshape(x, [-1, time, dim])
filter_ = tf.get_variable("conv1d/filters",
shape=[self.filter_size, dim, n_filters],
dtype='float')
out = tf.nn.conv1d(x, filter_, 1, "SAME" if self.same else "VALID")
if self.activation is not None:
bias = tf.get_variable("conv1d/bias",
shape=[self.num_filters],
dtype='float',
initializer=tf.zeros_initializer())
if self.activation == "glu":
gates, lin = tf.split(out, 2, -1)
out = tf.nn.sigmoid(gates + bias) * lin
else:
out = activation_fn(out + bias, self.activation)
if len(x_shape) != 3:
out = tf.reshape(out, x_shape[:-2] + get_shape_tuple(out)[-2:])
return out
def eval_triviaqa_op(logits, tokens, actual_answers, bound):
"""Tensorflow op to compute em/f1 scores using TriviaQA metrics"""
answer_spans = ops.get_best_span(logits, bound)
# Unlike SQuAD, for TriviaQA we don't bother properly untokenizing the
# span, and just return the tokens with space seperators, since
# that is almost always good enough for TriviaQA
answer_text = tf.map_fn(lambda i: tf.reduce_join(
tokens[i, answer_spans[i][0]:answer_spans[i][1] + 1
], 0, separator=" "),
tf.range(ops.get_shape_tuple(logits, 0)),
dtype=tf.string,
back_prop=False)
scores = tf.py_func(_eval_triviaqa_decode, [answer_text, actual_answers],
tf.float32, False)
scores.set_shape([logits.shape.as_list()[0], 2])
return scores
def compute_clf_loss(self, hidden, logits, bias, labels, mask=None): loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) label_one_hot = tf.one_hot(labels, ops.get_shape_tuple(logits, 1)) weights = 1 - tf.reduce_sum(tf.exp(bias) * label_one_hot, 1) return tf.reduce_sum(weights*loss) / tf.reduce_sum(weights)
def apply(self, is_train, x, mask=None, inital_states=None):
x_t = tf.transpose(x, [1, 0, 2])
batch = ops.get_shape_tuple(x_t, 1)
bidr = self.direction == "bi"
if inital_states is not None:
inital_states = tuple(tf.expand_dims(x, 0) for x in inital_states)
if self.learn_initial_states and inital_states is None:
if bidr:
names = ["fw_h", "fw_c", "bw_h", "bw_c"]
else:
names = ["fw_h", "fw_c"]
initial_states = []
for n in names:
v = tf.get_variable(n, (1, self.n_out), tf.float32,
tf.zeros_initializer())
initial_states.append(
tf.tile(tf.expand_dims(v, 1), [1, batch, 1]))
else:
initial_states = [
tf.zeros((1, batch, self.n_out)) for _ in range(2 + 2 * bidr)
]
if self.direction == 'bi':
with tf.variable_scope("forward"):
fw = self._apply_transposed(is_train,
x_t,
initial_states=initial_states[:2])
with tf.variable_scope("backward"):
x_bw = x_t[::-1] if mask is None else tf.reverse_sequence(
x_t, mask, 0, 1)
bw = self._apply_transposed(is_train,
x_bw,
initial_states=initial_states[2:])
bw = bw[::-1] if mask is None else tf.reverse_sequence(
bw, mask, 0, 1)
out = tf.concat([fw, bw], axis=2)
elif self.direction == "fw":
out = self._apply_transposed(is_train,
x_t,
initial_states=initial_states)
elif self.direction == "bw":
x_bw = x_t[::-1] if mask is None else tf.reverse_sequence(
x_t, mask, 0, 1)
bw = self._apply_transposed(is_train,
x_bw,
initial_states=initial_states)
out = bw[::-1] if mask is None else tf.reverse_sequence(
bw, mask, 0, 1)
else:
raise ValueError()
out = tf.transpose(out, [1, 0, 2])
if mask is not None:
out *= tf.expand_dims(
tf.cast(tf.sequence_mask(mask,
tf.shape(out)[1]), tf.float32), 2)
return out
def get_logit_masked_scores(self, is_train, tensor_1, tensor_2, mask1,
mask2):
atten = self.get_scores(is_train, tensor_1, tensor_2)
dim1, dim2 = ops.get_shape_tuple(atten)[1:]
mask = compute_attention_mask(mask1, mask2, dim1, dim2)
return ops.mask_logits(atten, mask)
def _embed_words_from_ids(self, is_train, tensors, additional_wids=None):
if self._input_vocab is None:
raise NotImplementedError()
wids = [x[0] for x in tensors]
shapes = [ops.get_shape_tuple(x) for x in wids]
unique_wids = [ops.flatten(x) for x in wids]
sizes = [ops.get_shape_tuple(x, 0) for x in unique_wids]
if additional_wids is not None:
unique_wids.append(additional_wids)
unique_wids = tf.concat(unique_wids, 0)
wixs, w_mapping = tf.unique(ops.flatten(unique_wids), tf.int32)
if additional_wids is not None:
w_mapping = w_mapping[:-ops.get_shape_tuple(additional_wids)[0]]
if self.use_word_vecs:
if self.embed_cpu:
with tf.device("/cpu:0"):
embeding_var = ops.as_initialized_variable(self._embeddings, "embeddings")
w_embed = [tf.nn.embedding_lookup(embeding_var, wixs)]
else:
embeding_var = ops.as_initialized_variable(self._embeddings, "embeddings")
w_embed = [tf.nn.embedding_lookup(embeding_var, wixs)]
else:
w_embed = []
if self.use_chars:
with tf.device("/cpu:0"):
cids = tf.gather(self._cached_char_ids, wixs)
# dim = self._cached_char_ids.shape[1]
# cids = tf.matmul(self._cached_char_ids,
# tf.one_hot(wixs, dim, dtype=tf.int32))
char_emb = char_encoder.embed_ids(cids, self.char_embed_dim)
with tf.variable_scope("char-map"):
char_emb = self.character_mapper.apply(is_train, char_emb)
with tf.variable_scope("char-pool"):
char_emb = self.character_pooler.apply(is_train, char_emb)
w_embed.append(char_emb)
w_embed = tf.concat(w_embed, 1)
unique_word_embeddings = w_embed
if self.word_mapper is not None:
with tf.variable_scope("word-map"):
w_embed = self.word_mapper.apply(is_train, w_embed)
dim = w_embed.shape.as_list()[-1]
# Undo tf.unique
w_embeds = tf.gather(w_embed, w_mapping)
# Undo tf.concat
w_embeds = tf.split(w_embeds, sizes, 0)
w_mapping = tf.split(w_mapping, sizes, 0)
# Undo ops.flatten
w_embeds = [tf.reshape(t, s + [dim]) for t, s in zip(w_embeds, shapes)]
w_mapping = [tf.reshape(t, s) for t, s in zip(w_mapping, shapes)]
return w_embeds, w_mapping, unique_word_embeddings