def get_features(self, words, train=False, update=True):
"""
get feature representations
"""
# word embeddings
wfeatures = np.array([
self.get_w_repr(word, train=train, update=update) for word in words
])
lex_features = []
if self.dictionary and not self.type_constraint:
## add lexicon features
lex_features = np.array(
[self.get_lex_repr(word) for word in words])
# char embeddings
if self.c_in_dim > 0:
cfeatures = [self.get_c_repr(word, train=train) for word in words]
if len(lex_features) > 0:
lex_features = dynet.inputTensor(lex_features)
features = [
dynet.concatenate([w, c, l])
for w, c, l in zip(wfeatures, cfeatures, lex_features)
]
else:
features = [
dynet.concatenate([w, c])
for w, c in zip(wfeatures, cfeatures)
]
else:
features = wfeatures
if train: # only do at training time
features = [dynet.noise(fe, self.noise_sigma) for fe in features]
return features
def predict(self,
feature_vector,
task_ids,
train=False,
soft_labels=False,
temperature=None,
dropout_rate=0.0,
orthogonality_weight=0.0,
domain_id=None):
dynet.renew_cg() # new graph
feature_vector = feature_vector.toarray()
feature_vector = np.squeeze(feature_vector, axis=0)
# self.input = dynet.vecInput(self.vocab_size)
# self.input.set(feature_vector)
# TODO this takes too long; can we speed this up somehow?
input = dynet.inputVector(feature_vector)
for i in range(self.h_layers):
if train: # add some noise
input = dynet.noise(input, self.noise_sigma)
input = dynet.dropout(input, dropout_rate)
input = self.layers[i](input)
outputs = []
for task_id in task_ids:
output = self.output_layers_dict[task_id](input,
soft_labels=soft_labels,
temperature=temperature)
outputs.append(output)
constraint, adv_loss = 0, 0
if orthogonality_weight != 0:
# put the orthogonality constraint either directly on the
# output layer or on the hidden layer if it's an MLP
F0_layer = self.output_layers_dict["F0"]
F1_layer = self.output_layers_dict["F1"]
F0_param = F0_layer.W_mlp if self.add_hidden else F0_layer.W
F1_param = F1_layer.W_mlp if self.add_hidden else F1_layer.W
F0_W = dynet.parameter(F0_param)
F1_W = dynet.parameter(F1_param)
# calculate the matrix product of the task matrix with both others
matrix_product = dynet.transpose(F0_W) * F1_W
# take the squared Frobenius norm by squaring
# every element and then summing them
squared_frobenius_norm = dynet.sum_elems(
dynet.square(matrix_product))
constraint += squared_frobenius_norm
# print('Constraint with first matrix:', squared_frobenius_norm.value())
if domain_id is not None:
# flip the gradient when back-propagating through here
adv_input = dynet.flip_gradient(input) # last state
adv_output = self.adv_layer(adv_input)
adv_loss = self.pick_neg_log(adv_output, domain_id)
# print('Adversarial loss:', avg_adv_loss.value())
return outputs, constraint, adv_loss
def predict(self, seq, train=False, output_confidences=False, unk_tag=None, update_embeds=True):
"""
predict tags for a sentence represented as char+word embeddings and compute losses for this instance
"""
if not train:
dynet.renew_cg()
features = self.get_features(seq.words, train=train, update=update_embeds)
output_expected_at_layer = self.predictors["task_expected_at"][seq.task_id]
output_expected_at_layer -=1
# go through layers
# input is now combination of w + char emb
prev = features
prev_rev = features
num_layers = self.h_layers
for i in range(0,num_layers):
predictor = self.predictors["inner"][i]
forward_sequence, backward_sequence = predictor.predict_sequence(prev, prev_rev)
if i > 0 and self.activation:
# activation between LSTM layers
forward_sequence = [self.activation(s) for s in forward_sequence]
backward_sequence = [self.activation(s) for s in backward_sequence]
if i == output_expected_at_layer:
output_predictor = self.predictors["output_layers_dict"][seq.task_id]
concat_layer = [dynet.concatenate([f, b]) for f, b in zip(forward_sequence,reversed(backward_sequence))]
if train and self.noise_sigma > 0.0:
concat_layer = [dynet.noise(fe,self.noise_sigma) for fe in concat_layer]
# fill-in predictions and get loss per tag
losses = output_predictor.predict_sequence(seq, concat_layer,
train=train, output_confidences=output_confidences,
unk_tag=unk_tag, dictionary=self.dictionary,
type_constraint=self.type_constraint)
prev = forward_sequence
prev_rev = backward_sequence
if train:
# return losses
return losses
else:
return seq.pred_tags, seq.tag_confidences
def predict(self, seq, train=False, output_confidences=False, unk_tag=None, update_embeds=True):
"""
predict tags for a sentence represented as char+word embeddings and compute losses for this instance
"""
if not train:
dynet.renew_cg()
features = self.get_features(seq.words, train=train, update=update_embeds)
output_expected_at_layer = self.predictors["task_expected_at"][seq.task_id]
output_expected_at_layer -=1
# go through layers
# input is now combination of w + char emb
prev = features
prev_rev = features
num_layers = self.h_layers
for i in range(0,num_layers):
predictor = self.predictors["inner"][i]
forward_sequence, backward_sequence = predictor.predict_sequence(prev, prev_rev)
if i > 0 and self.activation:
# activation between LSTM layers
forward_sequence = [self.activation(s) for s in forward_sequence]
backward_sequence = [self.activation(s) for s in backward_sequence]
if i == output_expected_at_layer:
output_predictor = self.predictors["output_layers_dict"][seq.task_id]
concat_layer = [dynet.concatenate([f, b]) for f, b in zip(forward_sequence,reversed(backward_sequence))]
if train and self.noise_sigma > 0.0:
concat_layer = [dynet.noise(fe,self.noise_sigma) for fe in concat_layer]
# fill-in predictions and get loss per tag
losses = output_predictor.predict_sequence(seq, concat_layer,
train=train, output_confidences=output_confidences,
unk_tag=unk_tag, dictionary=self.dictionary,
type_constraint=self.type_constraint)
prev = forward_sequence
prev_rev = backward_sequence
if train:
# return losses
return losses
else:
return seq.pred_tags, seq.tag_confidences
def get_features(self, words, train=False, update=True):
"""
get feature representations
"""
# word embeddings
wfeatures = np.array([self.get_w_repr(word, train=train, update=update) for word in words])
lex_features = []
if self.dictionary and not self.type_constraint:
## add lexicon features
lex_features = np.array([self.get_lex_repr(word) for word in words])
# char embeddings
if self.c_in_dim > 0:
cfeatures = [self.get_c_repr(word, train=train) for word in words]
if len(lex_features) > 0:
lex_features = dynet.inputTensor(lex_features)
features = [dynet.concatenate([w,c,l]) for w,c,l in zip(wfeatures,cfeatures,lex_features)]
else:
features = [dynet.concatenate([w, c]) for w, c in zip(wfeatures, cfeatures)]
else:
features = wfeatures
if train: # only do at training time
features = [dynet.noise(fe,self.noise_sigma) for fe in features]
return features
def predict(self,
feature_vector,
train=False,
soft_labels=False,
temperature=None,
dropout_rate=None):
dynet.renew_cg() # new graph
feature_vector = feature_vector.toarray()
feature_vector = np.squeeze(feature_vector, axis=0)
# self.input = dynet.vecInput(self.vocab_size)
# self.input.set(feature_vector)
# TODO this takes too long; can we speed this up somehow?
input = dynet.inputVector(feature_vector)
for i in range(self.h_layers - 1):
if train: # add some noise
input = dynet.noise(input, self.noise_sigma)
input = dynet.dropout(input, dropout_rate)
input = self.layers[i](input)
output = self.layers[-1](input,
soft_labels=soft_labels,
temperature=temperature)
return output
def predict(self,
word_indices,
char_indices,
train=False,
soft_labels=False,
temperature=None):
"""
predict tags for a sentence represented as char+word embeddings
"""
dynet.renew_cg() # new graph
char_emb = []
rev_char_emb = []
wfeatures = [self.wembeds[w] for w in word_indices]
if self.c_in_dim > 0:
# get representation for words
for chars_of_token in char_indices:
char_feats = [self.cembeds[c] for c in chars_of_token]
# use last state as word representation
f_char, b_char = self.char_rnn.predict_sequence(
char_feats, char_feats)
last_state = f_char[-1]
rev_last_state = b_char[-1]
char_emb.append(last_state)
rev_char_emb.append(rev_last_state)
features = [
dynet.concatenate([w, c, rev_c])
for w, c, rev_c in zip(wfeatures, char_emb, rev_char_emb)
]
else:
features = wfeatures
if train: # only do at training time
features = [dynet.noise(fe, self.noise_sigma) for fe in features]
output_expected_at_layer = self.h_layers
output_expected_at_layer -= 1
# go through layers
prev = features
prev_rev = features
num_layers = self.h_layers
for i in range(0, num_layers):
predictor = self.predictors["inner"][i]
forward_sequence, backward_sequence = predictor.predict_sequence(
prev, prev_rev)
if i > 0 and self.activation:
# activation between LSTM layers
forward_sequence = [
self.activation(s) for s in forward_sequence
]
backward_sequence = [
self.activation(s) for s in backward_sequence
]
if i == output_expected_at_layer:
output_predictor = self.predictors["output_layers_dict"]
concat_layer = [
dynet.concatenate([f, b]) for f, b in zip(
forward_sequence, reversed(backward_sequence))
]
if train and self.noise_sigma > 0.0:
concat_layer = [
dynet.noise(fe, self.noise_sigma)
for fe in concat_layer
]
output = output_predictor.predict_sequence(
concat_layer,
soft_labels=soft_labels,
temperature=temperature)
return output
prev = forward_sequence
prev_rev = backward_sequence
raise Exception("oops should not be here")
return None
def predict(self,
word_indices,
char_indices,
task_id,
train=False,
soft_labels=False,
temperature=None,
orthogonality_weight=0.0,
domain_id=None):
"""
predict tags for a sentence represented as char+word embeddings
:param domain_id: Predict adversarial loss if domain id is provided.
"""
dynet.renew_cg() # new graph
char_emb = []
rev_char_emb = []
wfeatures = [self.wembeds[w] for w in word_indices]
if self.c_in_dim > 0:
# get representation for words
for chars_of_token in char_indices:
char_feats = [self.cembeds[c] for c in chars_of_token]
# use last state as word representation
f_char, b_char = self.char_rnn.predict_sequence(
char_feats, char_feats)
last_state = f_char[-1]
rev_last_state = b_char[-1]
char_emb.append(last_state)
rev_char_emb.append(rev_last_state)
features = [
dynet.concatenate([w, c, rev_c])
for w, c, rev_c in zip(wfeatures, char_emb, rev_char_emb)
]
else:
features = wfeatures
if train: # only do at training time
features = [dynet.noise(fe, self.noise_sigma) for fe in features]
output_expected_at_layer = self.h_layers
output_expected_at_layer -= 1
# go through layers
prev = features
prev_rev = features
num_layers = self.h_layers
constraint = 0
adv_loss = 0
for i in range(0, num_layers):
predictor = self.predictors["inner"][i]
forward_sequence, backward_sequence = predictor.predict_sequence(
prev, prev_rev)
if i > 0 and self.activation:
# activation between LSTM layers
forward_sequence = [
self.activation(s) for s in forward_sequence
]
backward_sequence = [
self.activation(s) for s in backward_sequence
]
if i == output_expected_at_layer:
concat_layer = [
dynet.concatenate([f, b]) for f, b in zip(
forward_sequence, reversed(backward_sequence))
]
if train and self.noise_sigma > 0.0:
concat_layer = [
dynet.noise(fe, self.noise_sigma)
for fe in concat_layer
]
if task_id not in ["src", "trg"]:
output_predictor = self.predictors["output_layers_dict"][
task_id]
output = output_predictor.predict_sequence(
concat_layer,
soft_labels=soft_labels,
temperature=temperature)
else:
# one src example for all three outputs
output = [] # in this case it is a list
for t_id in self.task_ids:
output_predictor = self.predictors[
"output_layers_dict"][t_id]
output_t = output_predictor.predict_sequence(
concat_layer,
soft_labels=soft_labels,
temperature=temperature)
output.append(output_t)
if orthogonality_weight != 0 and task_id != "Ft":
# put the orthogonality constraint either directly on the
# output layer or on the hidden layer if it's an MLP
# use orthogonality_weight only between F0 and F1
builder = self.predictors["output_layers_dict"][
"F0"].network_builder
task_param = builder.W_mlp if self.add_hidden else builder.W
task_W = dynet.parameter(task_param)
builder = self.predictors["output_layers_dict"][
"F1"].network_builder
other_param = builder.W_mlp if self.add_hidden else builder.W
other_task_W = dynet.parameter(other_param)
# calculate the matrix product of the task matrix with the other
matrix_product_1 = dynet.transpose(task_W) * other_task_W
# take the squared Frobenius norm by squaring
# every element and then summing them
squared_frobenius_norm = dynet.sum_elems(
dynet.square(matrix_product_1))
constraint = squared_frobenius_norm
#print('Constraint with first matrix:', squared_frobenius_norm.value())
if domain_id is not None:
# flip the gradient when back-propagating through here
adv_input = dynet.flip_gradient(
concat_layer[-1]) # last state
adv_output = self.adv_layer(adv_input)
adv_loss = self.pick_neg_log(adv_output, domain_id)
#print('Adversarial loss:', avg_adv_loss.value())
# output is list if task_id = 'src'
return output, constraint, adv_loss
prev = forward_sequence
prev_rev = backward_sequence
raise Exception("oops should not be here")
return None