def test_get_dimension_is_correct(self):
encoder = CnnEncoder(embedding_dim=5, num_filters=4, ngram_filter_sizes=(3, 5))
assert encoder.get_output_dim() == 8
assert encoder.get_input_dim() == 5
encoder = CnnEncoder(embedding_dim=5, num_filters=4, ngram_filter_sizes=(3, 5), output_dim=7)
assert encoder.get_output_dim() == 7
assert encoder.get_input_dim() == 5
def test_get_dimension_is_correct(self):
encoder = CnnEncoder(embedding_dim=5, num_filters=4, ngram_filter_sizes=(3, 5))
assert encoder.get_output_dim() == 8
assert encoder.get_input_dim() == 5
encoder = CnnEncoder(
embedding_dim=5, num_filters=4, ngram_filter_sizes=(3, 5), output_dim=7
)
assert encoder.get_output_dim() == 7
assert encoder.get_input_dim() == 5
class Discriminator(Model):
def __init__(self, embedder: TextFieldEmbedder, embedding_size: int,
num_filters: int, vocab: Vocabulary) -> None:
super().__init__(vocab)
self.embedder = embedder
self.encoder = CnnEncoder(embedding_size, num_filters=num_filters)
self.linear = torch.nn.Linear(
in_features=self.encoder.get_output_dim(),
out_features=vocab.get_vocab_size('labels'))
self.loss_function = torch.nn.CrossEntropyLoss()
def forward(self,
tokens: Dict[str, torch.Tensor],
label: torch.Tensor = None) -> Dict[str, torch.Tensor]:
mask = get_text_field_mask(tokens)
embeddings = self.embedder(tokens)
encoder_out = self.encoder(embeddings, mask)
logits = self.linear(encoder_out)
output = {"logits": logits}
output["loss"] = self.loss_function(logits, label)
return output
class LabelEmbedModel(Model):
def __init__(self, vocab, encoder, attention, g, out_dim):
super(LabelEmbedModel, self).__init__(vocab)
_label_out_dim = 100
self.text_encoder = encoder
self.label_encoder = LabelEmbedding(g.num_features, _label_out_dim)
self.tl_attn = attention(self.text_encoder.get_output_dim(),
self.label_encoder.get_output_dim())
self.encoder = CnnEncoder(self.text_encoder.get_output_dim() +
self.label_encoder.get_output_dim(),
num_filters=100)
# self.encoder = CnnEncoder(self.text_encoder.get_output_dim(),
# num_filters=100)
self._classification_layer = torch.nn.Linear(
self.encoder.get_output_dim(), out_dim)
self._metric = MultiLabelMetric()
self._loss = torch.nn.BCEWithLogitsLoss()
def forward(self, text, label, graph):
text_vec = self.text_encoder(text)
label_vec = self.label_encoder(graph[0])
att_vec = self.tl_attn(text_vec, label_vec)
vec = torch.cat([text_vec, att_vec], dim=-1)
vec = self.encoder(vec, None)
logits = self._classification_layer(vec)
probs = torch.softmax(logits, dim=-1)
output_dict = {"logits": logits, "probs": probs}
if label is not None:
self._metric(predictions=logits, gold_labels=label)
output_dict['loss'] = self._loss(input=logits,
target=label.float())
return output_dict
def get_metrics(self, reset: bool = False):
metrics = {'f-score': self._metric.get_metric(reset)}
return metrics
class SentimentClassifier(Model):
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
dropout: float = 0.0,
input_dropout: float = 0.0,
label_smoothing: float = 0.1,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(SentimentClassifier, self).__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
share_rnn = nn.LSTM(input_size=self._text_field_embedder.get_output_dim(),
hidden_size=150,
batch_first=True,
# dropout=dropout,
bidirectional=True)
share_encoder = PytorchSeq2SeqWrapper(share_rnn)
self._encoder = RNNEncoder(vocab, share_encoder, input_dropout, regularizer)
self._seq_vec = CnnEncoder(self._encoder.get_output_dim(), 25)
self._de_dim = len(TASKS_NAME)
weight = torch.empty(self._de_dim, self._text_field_embedder.get_output_dim())
torch.nn.init.orthogonal_(weight)
self._domain_embeddings = Embedding(self._de_dim, self._text_field_embedder.get_output_dim(), weight=weight)
self._de_attention = BilinearAttention(self._seq_vec.get_output_dim(),
self._domain_embeddings.get_output_dim())
self._de_feedforward = FeedForward(self._domain_embeddings.get_output_dim(), 1,
self._seq_vec.get_output_dim(), Activation.by_name("elu")())
self._num_classes = self.vocab.get_vocab_size("label")
self._sentiment_discriminator = Discriminator(self._seq_vec.get_output_dim(), self._num_classes)
self._s_domain_discriminator = Discriminator(self._seq_vec.get_output_dim(), len(TASKS_NAME))
self._valid_discriminator = Discriminator(self._domain_embeddings.get_output_dim(), 2)
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
self._label_smoothing = label_smoothing
self.metrics = {
"s_domain_acc": CategoricalAccuracy(),
"valid_acc": CategoricalAccuracy()
}
for task_name in TASKS_NAME:
self.metrics["{}_stm_acc".format(task_name)] = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
self._domain_loss = torch.nn.CrossEntropyLoss()
# TODO torch.nn.BCELoss
self._valid_loss = torch.nn.BCEWithLogitsLoss()
initializer(self)
@overrides
def forward(self, # type: ignore
task_index: torch.IntTensor,
reverse: torch.ByteTensor,
for_training: torch.ByteTensor,
train_stage: torch.IntTensor,
tokens: Dict[str, torch.LongTensor],
label: torch.IntTensor = None) -> Dict[str, torch.Tensor]:
"""
:param task_index:
:param reverse:
:param for_training:
:param train_stage: ["share_senti", "share_classify",
"share_classify_adversarial", "domain_valid", "domain_valid_adversarial"]
:param tokens:
:param label:
:return:
"""
embedded_text = self._text_field_embedder(tokens)
mask = get_text_field_mask(tokens).float()
embed_tokens = self._encoder(embedded_text, mask)
batch_size = get_batch_size(embed_tokens)
# bs * (25*4)
seq_vec = self._seq_vec(embed_tokens, mask)
# TODO add linear layer
domain_embeddings = self._domain_embeddings(torch.arange(self._de_dim).cuda())
de_scores = F.softmax(
self._de_attention(seq_vec, domain_embeddings.expand(batch_size, *domain_embeddings.size())), dim=1)
de_valid = False
if np.random.rand() < 0.3:
de_valid = True
noise = 0.01 * torch.normal(mean=0.5,
# std=torch.std(domain_embeddings).sign_())
std=torch.empty(*de_scores.size()).fill_(1.0))
de_scores = de_scores + noise.cuda()
domain_embedding = torch.matmul(de_scores, domain_embeddings)
domain_embedding = self._de_feedforward(domain_embedding)
# train sentiment classify
if train_stage.cpu() == torch.tensor(0) or not for_training:
de_representation = torch.tanh(torch.add(domain_embedding, seq_vec))
sentiment_logits = self._sentiment_discriminator(de_representation)
if label is not None:
loss = self._loss(sentiment_logits, label)
self.metrics["{}_stm_acc".format(TASKS_NAME[task_index.cpu()])](sentiment_logits, label)
if train_stage.cpu() == torch.tensor(1) or not for_training:
s_domain_logits = self._s_domain_discriminator(seq_vec, reverse=reverse)
task_index = task_index.expand(batch_size)
loss = self._domain_loss(s_domain_logits, task_index)
self.metrics["s_domain_acc"](s_domain_logits, task_index)
if train_stage.cpu() == torch.tensor(2) or not for_training:
valid_logits = self._valid_discriminator(domain_embedding, reverse=reverse)
valid_label = torch.ones(batch_size).cuda()
if de_valid:
valid_label = torch.zeros(batch_size).cuda()
if self._label_smoothing is not None and self._label_smoothing > 0.0:
loss = sequence_cross_entropy_with_logits(valid_logits,
valid_label.unsqueeze(0).cuda(),
torch.tensor(1).unsqueeze(0).cuda(),
average="token",
label_smoothing=self._label_smoothing)
else:
loss = self._valid_loss(valid_logits,
torch.zeros(2).scatter_(0, valid_label, torch.tensor(1.0)).cuda())
self.metrics["valid_acc"](valid_logits, valid_label)
# TODO add orthogonal loss
output_dict = {"loss": loss}
return output_dict
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
Does a simple argmax over the class probabilities, converts indices to string labels, and
adds a ``"label"`` key to the dictionary with the result.
"""
class_probabilities = F.softmax(output_dict['logits'], dim=-1)
output_dict['class_probabilities'] = class_probabilities
predictions = class_probabilities.cpu().data.numpy()
argmax_indices = np.argmax(predictions, axis=-1)
labels = [self.vocab.get_token_from_index(x, namespace="label")
for x in argmax_indices]
output_dict['label'] = labels
return output_dict
@overrides
def get_metrics(self, task_name: str, reset: bool = False) -> Dict[str, float]:
return {metric_name: metric.get_metric(reset) for metric_name, metric in self.metrics.items() if
(task_name or "s_domain_acc" or "valid_acc") in metric_name}
class FRModel(Model):
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
pos_tag_embedding: Embedding = None,
users_embedding: Embedding = None,
dropout: float = 0.1,
label_namespace: str = "labels",
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: RegularizerApplicator = None) -> None:
super().__init__(vocab, regularizer)
self._label_namespace = label_namespace
self._dropout = Dropout(dropout)
self._text_field_embedder = text_field_embedder
self._pos_tag_embedding = pos_tag_embedding or None
representation_dim = self._text_field_embedder.get_output_dim()
if pos_tag_embedding is not None:
representation_dim += self._pos_tag_embedding.get_output_dim()
self._report_cnn = CnnEncoder(representation_dim, 25)
self._comment_cnn = CnnEncoder(representation_dim, 25)
lstm_input_dim = self._comment_cnn.get_output_dim()
self._user_embedding = users_embedding or None
if users_embedding is not None:
lstm_input_dim += self._user_embedding.get_output_dim()
rnn = nn.LSTM(input_size=lstm_input_dim,
hidden_size=150,
batch_first=True,
bidirectional=True)
self._encoder = PytorchSeq2SeqWrapper(rnn)
self._seq2vec = CnnEncoder(self._encoder.get_output_dim(), 25)
self._num_class = self.vocab.get_vocab_size(self._label_namespace)
self._bilinear_sim = BilinearSimilarity(self._encoder.get_output_dim(),
self._encoder.get_output_dim())
self._projector = FeedForward(self._seq2vec.get_output_dim(), 2,
[50, self._num_class],
Activation.by_name("sigmoid")(), dropout)
self._golden_instances = None
self._golden_instances_labels = None
self._golden_instances_id = None
self._metrics = {
"accuracy":
CategoricalAccuracy(),
"f-measure":
F1Measure(
positive_label=vocab.get_token_index("feature", "labels")),
}
self._loss = torch.nn.CrossEntropyLoss()
self._contrastive_loss = ContrastiveLoss()
self._mse_loss = torch.nn.MSELoss()
initializer(self)
@staticmethod
def contrastive_loss(left, right):
pairwise_distance = torch.nn.PairwiseDistance(p=1).cuda()
return torch.exp(-pairwise_distance(left, right)).cuda()
def _instance_forward(self,
dialog: Dict[str, torch.LongTensor],
users: torch.LongTensor = None,
pos_tags: torch.LongTensor = None):
dialog['tokens'] = pad_sequence2len(dialog['tokens'], -1, 5)
dialog_embedder = self._text_field_embedder(dialog)
dialog_embedder = self._dropout(dialog_embedder)
if pos_tags is not None and self._pos_tag_embedding is not None:
pos_tags = pad_sequence2len(pos_tags, -1, 5)
pos_tags_embedder = self._pos_tag_embedding(pos_tags)
dialog_embedder = torch.cat([dialog_embedder, pos_tags_embedder],
-1)
dialog_mask = get_text_field_mask(dialog, num_wrapping_dims=1).float()
dialog_shape = dialog_embedder.size()
dialog_embedder = dialog_embedder.view(
dialog_shape[0] * dialog_shape[1], -1, dialog_shape[-1])
dialog_out = self._comment_cnn(
dialog_embedder, dialog_mask.view(dialog_embedder.size()[:-1]))
dialog_out = dialog_out.view(*dialog_shape[:2], -1)
dialog_mask = torch.sum(dialog_mask, -1) > 0
if users is not None and self._user_embedding is not None:
users_embedder = self._user_embedding(users)
dialog_out = torch.cat([users_embedder, dialog_out], -1)
rnn_out = self._encoder(dialog_out, dialog_mask)
rnn_out = pad_sequence2len(rnn_out, 1, 5)
dialog_mask = pad_sequence2len(dialog_mask, -1, 5)
rnn2vec = self._seq2vec(rnn_out, dialog_mask)
return rnn2vec
def forward_gold_instances(self, d_id, dialog, user, pos_tag, label_tags):
if self._golden_instances is None:
self._golden_instances = torch.tensor(
self._instance_forward(dialog, user, pos_tag))
else:
self._golden_instances = torch.cat([
self._golden_instances,
self._instance_forward(dialog, user, pos_tag)
])
if self._golden_instances_labels is None:
self._golden_instances_labels = torch.tensor(label_tags)
else:
self._golden_instances_labels = torch.cat(
[self._golden_instances_labels, label_tags])
if self._golden_instances_id is None:
self._golden_instances_id = [d_id]
else:
self._golden_instances_id.append(d_id)
def forward(self,
dialog1: Dict[str, torch.LongTensor],
dialog2: Dict[str, torch.LongTensor] = None,
users1: torch.LongTensor = None,
pos_tags1: torch.LongTensor = None,
users2: torch.LongTensor = None,
pos_tags2: torch.LongTensor = None,
label: torch.IntTensor = None,
label_tags: torch.IntTensor = None,
metadata: List[Dict[str, Any]] = None) -> Dict[str, Any]:
output_dict = dict()
if metadata is not None and not metadata[0]["is_gold"]:
output_dict["pair_instance"] = [
meta["pair_instance"] for meta in metadata
]
if metadata is not None and metadata[0]["is_gold"]:
self.forward_gold_instances(metadata[0]["pair_instance"][0][0],
dialog1, users1, pos_tags1, label_tags)
return output_dict
rnn_vec1 = self._instance_forward(dialog1, users1, pos_tags1)
if self._golden_instances is not None:
logits = []
for gold in self._golden_instances:
logits.append(
self._projector(
torch.cat([rnn_vec1, gold.unsqueeze(0)], -1)))
output_dict['logits'] = logits
else:
logits = self._projector(rnn_vec1)
probs = nn.functional.softmax(logits, dim=-1)
output_dict["logits"] = logits
output_dict["probs"] = probs
if label is not None:
loss = self._loss(logits, label)
output_dict['loss'] = loss
output_dict['label_tags'] = label_tags
for metric_name, metric in self._metrics.items():
metric(logits, label)
return output_dict
def inference(self, predictions, label_tags, d_id):
if predictions.dim() == 2:
predictions_list = [
predictions[i] for i in range(predictions.shape[0])
]
else:
predictions_list = [predictions]
classes = []
for prediction in predictions_list:
label_idx = prediction.argmax(dim=-1).item()
label_str = (self.vocab.get_index_to_token_vocabulary(
self._label_namespace).get(label_idx, str(label_idx)))
classes.append(label_str)
golden_names = []
if len(label_tags) == 1:
label_tags = label_tags.expand(len(classes))
for tags in label_tags:
if tags == self.vocab.get_token_index("feature@feature",
"label_tags"):
golden_names.append("feature")
if tags == self.vocab.get_token_index("other@other", "label_tags"):
golden_names.append("other")
predict_labels = []
pos_ins = []
neg_ins = []
for class_name, golden_name in zip(classes, golden_names):
if class_name == "same":
if golden_name == "feature":
predict_labels.append(1 / 31)
pos_ins.append(d_id)
elif golden_name == "other":
predict_labels.append(0)
elif class_name == "diff":
if golden_name == "feature":
predict_labels.append(0)
elif golden_name == "other":
predict_labels.append(1 / 261)
# TODO return gold_id
return pos_ins + neg_ins, predict_labels
@overrides
def decode(self, output_dict: Dict[str, Any]) -> Dict[str, Any]:
"""
Does a simple argmax over the probabilities, converts index to string label, and
add ``"label"`` key to the dictionary with the result.
"""
if 'logits' not in output_dict.keys():
output_dict["label"] = ["gold"]
return output_dict
if isinstance(output_dict['logits'], list):
infered = np.array([
self.inference(nn.functional.softmax(logits, dim=-1),
label_tags, d_id)
for logits, label_tags, d_id in zip(
output_dict['logits'],
self._golden_instances_labels.unsqueeze(1),
self._golden_instances_id)
])
vote_id = infered[:, 0]
vote_id = [v[0] for v in vote_id if len(v) > 0]
predict_socres = infered[:, 1]
predict_socres = [float(p[0]) for p in predict_socres]
predict_socres = np.sum(predict_socres, -1)
output_dict["vote"] = []
ins_id = []
is_feature = []
if len(vote_id) > 10:
ins_id = vote_id
ins_id = sorted(ins_id)
if len(ins_id) > 0:
is_feature.append(" || ".join(ins_id))
output_dict["vote"].append(len(ins_id))
else:
is_feature.append("")
output_dict["vote"].append(0)
else:
_, _ = self.inference(output_dict['probs'], output_dict) > 0
predict_labels = [
"feature" if len(label) > 0 else "other" for label in is_feature
]
output_dict["label"] = predict_labels
for pred, ins, vote, vote_ins in zip(predict_labels,
output_dict["pair_instance"],
output_dict["vote"], is_feature):
if "feature" in pred:
dialog_id = ins[0][0]
with open("bs_pred_fr.txt", "a", encoding="utf8") as f:
f.write(
f"ID: {dialog_id}\tVote: {vote}\tVote_Ins: {vote_ins}\n"
)
return output_dict
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
metrics = dict()
metrics['accuracy'] = self._metrics['accuracy'].get_metric(reset)
precision, recall, fscore = self._metrics['f-measure'].get_metric(
reset)
metrics['precision'] = precision
metrics['recall'] = recall
metrics['fscore'] = fscore
return metrics