def __init__(
self,
vocab: Vocabulary,
passage_attention_to_count: Seq2SeqEncoder,
dropout: float = 0.2,
initializers: InitializerApplicator = InitializerApplicator(),
) -> None:
super(PassageAttnToCount, self).__init__(vocab=vocab)
self.scaling_vals = [1, 2, 5, 10]
self.passage_attention_to_count = passage_attention_to_count
assert len(self.scaling_vals
) == self.passage_attention_to_count.get_input_dim()
self.num_counts = 10
# self.passage_count_predictor = torch.nn.Linear(self.passage_attention_to_count.get_output_dim(),
# self.num_counts, bias=False)
# We want to predict a score for each passage token
self.passage_count_hidden2logits = torch.nn.Linear(
self.passage_attention_to_count.get_output_dim(), 1, bias=True)
self.passagelength_to_bias = torch.nn.Linear(1, 1, bias=True)
self.count_acc = Average()
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
initializers(self)
def __init__(self,
num_entities: int,
num_relations: int,
embedding_dim: int,
box_type: str = 'SigmoidBoxTensor',
single_box: bool = False,
softbox_temp: float = 10.,
number_of_negative_samples: int = 0,
debug: bool = False,
regularization_weight: float = 0,
init_interval_center: float = 0.25,
init_interval_delta: float = 0.1,
neg_samples_in_dataset_reader: int = 0,
gumbel_beta: float = 1.0,
n_samples: int = 10) -> None:
super().__init__(
num_entities,
num_relations,
embedding_dim,
box_type=box_type,
single_box=single_box,
softbox_temp=softbox_temp,
number_of_negative_samples=number_of_negative_samples,
debug=debug,
regularization_weight=regularization_weight,
init_interval_center=init_interval_center,
init_interval_delta=init_interval_delta,
neg_samples_in_dataset_reader=neg_samples_in_dataset_reader,
gumbel_beta=gumbel_beta,
n_samples=n_samples)
self.kl_div = Average()
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
seq2seq_encoder: Seq2SeqEncoder,
feedforward_encoder: Seq2SeqEncoder,
dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
):
super(SimpleGeneratorModel, self).__init__(vocab, regularizer)
self._vocabulary = vocab
self._text_field_embedder = text_field_embedder
self._seq2seq_encoder = seq2seq_encoder
self._dropout = torch.nn.Dropout(p=dropout)
self._feedforward_encoder = feedforward_encoder
self._classifier_input_dim = feedforward_encoder.get_output_dim()
self._classification_layer = torch.nn.Linear(
self._classifier_input_dim, 1)
self._rationale_f1_metric = F1Measure(positive_label=1)
self._rationale_length = Average()
self._rationale_supervision_loss = Average()
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
dropout: float = 0.2,
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(BidirectionalAttentionFlow, self).__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
encoding_dim = text_field_embedder.get_output_dim()
self._dropout = torch.nn.Dropout(p=dropout)
self._squad_metrics = SquadEmAndF1()
self.linear_start = nn.Linear(encoding_dim, 1)
self.linear_end = nn.Linear(encoding_dim, 1)
self.linear_type = nn.Linear(encoding_dim, 3)
self._loss_trackers = {
'loss': Average(),
'start_loss': Average(),
'end_loss': Average(),
'type_loss': Average()
}
def __init__(self,
vocab: Vocabulary,
action_embedding_dim: int,
text_field_embedder: TextFieldEmbedder = None,
dropout: float = 0.0,
rule_namespace: str = 'rule_labels',
debug: bool=False,
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(DROPParserBase, self).__init__(vocab=vocab, regularizer=regularizer)
self._denotation_accuracy = Average()
self._consistency = Average()
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
# This flag turns on the debugging mode which prints a bunch of stuff in self.decode (inside functions as well)
self._debug = debug
self._action_embedder = Embedding(num_embeddings=vocab.get_vocab_size(self._rule_namespace),
embedding_dim=action_embedding_dim,
vocab_namespace=self._rule_namespace)
self._action_embedding_dim = action_embedding_dim
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
torch.nn.init.normal_(self._first_action_embedding, mean=0.0, std=0.001)
def __init__(self,
num_entities: int,
num_relations: int,
embedding_dim: int,
box_type: str = 'SigmoidBoxTensor',
single_box: bool = False,
softbox_temp: float = 10.,
margin: float = 1.,
number_of_negative_samples: int = 0,
debug: bool = False,
regularization_weight: float = 0,
init_interval_center: float = 0.25,
init_interval_delta: float = 0.1) -> None:
super().__init__(num_entities,
num_relations,
embedding_dim,
box_type=box_type,
single_box=single_box,
softbox_temp=softbox_temp,
number_of_negative_samples=number_of_negative_samples,
debug=debug,
regularization_weight=regularization_weight,
init_interval_center=init_interval_center,
init_interval_delta=init_interval_delta)
self.number_of_negative_samples = number_of_negative_samples
self.centre_loss_metric = Average()
self.loss_f_centre: torch.nn.modules._Loss = torch.nn.MarginRankingLoss( # type: ignore
margin=margin, reduction='mean')
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
initializer: InitializerApplicator,
max_span_length: int = 30,
use_multi_label_loss: bool = False,
stats_report_freq: float = None,
debug_experiment_name: str = None) -> None:
super().__init__(vocab)
self._max_span_length = max_span_length
self._text_field_embedder = text_field_embedder
self._stats_report_freq = stats_report_freq
self._debug_experiment_name = debug_experiment_name
self._use_multi_label_loss = use_multi_label_loss
# see usage below for explanation
self.qa_outputs = torch.nn.Linear(
self._text_field_embedder.get_output_dim(), 2)
self.qa_yesno = torch.nn.Linear(
self._text_field_embedder.get_output_dim(), 3)
initializer(self)
self._official_f1 = Average()
self._official_EM = Average()
class STSBTask(Task):
''' Task class for Sentence Textual Similarity Benchmark. '''
def __init__(self, path, max_seq_len, name="sts_benchmark"):
''' '''
super(STSBTask, self).__init__(name, 1)
self.categorical = 0
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = Average()
self.load_data(path, max_seq_len)
def load_data(self, path, max_seq_len):
''' '''
tr_data = load_tsv(os.path.join(path, 'train.tsv'), max_seq_len, skip_rows=1,
s1_idx=7, s2_idx=8, targ_idx=9, targ_fn=lambda x: float(x) / 5)
val_data = load_tsv(os.path.join(path, 'dev.tsv'), max_seq_len, skip_rows=1,
s1_idx=7, s2_idx=8, targ_idx=9, targ_fn=lambda x: float(x) / 5)
te_data = load_tsv(os.path.join(path, 'test.tsv'), max_seq_len,
s1_idx=7, s2_idx=8, targ_idx=None, idx_idx=0, skip_rows=1)
self.train_data_text = tr_data
self.val_data_text = val_data
self.test_data_text = te_data
log.info("\tFinished loading STS Benchmark data.")
def get_metrics(self, reset=False):
# NB: I think I call it accuracy b/c something weird in training
return {'accuracy': self.scorer1.get_metric(reset),
'spearmanr': self.scorer2.get_metric(reset)}
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
entity_encoder: Seq2VecEncoder,
max_decoding_steps: int,
use_neighbor_similarity_for_linking: bool = False,
dropout: float = 0.0,
num_linking_features: int = 10,
rule_namespace: str = 'rule_labels',
tables_directory: str = '/wikitables/') -> None:
super(WikiTablesSemanticParser, self).__init__(vocab)
self._question_embedder = question_embedder
self._encoder = encoder
self._entity_encoder = TimeDistributed(entity_encoder)
self._max_decoding_steps = max_decoding_steps
self._use_neighbor_similarity_for_linking = use_neighbor_similarity_for_linking
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._denotation_accuracy = WikiTablesAccuracy(tables_directory)
self._action_sequence_accuracy = Average()
self._has_logical_form = Average()
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._action_biases = Embedding(num_embeddings=num_actions, embedding_dim=1)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous question attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_question = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_question)
check_dimensions_match(entity_encoder.get_output_dim(), question_embedder.get_output_dim(),
"entity word average embedding dim", "question embedding dim")
self._num_entity_types = 4 # TODO(mattg): get this in a more principled way somehow?
self._num_start_types = 5 # TODO(mattg): get this in a more principled way somehow?
self._embedding_dim = question_embedder.get_output_dim()
self._type_params = torch.nn.Linear(self._num_entity_types, self._embedding_dim)
self._neighbor_params = torch.nn.Linear(self._embedding_dim, self._embedding_dim)
if num_linking_features > 0:
self._linking_params = torch.nn.Linear(num_linking_features, 1)
else:
self._linking_params = None
if self._use_neighbor_similarity_for_linking:
self._question_entity_params = torch.nn.Linear(1, 1)
self._question_neighbor_params = torch.nn.Linear(1, 1)
else:
self._question_entity_params = None
self._question_neighbor_params = None
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
initializer: InitializerApplicator,
dropout: float = 0.2,
max_span_length: int = 30,
predictions_file = None,
use_multi_label_loss: bool = False,
stats_report_freq:float = None,
debug_experiment_name:str = None) -> None:
super().__init__(vocab)
self._max_span_length = max_span_length
self._text_field_embedder = text_field_embedder
self._stats_report_freq = stats_report_freq
self._debug_experiment_name = debug_experiment_name
self._use_multi_label_loss = use_multi_label_loss
self._predictions_file = predictions_file
# TODO move to predict
if predictions_file is not None and os.path.isfile(predictions_file):
os.remove(predictions_file)
# see usage below for explanation
self._all_qa_count = 0
self._qas_used_fraction = 1.0
self.qa_outputs = torch.nn.Linear(self._text_field_embedder.get_output_dim(), 2)
initializer(self)
self._official_f1 = Average()
self._official_EM = Average()
def __init__(
self,
vocab: Vocabulary,
sentence_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
dropout: float = 0.0,
rule_namespace: str = "rule_labels",
) -> None:
super(NlvrSemanticParser, self).__init__(vocab=vocab)
self._sentence_embedder = sentence_embedder
self._denotation_accuracy = Average()
self._consistency = Average()
self._encoder = encoder
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._action_embedder = Embedding(
num_embeddings=vocab.get_vocab_size(self._rule_namespace),
embedding_dim=action_embedding_dim,
)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action.
self._first_action_embedding = torch.nn.Parameter(
torch.FloatTensor(action_embedding_dim))
torch.nn.init.normal_(self._first_action_embedding)
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
entity_encoder: Seq2VecEncoder,
max_decoding_steps: int,
use_neighbor_similarity_for_linking: bool = False,
dropout: float = 0.0,
num_linking_features: int = 10,
rule_namespace: str = 'rule_labels',
tables_directory: str = '/wikitables/') -> None:
super(WikiTablesSemanticParser, self).__init__(vocab)
self._question_embedder = question_embedder
self._encoder = encoder
self._entity_encoder = TimeDistributed(entity_encoder)
self._max_decoding_steps = max_decoding_steps
self._use_neighbor_similarity_for_linking = use_neighbor_similarity_for_linking
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._denotation_accuracy = WikiTablesAccuracy(tables_directory)
self._action_sequence_accuracy = Average()
self._has_logical_form = Average()
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._action_biases = Embedding(num_embeddings=num_actions, embedding_dim=1)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous question attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_question = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_question)
check_dimensions_match(entity_encoder.get_output_dim(), question_embedder.get_output_dim(),
"entity word average embedding dim", "question embedding dim")
self._num_entity_types = 4 # TODO(mattg): get this in a more principled way somehow?
self._num_start_types = 5 # TODO(mattg): get this in a more principled way somehow?
self._embedding_dim = question_embedder.get_output_dim()
self._type_params = torch.nn.Linear(self._num_entity_types, self._embedding_dim)
self._neighbor_params = torch.nn.Linear(self._embedding_dim, self._embedding_dim)
if num_linking_features > 0:
self._linking_params = torch.nn.Linear(num_linking_features, 1)
else:
self._linking_params = None
if self._use_neighbor_similarity_for_linking:
self._question_entity_params = torch.nn.Linear(1, 1)
self._question_neighbor_params = torch.nn.Linear(1, 1)
else:
self._question_entity_params = None
self._question_neighbor_params = None
def __init__(self,
vocab: Vocabulary,
mydatabase: str,
schema_path: str,
utterance_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
input_attention: Attention,
add_action_bias: bool = True,
dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._utterance_embedder = utterance_embedder
self._encoder = encoder
self._max_decoding_steps = max_decoding_steps
self._add_action_bias = add_action_bias
self._dropout = torch.nn.Dropout(p=dropout)
self._exact_match = Average()
self._action_similarity = Average()
self._valid_sql_query = SqlValidity(mydatabase=mydatabase)
self._token_match = TokenSequenceAccuracy()
self._kb_match = KnowledgeBaseConstsAccuracy(schema_path=schema_path)
self._schema_free_match = GlobalTemplAccuracy(schema_path=schema_path)
self._coverage_loss = CoverageAttentionLossMetric()
# the padding value used by IndexField
self._action_padding_index = -1
num_actions = vocab.get_vocab_size("rule_labels")
input_action_dim = action_embedding_dim
if self._add_action_bias:
input_action_dim += 1
self._action_embedder = Embedding(num_embeddings=num_actions,
embedding_dim=input_action_dim)
self._output_action_embedder = Embedding(
num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous utterance attention.
self._first_action_embedding = torch.nn.Parameter(
torch.FloatTensor(action_embedding_dim))
self._first_attended_utterance = torch.nn.Parameter(
torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_utterance)
self._beam_search = decoder_beam_search
self._decoder_trainer = MaximumMarginalLikelihood(beam_size=1)
self._transition_function = BasicTransitionFunction(
encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=input_attention,
add_action_bias=self._add_action_bias,
dropout=dropout)
initializer(self)
def __init__(
self,
vocab: Vocabulary,
variational_encoder: VariationalEncoder,
decoder: Decoder,
kl_weight: LossWeight,
temperature: float = 1.0,
initializer: InitializerApplicator = InitializerApplicator()
) -> None:
super(VAE, self).__init__(vocab)
self._encoder = variational_encoder
self._decoder = decoder
self._latent_dim = variational_encoder.latent_dim
self._encoder_output_dim = self._encoder.get_encoder_output_dim()
self._start_index = self.vocab.get_token_index(START_SYMBOL)
self._end_index = self.vocab.get_token_index(END_SYMBOL)
self._pad_index = self.vocab.get_token_index(self.vocab._padding_token) # pylint: disable=protected-access
self._bleu = BLEU(exclude_indices={
self._pad_index, self._end_index, self._start_index
})
self._kl_metric = Average()
self.kl_weight = kl_weight
self._temperature = temperature
initializer(self)
class STSBTask(Task):
''' Task class for Sentence Textual Similarity Benchmark. '''
def __init__(self, path, max_seq_len, name="sts_benchmark"):
''' '''
super(STSBTask, self).__init__(name, 1)
self.categorical = 0
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = Average()
self.load_data(path, max_seq_len)
def load_data(self, path, max_seq_len):
''' '''
tr_data = load_tsv(os.path.join(path, 'train.tsv'), max_seq_len, skip_rows=1,
s1_idx=7, s2_idx=8, targ_idx=9, targ_fn=lambda x: float(x) / 5)
val_data = load_tsv(os.path.join(path, 'dev.tsv'), max_seq_len, skip_rows=1,
s1_idx=7, s2_idx=8, targ_idx=9, targ_fn=lambda x: float(x) / 5)
te_data = load_tsv(os.path.join(path, 'test.tsv'), max_seq_len,
s1_idx=7, s2_idx=8, targ_idx=None, idx_idx=0, skip_rows=1)
self.train_data_text = tr_data
self.val_data_text = val_data
self.test_data_text = te_data
log.info("\tFinished loading STS Benchmark data.")
def get_metrics(self, reset=False):
# NB: I think I call it accuracy b/c something weird in training
return {'accuracy': self.scorer1.get_metric(reset),
'spearmanr': self.scorer2.get_metric(reset)}
def __init__(self,
vocab: Vocabulary,
sentence_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
attention: Attention,
beam_size: int,
max_decoding_steps: int,
max_num_finished_states: int = None,
dropout: float = 0.0,
normalize_beam_score_by_length: bool = False,
checklist_cost_weight: float = 0.6,
dynamic_cost_weight: Dict[str, Union[int, float]] = None,
penalize_non_agenda_actions: bool = False,
initial_mml_model_file: str = None) -> None:
super(NlvrCoverageSemanticParser, self).__init__(vocab=vocab,
sentence_embedder=sentence_embedder,
action_embedding_dim=action_embedding_dim,
encoder=encoder,
dropout=dropout)
self._agenda_coverage = Average()
self._decoder_trainer: DecoderTrainer[Callable[[CoverageState], torch.Tensor]] = \
ExpectedRiskMinimization(beam_size=beam_size,
normalize_by_length=normalize_beam_score_by_length,
max_decoding_steps=max_decoding_steps,
max_num_finished_states=max_num_finished_states)
# Instantiating an empty NlvrWorld just to get the number of terminals.
self._terminal_productions = set(NlvrWorld([]).terminal_productions.values())
self._decoder_step = CoverageTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=attention,
num_start_types=1,
activation=Activation.by_name('tanh')(),
predict_start_type_separately=False,
add_action_bias=False,
dropout=dropout)
self._checklist_cost_weight = checklist_cost_weight
self._dynamic_cost_wait_epochs = None
self._dynamic_cost_rate = None
if dynamic_cost_weight:
self._dynamic_cost_wait_epochs = dynamic_cost_weight["wait_num_epochs"]
self._dynamic_cost_rate = dynamic_cost_weight["rate"]
self._penalize_non_agenda_actions = penalize_non_agenda_actions
self._last_epoch_in_forward: int = None
# TODO (pradeep): Checking whether file exists here to avoid raising an error when we've
# copied a trained ERM model from a different machine and the original MML model that was
# used to initialize it does not exist on the current machine. This may not be the best
# solution for the problem.
if initial_mml_model_file is not None:
if os.path.isfile(initial_mml_model_file):
archive = load_archive(initial_mml_model_file)
self._initialize_weights_from_archive(archive)
else:
# A model file is passed, but it does not exist. This is expected to happen when
# you're using a trained ERM model to decode. But it may also happen if the path to
# the file is really just incorrect. So throwing a warning.
logger.warning("MML model file for initializing weights is passed, but does not exist."
" This is fine if you're just decoding.")
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
phrase_layer: Seq2SeqEncoder,
residual_encoder: Seq2SeqEncoder,
span_start_encoder: Seq2SeqEncoder,
span_end_encoder: Seq2SeqEncoder,
initializer: InitializerApplicator,
dropout: float = 0.2,
pair2vec_dropout: float = 0.15,
max_span_length: int = 30,
pair2vec_model_file: str = None,
pair2vec_config_file: str = None) -> None:
super().__init__(vocab)
self._max_span_length = max_span_length
self._text_field_embedder = text_field_embedder
self._phrase_layer = phrase_layer
self._encoding_dim = phrase_layer.get_output_dim()
self.pair2vec = pair2vec_util.get_pair2vec(pair2vec_config_file,
pair2vec_model_file)
self._pair2vec_dropout = torch.nn.Dropout(pair2vec_dropout)
self._matrix_attention = LinearMatrixAttention(self._encoding_dim,
self._encoding_dim,
'x,y,x*y')
# atten_dim = self._encoding_dim * 4 + 600 if ablation_type == 'attn_over_rels' else self._encoding_dim * 4
atten_dim = self._encoding_dim * 4 + 600
self._merge_atten = TimeDistributed(
torch.nn.Linear(atten_dim, self._encoding_dim))
self._residual_encoder = residual_encoder
self._self_attention = LinearMatrixAttention(self._encoding_dim,
self._encoding_dim,
'x,y,x*y')
self._merge_self_attention = TimeDistributed(
torch.nn.Linear(self._encoding_dim * 3, self._encoding_dim))
self._span_start_encoder = span_start_encoder
self._span_end_encoder = span_end_encoder
self._span_start_predictor = TimeDistributed(
torch.nn.Linear(self._encoding_dim, 1))
self._span_end_predictor = TimeDistributed(
torch.nn.Linear(self._encoding_dim, 1))
self._squad_metrics = SquadEmAndF1()
initializer(self)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._official_em = Average()
self._official_f1 = Average()
self._span_accuracy = BooleanAccuracy()
self._variational_dropout = InputVariationalDropout(dropout)
def __init__(self, vocab: Vocabulary) -> None:
super().__init__(vocab)
self.vocab = vocab
self.current_task = "default"
self.tasks_vocabulary = {"default": vocab}
self.classification_layers = torch.nn.ModuleList([torch.nn.Linear(in_features=10, out_features=self.vocab.get_vocab_size('labels'))])
self.loss_function = torch.nn.CrossEntropyLoss()
self.accuracy = CategoricalAccuracy()
self.average = Average()
def __init__(self, path, max_seq_len, name="sts_benchmark"):
''' '''
super(STSBTask, self).__init__(name, 1)
self.categorical = 0
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = Average()
self.load_data(path, max_seq_len)
def __init__(self, path, max_seq_len, name="acceptability"):
''' '''
super(CoLATask, self).__init__(name, 2)
self.pair_input = 0
self.load_data(path, max_seq_len)
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = CategoricalAccuracy()
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
phrase_layer: Seq2SeqEncoder,
residual_encoder: Seq2SeqEncoder,
span_start_encoder: Seq2SeqEncoder,
span_end_encoder: Seq2SeqEncoder,
initializer: InitializerApplicator,
dropout: float = 0.2,
num_context_answers: int = 0,
marker_embedding_dim: int = 10,
max_span_length: int = 30) -> None:
super().__init__(vocab)
self._num_context_answers = num_context_answers
self._max_span_length = max_span_length
self._text_field_embedder = text_field_embedder
self._phrase_layer = phrase_layer
self._marker_embedding_dim = marker_embedding_dim
self._encoding_dim = phrase_layer.get_output_dim()
max_turn_length = 12
self._matrix_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._merge_atten = TimeDistributed(torch.nn.Linear(self._encoding_dim * 4, self._encoding_dim))
self._residual_encoder = residual_encoder
if num_context_answers > 0:
self._question_num_marker = torch.nn.Embedding(max_turn_length,
marker_embedding_dim * num_context_answers)
self._prev_ans_marker = torch.nn.Embedding((num_context_answers * 4) + 1, marker_embedding_dim)
self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._followup_lin = torch.nn.Linear(self._encoding_dim, 3)
self._merge_self_attention = TimeDistributed(torch.nn.Linear(self._encoding_dim * 3,
self._encoding_dim))
self._span_start_encoder = span_start_encoder
self._span_end_encoder = span_end_encoder
self._span_start_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1))
self._span_end_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1))
self._span_yesno_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 3))
self._span_followup_predictor = TimeDistributed(self._followup_lin)
initializer(self)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_yesno_accuracy = CategoricalAccuracy()
self._span_followup_accuracy = CategoricalAccuracy()
self._span_gt_yesno_accuracy = CategoricalAccuracy()
self._span_gt_followup_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._official_f1 = Average()
self._variational_dropout = InputVariationalDropout(dropout)
def __init__(self, path, max_seq_len, name="sts_benchmark"):
''' '''
super(STSBTask, self).__init__(name, 1)
self.categorical = 0
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = Average()
self.load_data(path, max_seq_len)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
num_highway_layers: int,
phrase_layer: Seq2SeqEncoder,
attention_similarity_function: SimilarityFunction,
modeling_layer: Seq2SeqEncoder,
span_end_encoder: Seq2SeqEncoder,
initializer: InitializerApplicator,
dropout: float = 0.2,
mask_lstms: bool = True,
evaluation_json_file: str = None) -> None:
super(BidirectionalAttentionFlow, self).__init__(vocab)
self._text_field_embedder = text_field_embedder
self._highway_layer = TimeDistributed(Highway(text_field_embedder.get_output_dim(),
num_highway_layers))
self._phrase_layer = phrase_layer
self._matrix_attention = MatrixAttention(attention_similarity_function)
self._modeling_layer = modeling_layer
self._span_end_encoder = span_end_encoder
encoding_dim = phrase_layer.get_output_dim()
modeling_dim = modeling_layer.get_output_dim()
span_start_input_dim = encoding_dim * 4 + modeling_dim
self._span_start_predictor = TimeDistributed(torch.nn.Linear(span_start_input_dim, 1))
span_end_encoding_dim = span_end_encoder.get_output_dim()
span_end_input_dim = encoding_dim * 4 + span_end_encoding_dim
self._span_end_predictor = TimeDistributed(torch.nn.Linear(span_end_input_dim, 1))
initializer(self)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._official_em = Average()
self._official_f1 = Average()
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._mask_lstms = mask_lstms
if evaluation_json_file:
logger.info("Prepping official evaluation dataset from %s", evaluation_json_file)
with open(evaluation_json_file) as dataset_file:
dataset_json = json.load(dataset_file)
question_to_answers = {}
for article in dataset_json['data']:
for paragraph in article['paragraphs']:
for question in paragraph['qas']:
question_id = question['id']
answers = [answer['text'] for answer in question['answers']]
question_to_answers[question_id] = answers
self._official_eval_dataset = question_to_answers
else:
self._official_eval_dataset = None
def __init__(self,
vocab: Vocabulary,
input_embedder: TextFieldEmbedder,
nli_projection_layer: FeedForward,
training_tasks: Any,
validation_tasks: Any,
langs_print_train: List[str] = ["en", "fr", "de", "ur", "sw"],
dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
feed_lang_ids: bool = True,
avg: bool = False) -> None:
super(SimpleProjectionXlm, self).__init__(vocab, regularizer)
self._avg = avg
if type(training_tasks) == dict:
self._training_tasks = list(training_tasks.keys())
else:
self._training_tasks = training_tasks
if type(validation_tasks) == dict:
self._validation_tasks = list(validation_tasks.keys())
else:
self._validation_tasks = validation_tasks
self._input_embedder = input_embedder
self._label_namespace = "labels"
self._num_labels = vocab.get_vocab_size(
namespace=self._label_namespace)
self._nli_projection_layer = nli_projection_layer
print(
vocab.get_token_to_index_vocabulary(
namespace=self._label_namespace))
assert nli_projection_layer.get_output_dim() == self._num_labels
self._dropout = torch.nn.Dropout(p=dropout)
self._loss = torch.nn.CrossEntropyLoss()
initializer(self._nli_projection_layer)
self._nli_per_lang_acc: Dict[str, CategoricalAccuracy] = dict()
for taskname in self._validation_tasks:
# this will hide some metrics from tqdm, but they will still be computed
self._nli_per_lang_acc[taskname] = CategoricalAccuracy()
self._nli_avg_acc = Average()
self._langs_pring_train = langs_print_train or "en"
if '*' in self._langs_pring_train:
self._langs_pring_train = [t.split("")[-1] for t in training_tasks]
self._feed_lang_ids = feed_lang_ids
def __init__(self, vocab: Vocabulary, text_model_name: str):
super().__init__(vocab)
self._text_model = BertForSequenceClassification.from_pretrained(
text_model_name)
self._num_labels = vocab.get_vocab_size()
self._accuracy = Average()
self._auc = Auc()
self._softmax = torch.nn.Softmax(dim=1)
def __init__(self, path, max_seq_len, name, **kw):
''' '''
super().__init__(name, **kw)
self.scorer1 = Average() # CategoricalAccuracy()
self.scorer2 = None
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.files_by_split = {split: os.path.join(path, "%s.csv" % split) for
split in ["train", "val", "test"]}
self.max_seq_len = max_seq_len
def __init__(
self,
text_field_embedder: TextFieldEmbedder,
vocab: Vocabulary,
seq2vec_encoder: Seq2VecEncoder = None,
dropout: float = None,
regularizer: RegularizerApplicator = None,
):
super().__init__(vocab, regularizer)
if dropout:
self._dropout = torch.nn.Dropout(dropout)
else:
self._dropout = None
self.sym_size = sym_size
self.embeddings = text_field_embedder
self.vec_encoder = seq2vec_encoder
self.hidden_dim = self.vec_encoder.get_output_dim()
self.linear_class = torch.nn.Linear(self.hidden_dim, self.sym_size)
# self.f_linear = torch.nn.Linear(self.hidden_dim * 2, self.hidden_dim * 2)
with open('data/gcn_graph.pk', 'rb') as f:
self.graph = torch.tensor(pickle.load(f)).cuda()
self.topic_acc = Average()
self.topic_rec = Average()
self.topic_f1 = Average()
self.macro_f = MacroF(self.sym_size)
self.turn_acc = Average()
# self.micro_f = FBetaMeasure(beta=1, average='micro')
# self.macro_f = FBetaMeasure(beta=1, average='macro')
self.future_acc = Average()
def __init__(self, path, max_seq_len, name="sts14"):
''' '''
super(STS14Task, self).__init__(name, 1)
self.name = name
self.pair_input = 1
self.categorical = 0
#self.val_metric = "%s_accuracy" % self.name
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer = Average()
self.load_data(path, max_seq_len)
def __init__(
self,
vocab: Vocabulary,
elmo_embedder: TextFieldEmbedder,
tokens_embedder: TextFieldEmbedder,
features_embedder: TextFieldEmbedder,
phrase_layer: Seq2SeqEncoder,
projected_layer: Seq2SeqEncoder,
contextual_passage: Seq2SeqEncoder,
contextual_question: Seq2SeqEncoder,
dropout: float = 0.2,
regularizer: Optional[RegularizerApplicator] = None,
initializer: InitializerApplicator = InitializerApplicator(),
):
super(MultiGranularityHierarchicalAttentionFusionNetworks,
self).__init__(vocab, regularizer)
self.elmo_embedder = elmo_embedder
self.tokens_embedder = tokens_embedder
self.features_embedder = features_embedder
self._phrase_layer = phrase_layer
self._encoding_dim = self._phrase_layer.get_output_dim()
self.projected_layer = torch.nn.Linear(self._encoding_dim + 1024,
self._encoding_dim)
self.fuse_p = FusionLayer(self._encoding_dim)
self.fuse_q = FusionLayer(self._encoding_dim)
self.fuse_s = FusionLayer(self._encoding_dim)
self.projected_lstm = projected_layer
self.contextual_layer_p = contextual_passage
self.contextual_layer_q = contextual_question
self.linear_self_align = torch.nn.Linear(self._encoding_dim, 1)
# self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._self_attention = BilinearMatrixAttention(self._encoding_dim,
self._encoding_dim)
self.bilinear_layer_s = BilinearSeqAtt(self._encoding_dim,
self._encoding_dim)
self.bilinear_layer_e = BilinearSeqAtt(self._encoding_dim,
self._encoding_dim)
self.yesno_predictor = FeedForward(self._encoding_dim,
self._encoding_dim, 3)
self.relu = torch.nn.ReLU()
self._max_span_length = 30
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._squad_metrics = SquadEmAndF1()
self._span_yesno_accuracy = CategoricalAccuracy()
self._official_f1 = Average()
self._variational_dropout = InputVariationalDropout(dropout)
self._loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(self, vocab: Vocabulary):
super().__init__() # type: ignore
self.vocab = vocab
self._nll = Average()
self._ppl = WordPPL()
self._start_index = self.vocab.get_token_index(START_SYMBOL)
self._end_index = self.vocab.get_token_index(END_SYMBOL)
self._pad_index = self.vocab.get_token_index(
self.vocab._padding_token) # noqa: WPS437
self._bleu = BLEU(exclude_indices={
self._pad_index, self._end_index, self._start_index
})
def perplexity(lm_path, sample_file):
import kenlm
model = kenlm.LanguageModel(lm_path)
ppl = Average()
num_lines = file_utils.get_num_lines(sample_file)
with open(sample_file) as sf:
with tqdm(sf, total=num_lines, desc='Computing PPL') as pbar:
for sentence in pbar:
ppl(model.perplexity(sentence))
pbar.set_postfix({'PPL': ppl.get_metric()})
logger.info(f'PPL for file {sample_file} = {ppl.get_metric()}')
def __init__(self,
vocab: Vocabulary,
encoder: VariationalEncoder,
decoder: VariationalDecoder,
generator: Model,
discriminator: Model,
mse_weight: float = 2.0,
train_temperature: float = 1.0,
inference_temperature: float = 1e-5,
num_responses: int = 10) -> None:
super().__init__(vocab)
self._encoder = encoder
self._decoder = decoder
self._mse_weight = mse_weight
self.train_temperature = train_temperature
self.inference_temperature = inference_temperature
self._num_responses = num_responses
self._start_index = self.vocab.get_token_index(START_SYMBOL)
self._end_index = self.vocab.get_token_index(END_SYMBOL)
self._pad_index = self.vocab.get_token_index(self.vocab._padding_token) # pylint: disable=protected-access
self.s_bleu4 = NLTKSentenceBLEU(
n_hyps=self._num_responses,
smoothing_function=SmoothingFunction().method7,
exclude_indices={
self._pad_index, self._end_index, self._start_index
},
prefix='_S_BLEU4')
self.n_bleu2 = NLTKSentenceBLEU(ngram_weights=(1 / 2, 1 / 2),
n_hyps=self._num_responses,
exclude_indices={
self._pad_index, self._end_index,
self._start_index
},
prefix='_BLEU2')
# We need our optimizer to know which parameters came from
# which model, so we cheat by adding tags to them.
for param in generator.parameters():
setattr(param, '_generator', True)
for param in discriminator.parameters():
setattr(param, '_discriminator', True)
self.generator = generator
self.discriminator = discriminator
self._disc_metrics = {
"dfl": Average(),
"dfacc": Average(),
"drl": Average(),
"dracc": Average(),
}
self._gen_metrics = {
"_gl": Average(),
"gce": Average(),
"_gmse": Average(),
"_mean": Average(),
"_stdev": Average()
}
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
num_highway_layers: int,
phrase_layer: Seq2SeqEncoder,
matrix_attention_layer: MatrixAttention,
modeling_layer: Seq2SeqEncoder,
dropout: float = 0.0,
debug: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
) -> None:
super(QANet, self).__init__(vocab=vocab, regularizer=regularizer)
question_encoding_dim = phrase_layer.get_output_dim()
self._text_field_embedder = text_field_embedder
text_embed_dim = self._text_field_embedder.get_output_dim()
encoding_in_dim = phrase_layer.get_input_dim()
encoding_out_dim = phrase_layer.get_output_dim()
modeling_in_dim = modeling_layer.get_input_dim()
modeling_out_dim = modeling_layer.get_output_dim()
self._embedding_proj_layer = torch.nn.Linear(text_embed_dim,
encoding_in_dim)
self._highway_layer = Highway(encoding_in_dim, num_highway_layers)
self._encoding_proj_layer = torch.nn.Linear(encoding_in_dim,
encoding_in_dim)
self._phrase_layer = phrase_layer
self._matrix_attention = matrix_attention_layer
self._modeling_proj_layer = torch.nn.Linear(encoding_out_dim * 4,
modeling_in_dim)
self._modeling_layer = modeling_layer
self._span_start_predictor = torch.nn.Linear(modeling_out_dim * 2, 1)
self._span_end_predictor = torch.nn.Linear(modeling_out_dim * 2, 1)
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self.modelloss_metric = Average()
self._drop_metrics = DropEmAndF1()
initializer(self)
def __init__(self,
vocab: Vocabulary,
utterance_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
input_attention: Attention,
add_action_bias: bool = True,
dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._utterance_embedder = utterance_embedder
self._encoder = encoder
self._max_decoding_steps = max_decoding_steps
self._add_action_bias = add_action_bias
self._dropout = torch.nn.Dropout(p=dropout)
self._exact_match = Average()
self._valid_sql_query = Average()
self._action_similarity = Average()
self._denotation_accuracy = Average()
# the padding value used by IndexField
self._action_padding_index = -1
num_actions = vocab.get_vocab_size("rule_labels")
input_action_dim = action_embedding_dim
if self._add_action_bias:
input_action_dim += 1
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=input_action_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous utterance attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_utterance = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_utterance)
self._beam_search = decoder_beam_search
self._decoder_trainer = MaximumMarginalLikelihood(beam_size=1)
self._transition_function = BasicTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=input_attention,
predict_start_type_separately=False,
add_action_bias=self._add_action_bias,
dropout=dropout)
initializer(self)
class CoLATask(Task):
'''Class for Warstdadt acceptability task'''
def __init__(self, path, max_seq_len, name="acceptability"):
''' '''
super(CoLATask, self).__init__(name, 2)
self.pair_input = 0
self.load_data(path, max_seq_len)
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = CategoricalAccuracy()
def load_data(self, path, max_seq_len):
'''Load the data'''
tr_data = load_tsv(os.path.join(path, "train.tsv"), max_seq_len,
s1_idx=3, s2_idx=None, targ_idx=1)
val_data = load_tsv(os.path.join(path, "dev.tsv"), max_seq_len,
s1_idx=3, s2_idx=None, targ_idx=1)
te_data = load_tsv(os.path.join(path, 'test.tsv'), max_seq_len,
s1_idx=1, s2_idx=None, targ_idx=None, idx_idx=0, skip_rows=1)
self.train_data_text = tr_data
self.val_data_text = val_data
self.test_data_text = te_data
log.info("\tFinished loading CoLA.")
def get_metrics(self, reset=False):
# NB: I think I call it accuracy b/c something weird in training
return {'accuracy': self.scorer1.get_metric(reset),
'acc': self.scorer2.get_metric(reset)}
def __init__(self,
vocab: Vocabulary,
sentence_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
attention: Attention,
beam_size: int,
max_decoding_steps: int,
max_num_finished_states: int = None,
dropout: float = 0.0,
normalize_beam_score_by_length: bool = False,
checklist_cost_weight: float = 0.6,
dynamic_cost_weight: Dict[str, Union[int, float]] = None,
penalize_non_agenda_actions: bool = False,
initial_mml_model_file: str = None) -> None:
super(NlvrCoverageSemanticParser, self).__init__(vocab=vocab,
sentence_embedder=sentence_embedder,
action_embedding_dim=action_embedding_dim,
encoder=encoder,
dropout=dropout)
self._agenda_coverage = Average()
self._decoder_trainer: DecoderTrainer[Callable[[CoverageState], torch.Tensor]] = \
ExpectedRiskMinimization(beam_size=beam_size,
normalize_by_length=normalize_beam_score_by_length,
max_decoding_steps=max_decoding_steps,
max_num_finished_states=max_num_finished_states)
# Instantiating an empty NlvrWorld just to get the number of terminals.
self._terminal_productions = set(NlvrWorld([]).terminal_productions.values())
self._decoder_step = CoverageTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=attention,
num_start_types=1,
activation=Activation.by_name('tanh')(),
predict_start_type_separately=False,
add_action_bias=False,
dropout=dropout)
self._checklist_cost_weight = checklist_cost_weight
self._dynamic_cost_wait_epochs = None
self._dynamic_cost_rate = None
if dynamic_cost_weight:
self._dynamic_cost_wait_epochs = dynamic_cost_weight["wait_num_epochs"]
self._dynamic_cost_rate = dynamic_cost_weight["rate"]
self._penalize_non_agenda_actions = penalize_non_agenda_actions
self._last_epoch_in_forward: int = None
# TODO (pradeep): Checking whether file exists here to avoid raising an error when we've
# copied a trained ERM model from a different machine and the original MML model that was
# used to initialize it does not exist on the current machine. This may not be the best
# solution for the problem.
if initial_mml_model_file is not None:
if os.path.isfile(initial_mml_model_file):
archive = load_archive(initial_mml_model_file)
self._initialize_weights_from_archive(archive)
else:
# A model file is passed, but it does not exist. This is expected to happen when
# you're using a trained ERM model to decode. But it may also happen if the path to
# the file is really just incorrect. So throwing a warning.
logger.warning("MML model file for initializing weights is passed, but does not exist."
" This is fine if you're just decoding.")
def __init__(self, path, max_seq_len, name="acceptability"):
''' '''
super(CoLATask, self).__init__(name, 2)
self.pair_input = 0
self.load_data(path, max_seq_len)
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer1 = Average()
self.scorer2 = CategoricalAccuracy()
def __init__(self, path, max_seq_len, name="sts14"):
''' '''
super(STS14Task, self).__init__(name, 1)
self.name = name
self.pair_input = 1
self.categorical = 0
#self.val_metric = "%s_accuracy" % self.name
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer = Average()
self.load_data(path, max_seq_len)
class WikiTablesErmSemanticParser(WikiTablesSemanticParser):
"""
A ``WikiTablesErmSemanticParser`` is a :class:`WikiTablesSemanticParser` that learns to search
for logical forms that yield the correct denotations.
Parameters
----------
vocab : ``Vocabulary``
question_embedder : ``TextFieldEmbedder``
Embedder for questions. Passed to super class.
action_embedding_dim : ``int``
Dimension to use for action embeddings. Passed to super class.
encoder : ``Seq2SeqEncoder``
The encoder to use for the input question. Passed to super class.
entity_encoder : ``Seq2VecEncoder``
The encoder to used for averaging the words of an entity. Passed to super class.
attention : ``Attention``
We compute an attention over the input question at each step of the decoder, using the
decoder hidden state as the query. Passed to the transition function.
decoder_beam_size : ``int``
Beam size to be used by the ExpectedRiskMinimization algorithm.
decoder_num_finished_states : ``int``
Number of finished states for which costs will be computed by the ExpectedRiskMinimization
algorithm.
max_decoding_steps : ``int``
Maximum number of steps the decoder should take before giving up. Used both during training
and evaluation. Passed to super class.
add_action_bias : ``bool``, optional (default=True)
If ``True``, we will learn a bias weight for each action that gets used when predicting
that action, in addition to its embedding. Passed to super class.
normalize_beam_score_by_length : ``bool``, optional (default=False)
Should we normalize the log-probabilities by length before renormalizing the beam? This was
shown to work better for NML by Edunov et al., but that many not be the case for semantic
parsing.
checklist_cost_weight : ``float``, optional (default=0.6)
Mixture weight (0-1) for combining coverage cost and denotation cost. As this increases, we
weigh the coverage cost higher, with a value of 1.0 meaning that we do not care about
denotation accuracy.
use_neighbor_similarity_for_linking : ``bool``, optional (default=False)
If ``True``, we will compute a max similarity between a question token and the `neighbors`
of an entity as a component of the linking scores. This is meant to capture the same kind
of information as the ``related_column`` feature. Passed to super class.
dropout : ``float``, optional (default=0)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer). Passed to super class.
num_linking_features : ``int``, optional (default=10)
We need to construct a parameter vector for the linking features, so we need to know how
many there are. The default of 10 here matches the default in the ``KnowledgeGraphField``,
which is to use all ten defined features. If this is 0, another term will be added to the
linking score. This term contains the maximum similarity value from the entity's neighbors
and the question. Passed to super class.
rule_namespace : ``str``, optional (default=rule_labels)
The vocabulary namespace to use for production rules. The default corresponds to the
default used in the dataset reader, so you likely don't need to modify this. Passed to super
class.
tables_directory : ``str``, optional (default=/wikitables/)
The directory to find tables when evaluating logical forms. We rely on a call to SEMPRE to
evaluate logical forms, and SEMPRE needs to read the table from disk itself. This tells
SEMPRE where to find the tables. Passed to super class.
mml_model_file : ``str``, optional (default=None)
If you want to initialize this model using weights from another model trained using MML,
pass the path to the ``model.tar.gz`` file of that model here.
"""
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
entity_encoder: Seq2VecEncoder,
attention: Attention,
decoder_beam_size: int,
decoder_num_finished_states: int,
max_decoding_steps: int,
mixture_feedforward: FeedForward = None,
add_action_bias: bool = True,
normalize_beam_score_by_length: bool = False,
checklist_cost_weight: float = 0.6,
use_neighbor_similarity_for_linking: bool = False,
dropout: float = 0.0,
num_linking_features: int = 10,
rule_namespace: str = 'rule_labels',
tables_directory: str = '/wikitables/',
mml_model_file: str = None) -> None:
use_similarity = use_neighbor_similarity_for_linking
super().__init__(vocab=vocab,
question_embedder=question_embedder,
action_embedding_dim=action_embedding_dim,
encoder=encoder,
entity_encoder=entity_encoder,
max_decoding_steps=max_decoding_steps,
add_action_bias=add_action_bias,
use_neighbor_similarity_for_linking=use_similarity,
dropout=dropout,
num_linking_features=num_linking_features,
rule_namespace=rule_namespace,
tables_directory=tables_directory)
# Not sure why mypy needs a type annotation for this!
self._decoder_trainer: ExpectedRiskMinimization = \
ExpectedRiskMinimization(beam_size=decoder_beam_size,
normalize_by_length=normalize_beam_score_by_length,
max_decoding_steps=self._max_decoding_steps,
max_num_finished_states=decoder_num_finished_states)
unlinked_terminals_global_indices = []
global_vocab = self.vocab.get_token_to_index_vocabulary(rule_namespace)
for production, index in global_vocab.items():
right_side = production.split(" -> ")[1]
if right_side in types.COMMON_NAME_MAPPING:
# This is a terminal production.
unlinked_terminals_global_indices.append(index)
self._num_unlinked_terminals = len(unlinked_terminals_global_indices)
self._decoder_step = LinkingCoverageTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=attention,
num_start_types=self._num_start_types,
predict_start_type_separately=True,
add_action_bias=self._add_action_bias,
mixture_feedforward=mixture_feedforward,
dropout=dropout)
self._checklist_cost_weight = checklist_cost_weight
self._agenda_coverage = Average()
# TODO (pradeep): Checking whether file exists here to avoid raising an error when we've
# copied a trained ERM model from a different machine and the original MML model that was
# used to initialize it does not exist on the current machine. This may not be the best
# solution for the problem.
if mml_model_file is not None:
if os.path.isfile(mml_model_file):
archive = load_archive(mml_model_file)
self._initialize_weights_from_archive(archive)
else:
# A model file is passed, but it does not exist. This is expected to happen when
# you're using a trained ERM model to decode. But it may also happen if the path to
# the file is really just incorrect. So throwing a warning.
logger.warning("MML model file for initializing weights is passed, but does not exist."
" This is fine if you're just decoding.")
def _initialize_weights_from_archive(self, archive: Archive) -> None:
logger.info("Initializing weights from MML model.")
model_parameters = dict(self.named_parameters())
archived_parameters = dict(archive.model.named_parameters())
question_embedder_weight = "_question_embedder.token_embedder_tokens.weight"
if question_embedder_weight not in archived_parameters or \
question_embedder_weight not in model_parameters:
raise RuntimeError("When initializing model weights from an MML model, we need "
"the question embedder to be a TokenEmbedder using namespace called "
"tokens.")
for name, weights in archived_parameters.items():
if name in model_parameters:
if name == question_embedder_weight:
# The shapes of embedding weights will most likely differ between the two models
# because the vocabularies will most likely be different. We will get a mapping
# of indices from this model's token indices to the archived model's and copy
# the tensor accordingly.
vocab_index_mapping = self._get_vocab_index_mapping(archive.model.vocab)
archived_embedding_weights = weights.data
new_weights = model_parameters[name].data.clone()
for index, archived_index in vocab_index_mapping:
new_weights[index] = archived_embedding_weights[archived_index]
logger.info("Copied embeddings of %d out of %d tokens",
len(vocab_index_mapping), new_weights.size()[0])
else:
new_weights = weights.data
logger.info("Copying parameter %s", name)
model_parameters[name].data.copy_(new_weights)
def _get_vocab_index_mapping(self, archived_vocab: Vocabulary) -> List[Tuple[int, int]]:
vocab_index_mapping: List[Tuple[int, int]] = []
for index in range(self.vocab.get_vocab_size(namespace='tokens')):
token = self.vocab.get_token_from_index(index=index, namespace='tokens')
archived_token_index = archived_vocab.get_token_index(token, namespace='tokens')
# Checking if we got the UNK token index, because we don't want all new token
# representations initialized to UNK token's representation. We do that by checking if
# the two tokens are the same. They will not be if the token at the archived index is
# UNK.
if archived_vocab.get_token_from_index(archived_token_index, namespace="tokens") == token:
vocab_index_mapping.append((index, archived_token_index))
return vocab_index_mapping
@overrides
def forward(self, # type: ignore
question: Dict[str, torch.LongTensor],
table: Dict[str, torch.LongTensor],
world: List[WikiTablesWorld],
actions: List[List[ProductionRule]],
agenda: torch.LongTensor,
example_lisp_string: List[str],
metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Parameters
----------
question : Dict[str, torch.LongTensor]
The output of ``TextField.as_array()`` applied on the question ``TextField``. This will
be passed through a ``TextFieldEmbedder`` and then through an encoder.
table : ``Dict[str, torch.LongTensor]``
The output of ``KnowledgeGraphField.as_array()`` applied on the table
``KnowledgeGraphField``. This output is similar to a ``TextField`` output, where each
entity in the table is treated as a "token", and we will use a ``TextFieldEmbedder`` to
get embeddings for each entity.
world : ``List[WikiTablesWorld]``
We use a ``MetadataField`` to get the ``World`` for each input instance. Because of
how ``MetadataField`` works, this gets passed to us as a ``List[WikiTablesWorld]``,
actions : ``List[List[ProductionRule]]``
A list of all possible actions for each ``World`` in the batch, indexed into a
``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these
and use the embeddings to determine which action to take at each timestep in the
decoder.
example_lisp_string : ``List[str]``
The example (lisp-formatted) string corresponding to the given input. This comes
directly from the ``.examples`` file provided with the dataset. We pass this to SEMPRE
when evaluating denotation accuracy; it is otherwise unused.
metadata : ``List[Dict[str, Any]]``, optional, (default = None)
Metadata containing the original tokenized question within a 'question_tokens' key.
"""
batch_size = list(question.values())[0].size(0)
# Each instance's agenda is of size (agenda_size, 1)
agenda_list = [agenda[i] for i in range(batch_size)]
checklist_states = []
all_terminal_productions = [set(instance_world.terminal_productions.values())
for instance_world in world]
max_num_terminals = max([len(terminals) for terminals in all_terminal_productions])
for instance_actions, instance_agenda, terminal_productions in zip(actions,
agenda_list,
all_terminal_productions):
checklist_info = self._get_checklist_info(instance_agenda,
instance_actions,
terminal_productions,
max_num_terminals)
checklist_target, terminal_actions, checklist_mask = checklist_info
initial_checklist = checklist_target.new_zeros(checklist_target.size())
checklist_states.append(ChecklistStatelet(terminal_actions=terminal_actions,
checklist_target=checklist_target,
checklist_mask=checklist_mask,
checklist=initial_checklist))
outputs: Dict[str, Any] = {}
rnn_state, grammar_state = self._get_initial_rnn_and_grammar_state(question,
table,
world,
actions,
outputs)
batch_size = len(rnn_state)
initial_score = rnn_state[0].hidden_state.new_zeros(batch_size)
initial_score_list = [initial_score[i] for i in range(batch_size)]
initial_state = CoverageState(batch_indices=list(range(batch_size)), # type: ignore
action_history=[[] for _ in range(batch_size)],
score=initial_score_list,
rnn_state=rnn_state,
grammar_state=grammar_state,
checklist_state=checklist_states,
possible_actions=actions,
extras=example_lisp_string,
debug_info=None)
if not self.training:
initial_state.debug_info = [[] for _ in range(batch_size)]
outputs = self._decoder_trainer.decode(initial_state, # type: ignore
self._decoder_step,
partial(self._get_state_cost, world))
best_final_states = outputs['best_final_states']
if not self.training:
batch_size = len(actions)
agenda_indices = [actions_[:, 0].cpu().data for actions_ in agenda]
action_mapping = {}
for batch_index, batch_actions in enumerate(actions):
for action_index, action in enumerate(batch_actions):
action_mapping[(batch_index, action_index)] = action[0]
for i in range(batch_size):
in_agenda_ratio = 0.0
# Decoding may not have terminated with any completed logical forms, if `num_steps`
# isn't long enough (or if the model is not trained enough and gets into an
# infinite action loop).
if i in best_final_states:
action_sequence = best_final_states[i][0].action_history[0]
action_strings = [action_mapping[(i, action_index)] for action_index in action_sequence]
instance_possible_actions = actions[i]
agenda_actions = []
for rule_id in agenda_indices[i]:
rule_id = int(rule_id)
if rule_id == -1:
continue
action_string = instance_possible_actions[rule_id][0]
agenda_actions.append(action_string)
actions_in_agenda = [action in action_strings for action in agenda_actions]
if actions_in_agenda:
# Note: This means that when there are no actions on agenda, agenda coverage
# will be 0, not 1.
in_agenda_ratio = sum(actions_in_agenda) / len(actions_in_agenda)
self._agenda_coverage(in_agenda_ratio)
self._compute_validation_outputs(actions,
best_final_states,
world,
example_lisp_string,
metadata,
outputs)
return outputs
@staticmethod
def _get_checklist_info(agenda: torch.LongTensor,
all_actions: List[ProductionRule],
terminal_productions: Set[str],
max_num_terminals: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Takes an agenda, a list of all actions, a set of terminal productions in the corresponding
world, and a length to pad the checklist vectors to, and returns a target checklist against
which the checklist at each state will be compared to compute a loss, indices of
``terminal_actions``, and a ``checklist_mask`` that indicates which of the terminal actions
are relevant for checklist loss computation.
Parameters
----------
``agenda`` : ``torch.LongTensor``
Agenda of one instance of size ``(agenda_size, 1)``.
``all_actions`` : ``List[ProductionRule]``
All actions for one instance.
``terminal_productions`` : ``Set[str]``
String representations of terminal productions in the corresponding world.
``max_num_terminals`` : ``int``
Length to which the checklist vectors will be padded till. This is the max number of
terminal productions in all the worlds in the batch.
"""
terminal_indices = []
target_checklist_list = []
agenda_indices_set = set([int(x) for x in agenda.squeeze(0).detach().cpu().numpy()])
# We want to return checklist target and terminal actions that are column vectors to make
# computing softmax over the difference between checklist and target easier.
for index, action in enumerate(all_actions):
# Each action is a ProductionRule, a tuple where the first item is the production
# rule string.
if action[0] in terminal_productions:
terminal_indices.append([index])
if index in agenda_indices_set:
target_checklist_list.append([1])
else:
target_checklist_list.append([0])
while len(target_checklist_list) < max_num_terminals:
target_checklist_list.append([0])
terminal_indices.append([-1])
# (max_num_terminals, 1)
terminal_actions = agenda.new_tensor(terminal_indices)
# (max_num_terminals, 1)
target_checklist = agenda.new_tensor(target_checklist_list, dtype=torch.float)
checklist_mask = (target_checklist != 0).float()
return target_checklist, terminal_actions, checklist_mask
def _get_state_cost(self, worlds: List[WikiTablesWorld], state: CoverageState) -> torch.Tensor:
if not state.is_finished():
raise RuntimeError("_get_state_cost() is not defined for unfinished states!")
world = worlds[state.batch_indices[0]]
# Our checklist cost is a sum of squared error from where we want to be, making sure we
# take into account the mask. We clamp the lower limit of the balance at 0 to avoid
# penalizing agenda actions produced multiple times.
checklist_balance = torch.clamp(state.checklist_state[0].get_balance(), min=0.0)
checklist_cost = torch.sum((checklist_balance) ** 2)
# This is the number of items on the agenda that we want to see in the decoded sequence.
# We use this as the denotation cost if the path is incorrect.
denotation_cost = torch.sum(state.checklist_state[0].checklist_target.float())
checklist_cost = self._checklist_cost_weight * checklist_cost
action_history = state.action_history[0]
batch_index = state.batch_indices[0]
action_strings = [state.possible_actions[batch_index][i][0] for i in action_history]
logical_form = world.get_logical_form(action_strings)
lisp_string = state.extras[batch_index]
if self._executor.evaluate_logical_form(logical_form, lisp_string):
cost = checklist_cost
else:
cost = checklist_cost + (1 - self._checklist_cost_weight) * denotation_cost
return cost
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
"""
The base class returns a dict with dpd accuracy, denotation accuracy, and logical form
percentage metrics. We add the agenda coverage metric here.
"""
metrics = super().get_metrics(reset)
metrics["agenda_coverage"] = self._agenda_coverage.get_metric(reset)
return metrics
class WikiTablesSemanticParser(Model):
"""
A ``WikiTablesSemanticParser`` is a :class:`Model` which takes as input a table and a question,
and produces a logical form that answers the question when executed over the table. The
logical form is generated by a `type-constrained`, `transition-based` parser. This is an
abstract class that defines most of the functionality related to the transition-based parser. It
does not contain the implementation for actually training the parser. You may want to train it
using a learning-to-search algorithm, in which case you will want to use
``WikiTablesErmSemanticParser``, or if you have a set of approximate logical forms that give the
correct denotation, you will want to use ``WikiTablesMmlSemanticParser``.
Parameters
----------
vocab : ``Vocabulary``
question_embedder : ``TextFieldEmbedder``
Embedder for questions.
action_embedding_dim : ``int``
Dimension to use for action embeddings.
encoder : ``Seq2SeqEncoder``
The encoder to use for the input question.
entity_encoder : ``Seq2VecEncoder``
The encoder to used for averaging the words of an entity.
max_decoding_steps : ``int``
When we're decoding with a beam search, what's the maximum number of steps we should take?
This only applies at evaluation time, not during training.
add_action_bias : ``bool``, optional (default=True)
If ``True``, we will learn a bias weight for each action that gets used when predicting
that action, in addition to its embedding.
use_neighbor_similarity_for_linking : ``bool``, optional (default=False)
If ``True``, we will compute a max similarity between a question token and the `neighbors`
of an entity as a component of the linking scores. This is meant to capture the same kind
of information as the ``related_column`` feature.
dropout : ``float``, optional (default=0)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer).
num_linking_features : ``int``, optional (default=10)
We need to construct a parameter vector for the linking features, so we need to know how
many there are. The default of 8 here matches the default in the ``KnowledgeGraphField``,
which is to use all eight defined features. If this is 0, another term will be added to the
linking score. This term contains the maximum similarity value from the entity's neighbors
and the question.
rule_namespace : ``str``, optional (default=rule_labels)
The vocabulary namespace to use for production rules. The default corresponds to the
default used in the dataset reader, so you likely don't need to modify this.
tables_directory : ``str``, optional (default=/wikitables/)
The directory to find tables when evaluating logical forms. We rely on a call to SEMPRE to
evaluate logical forms, and SEMPRE needs to read the table from disk itself. This tells
SEMPRE where to find the tables.
"""
# pylint: disable=abstract-method
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
entity_encoder: Seq2VecEncoder,
max_decoding_steps: int,
add_action_bias: bool = True,
use_neighbor_similarity_for_linking: bool = False,
dropout: float = 0.0,
num_linking_features: int = 10,
rule_namespace: str = 'rule_labels',
tables_directory: str = '/wikitables/') -> None:
super().__init__(vocab)
self._question_embedder = question_embedder
self._encoder = encoder
self._entity_encoder = TimeDistributed(entity_encoder)
self._max_decoding_steps = max_decoding_steps
self._add_action_bias = add_action_bias
self._use_neighbor_similarity_for_linking = use_neighbor_similarity_for_linking
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._executor = WikiTablesSempreExecutor(tables_directory)
self._denotation_accuracy = Average()
self._action_sequence_accuracy = Average()
self._has_logical_form = Average()
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
if self._add_action_bias:
self._action_biases = Embedding(num_embeddings=num_actions, embedding_dim=1)
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous question attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_question = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_question)
check_dimensions_match(entity_encoder.get_output_dim(), question_embedder.get_output_dim(),
"entity word average embedding dim", "question embedding dim")
self._num_entity_types = 4 # TODO(mattg): get this in a more principled way somehow?
self._num_start_types = 5 # TODO(mattg): get this in a more principled way somehow?
self._embedding_dim = question_embedder.get_output_dim()
self._entity_type_encoder_embedding = Embedding(self._num_entity_types, self._embedding_dim)
self._entity_type_decoder_embedding = Embedding(self._num_entity_types, action_embedding_dim)
self._neighbor_params = torch.nn.Linear(self._embedding_dim, self._embedding_dim)
if num_linking_features > 0:
self._linking_params = torch.nn.Linear(num_linking_features, 1)
else:
self._linking_params = None
if self._use_neighbor_similarity_for_linking:
self._question_entity_params = torch.nn.Linear(1, 1)
self._question_neighbor_params = torch.nn.Linear(1, 1)
else:
self._question_entity_params = None
self._question_neighbor_params = None
def _get_initial_rnn_and_grammar_state(self,
question: Dict[str, torch.LongTensor],
table: Dict[str, torch.LongTensor],
world: List[WikiTablesWorld],
actions: List[List[ProductionRule]],
outputs: Dict[str, Any]) -> Tuple[List[RnnStatelet],
List[LambdaGrammarStatelet]]:
"""
Encodes the question and table, computes a linking between the two, and constructs an
initial RnnStatelet and LambdaGrammarStatelet for each batch instance to pass to the
decoder.
We take ``outputs`` as a parameter here and `modify` it, adding things that we want to
visualize in a demo.
"""
table_text = table['text']
# (batch_size, question_length, embedding_dim)
embedded_question = self._question_embedder(question)
question_mask = util.get_text_field_mask(question).float()
# (batch_size, num_entities, num_entity_tokens, embedding_dim)
embedded_table = self._question_embedder(table_text, num_wrapping_dims=1)
table_mask = util.get_text_field_mask(table_text, num_wrapping_dims=1).float()
batch_size, num_entities, num_entity_tokens, _ = embedded_table.size()
num_question_tokens = embedded_question.size(1)
# (batch_size, num_entities, embedding_dim)
encoded_table = self._entity_encoder(embedded_table, table_mask)
# (batch_size, num_entities, num_neighbors)
neighbor_indices = self._get_neighbor_indices(world, num_entities, encoded_table)
# Neighbor_indices is padded with -1 since 0 is a potential neighbor index.
# Thus, the absolute value needs to be taken in the index_select, and 1 needs to
# be added for the mask since that method expects 0 for padding.
# (batch_size, num_entities, num_neighbors, embedding_dim)
embedded_neighbors = util.batched_index_select(encoded_table, torch.abs(neighbor_indices))
neighbor_mask = util.get_text_field_mask({'ignored': neighbor_indices + 1},
num_wrapping_dims=1).float()
# Encoder initialized to easily obtain a masked average.
neighbor_encoder = TimeDistributed(BagOfEmbeddingsEncoder(self._embedding_dim, averaged=True))
# (batch_size, num_entities, embedding_dim)
embedded_neighbors = neighbor_encoder(embedded_neighbors, neighbor_mask)
# entity_types: tensor with shape (batch_size, num_entities), where each entry is the
# entity's type id.
# entity_type_dict: Dict[int, int], mapping flattened_entity_index -> type_index
# These encode the same information, but for efficiency reasons later it's nice
# to have one version as a tensor and one that's accessible on the cpu.
entity_types, entity_type_dict = self._get_type_vector(world, num_entities, encoded_table)
entity_type_embeddings = self._entity_type_encoder_embedding(entity_types)
projected_neighbor_embeddings = self._neighbor_params(embedded_neighbors.float())
# (batch_size, num_entities, embedding_dim)
entity_embeddings = torch.tanh(entity_type_embeddings + projected_neighbor_embeddings)
# Compute entity and question word similarity. We tried using cosine distance here, but
# because this similarity is the main mechanism that the model can use to push apart logit
# scores for certain actions (like "n -> 1" and "n -> -1"), this needs to have a larger
# output range than [-1, 1].
question_entity_similarity = torch.bmm(embedded_table.view(batch_size,
num_entities * num_entity_tokens,
self._embedding_dim),
torch.transpose(embedded_question, 1, 2))
question_entity_similarity = question_entity_similarity.view(batch_size,
num_entities,
num_entity_tokens,
num_question_tokens)
# (batch_size, num_entities, num_question_tokens)
question_entity_similarity_max_score, _ = torch.max(question_entity_similarity, 2)
# (batch_size, num_entities, num_question_tokens, num_features)
linking_features = table['linking']
linking_scores = question_entity_similarity_max_score
if self._use_neighbor_similarity_for_linking:
# The linking score is computed as a linear projection of two terms. The first is the
# maximum similarity score over the entity's words and the question token. The second
# is the maximum similarity over the words in the entity's neighbors and the question
# token.
#
# The second term, projected_question_neighbor_similarity, is useful when a column
# needs to be selected. For example, the question token might have no similarity with
# the column name, but is similar with the cells in the column.
#
# Note that projected_question_neighbor_similarity is intended to capture the same
# information as the related_column feature.
#
# Also note that this block needs to be _before_ the `linking_params` block, because
# we're overwriting `linking_scores`, not adding to it.
# (batch_size, num_entities, num_neighbors, num_question_tokens)
question_neighbor_similarity = util.batched_index_select(question_entity_similarity_max_score,
torch.abs(neighbor_indices))
# (batch_size, num_entities, num_question_tokens)
question_neighbor_similarity_max_score, _ = torch.max(question_neighbor_similarity, 2)
projected_question_entity_similarity = self._question_entity_params(
question_entity_similarity_max_score.unsqueeze(-1)).squeeze(-1)
projected_question_neighbor_similarity = self._question_neighbor_params(
question_neighbor_similarity_max_score.unsqueeze(-1)).squeeze(-1)
linking_scores = projected_question_entity_similarity + projected_question_neighbor_similarity
feature_scores = None
if self._linking_params is not None:
feature_scores = self._linking_params(linking_features).squeeze(3)
linking_scores = linking_scores + feature_scores
# (batch_size, num_question_tokens, num_entities)
linking_probabilities = self._get_linking_probabilities(world, linking_scores.transpose(1, 2),
question_mask, entity_type_dict)
# (batch_size, num_question_tokens, embedding_dim)
link_embedding = util.weighted_sum(entity_embeddings, linking_probabilities)
encoder_input = torch.cat([link_embedding, embedded_question], 2)
# (batch_size, question_length, encoder_output_dim)
encoder_outputs = self._dropout(self._encoder(encoder_input, question_mask))
# This will be our initial hidden state and memory cell for the decoder LSTM.
final_encoder_output = util.get_final_encoder_states(encoder_outputs,
question_mask,
self._encoder.is_bidirectional())
memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder.get_output_dim())
# To make grouping states together in the decoder easier, we convert the batch dimension in
# all of our tensors into an outer list. For instance, the encoder outputs have shape
# `(batch_size, question_length, encoder_output_dim)`. We need to convert this into a list
# of `batch_size` tensors, each of shape `(question_length, encoder_output_dim)`. Then we
# won't have to do any index selects, or anything, we'll just do some `torch.cat()`s.
encoder_output_list = [encoder_outputs[i] for i in range(batch_size)]
question_mask_list = [question_mask[i] for i in range(batch_size)]
initial_rnn_state = []
for i in range(batch_size):
initial_rnn_state.append(RnnStatelet(final_encoder_output[i],
memory_cell[i],
self._first_action_embedding,
self._first_attended_question,
encoder_output_list,
question_mask_list))
initial_grammar_state = [self._create_grammar_state(world[i],
actions[i],
linking_scores[i],
entity_types[i])
for i in range(batch_size)]
if not self.training:
# We add a few things to the outputs that will be returned from `forward` at evaluation
# time, for visualization in a demo.
outputs['linking_scores'] = linking_scores
if feature_scores is not None:
outputs['feature_scores'] = feature_scores
outputs['similarity_scores'] = question_entity_similarity_max_score
return initial_rnn_state, initial_grammar_state
@staticmethod
def _get_neighbor_indices(worlds: List[WikiTablesWorld],
num_entities: int,
tensor: torch.Tensor) -> torch.LongTensor:
"""
This method returns the indices of each entity's neighbors. A tensor
is accepted as a parameter for copying purposes.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
num_entities : ``int``
tensor : ``torch.Tensor``
Used for copying the constructed list onto the right device.
Returns
-------
A ``torch.LongTensor`` with shape ``(batch_size, num_entities, num_neighbors)``. It is padded
with -1 instead of 0, since 0 is a valid neighbor index.
"""
num_neighbors = 0
for world in worlds:
for entity in world.table_graph.entities:
if len(world.table_graph.neighbors[entity]) > num_neighbors:
num_neighbors = len(world.table_graph.neighbors[entity])
batch_neighbors = []
for world in worlds:
# Each batch instance has its own world, which has a corresponding table.
entities = world.table_graph.entities
entity2index = {entity: i for i, entity in enumerate(entities)}
entity2neighbors = world.table_graph.neighbors
neighbor_indexes = []
for entity in entities:
entity_neighbors = [entity2index[n] for n in entity2neighbors[entity]]
# Pad with -1 instead of 0, since 0 represents a neighbor index.
padded = pad_sequence_to_length(entity_neighbors, num_neighbors, lambda: -1)
neighbor_indexes.append(padded)
neighbor_indexes = pad_sequence_to_length(neighbor_indexes,
num_entities,
lambda: [-1] * num_neighbors)
batch_neighbors.append(neighbor_indexes)
return tensor.new_tensor(batch_neighbors, dtype=torch.long)
@staticmethod
def _get_type_vector(worlds: List[WikiTablesWorld],
num_entities: int,
tensor: torch.Tensor) -> Tuple[torch.LongTensor, Dict[int, int]]:
"""
Produces a tensor with shape ``(batch_size, num_entities)`` that encodes each entity's
type. In addition, a map from a flattened entity index to type is returned to combine
entity type operations into one method.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
num_entities : ``int``
tensor : ``torch.Tensor``
Used for copying the constructed list onto the right device.
Returns
-------
A ``torch.LongTensor`` with shape ``(batch_size, num_entities)``.
entity_types : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
"""
entity_types = {}
batch_types = []
for batch_index, world in enumerate(worlds):
types = []
for entity_index, entity in enumerate(world.table_graph.entities):
# We need numbers to be first, then cells, then parts, then row, because our
# entities are going to be sorted. We do a split by type and then a merge later,
# and it relies on this sorting.
if entity.startswith('fb:cell'):
entity_type = 1
elif entity.startswith('fb:part'):
entity_type = 2
elif entity.startswith('fb:row'):
entity_type = 3
else:
entity_type = 0
types.append(entity_type)
# For easier lookups later, we're actually using a _flattened_ version
# of (batch_index, entity_index) for the key, because this is how the
# linking scores are stored.
flattened_entity_index = batch_index * num_entities + entity_index
entity_types[flattened_entity_index] = entity_type
padded = pad_sequence_to_length(types, num_entities, lambda: 0)
batch_types.append(padded)
return tensor.new_tensor(batch_types, dtype=torch.long), entity_types
def _get_linking_probabilities(self,
worlds: List[WikiTablesWorld],
linking_scores: torch.FloatTensor,
question_mask: torch.LongTensor,
entity_type_dict: Dict[int, int]) -> torch.FloatTensor:
"""
Produces the probability of an entity given a question word and type. The logic below
separates the entities by type since the softmax normalization term sums over entities
of a single type.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
linking_scores : ``torch.FloatTensor``
Has shape (batch_size, num_question_tokens, num_entities).
question_mask: ``torch.LongTensor``
Has shape (batch_size, num_question_tokens).
entity_type_dict : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
Returns
-------
batch_probabilities : ``torch.FloatTensor``
Has shape ``(batch_size, num_question_tokens, num_entities)``.
Contains all the probabilities for an entity given a question word.
"""
_, num_question_tokens, num_entities = linking_scores.size()
batch_probabilities = []
for batch_index, world in enumerate(worlds):
all_probabilities = []
num_entities_in_instance = 0
# NOTE: The way that we're doing this here relies on the fact that entities are
# implicitly sorted by their types when we sort them by name, and that numbers come
# before "fb:cell", and "fb:cell" comes before "fb:row". This is not a great
# assumption, and could easily break later, but it should work for now.
for type_index in range(self._num_entity_types):
# This index of 0 is for the null entity for each type, representing the case where a
# word doesn't link to any entity.
entity_indices = [0]
entities = world.table_graph.entities
for entity_index, _ in enumerate(entities):
if entity_type_dict[batch_index * num_entities + entity_index] == type_index:
entity_indices.append(entity_index)
if len(entity_indices) == 1:
# No entities of this type; move along...
continue
# We're subtracting one here because of the null entity we added above.
num_entities_in_instance += len(entity_indices) - 1
# We separate the scores by type, since normalization is done per type. There's an
# extra "null" entity per type, also, so we have `num_entities_per_type + 1`. We're
# selecting from a (num_question_tokens, num_entities) linking tensor on _dimension 1_,
# so we get back something of shape (num_question_tokens,) for each index we're
# selecting. All of the selected indices together then make a tensor of shape
# (num_question_tokens, num_entities_per_type + 1).
indices = linking_scores.new_tensor(entity_indices, dtype=torch.long)
entity_scores = linking_scores[batch_index].index_select(1, indices)
# We used index 0 for the null entity, so this will actually have some values in it.
# But we want the null entity's score to be 0, so we set that here.
entity_scores[:, 0] = 0
# No need for a mask here, as this is done per batch instance, with no padding.
type_probabilities = torch.nn.functional.softmax(entity_scores, dim=1)
all_probabilities.append(type_probabilities[:, 1:])
# We need to add padding here if we don't have the right number of entities.
if num_entities_in_instance != num_entities:
zeros = linking_scores.new_zeros(num_question_tokens,
num_entities - num_entities_in_instance)
all_probabilities.append(zeros)
# (num_question_tokens, num_entities)
probabilities = torch.cat(all_probabilities, dim=1)
batch_probabilities.append(probabilities)
batch_probabilities = torch.stack(batch_probabilities, dim=0)
return batch_probabilities * question_mask.unsqueeze(-1).float()
@staticmethod
def _action_history_match(predicted: List[int], targets: torch.LongTensor) -> int:
# TODO(mattg): this could probably be moved into a FullSequenceMatch metric, or something.
# Check if target is big enough to cover prediction (including start/end symbols)
if len(predicted) > targets.size(1):
return 0
predicted_tensor = targets.new_tensor(predicted)
targets_trimmed = targets[:, :len(predicted)]
# Return 1 if the predicted sequence is anywhere in the list of targets.
return torch.max(torch.min(targets_trimmed.eq(predicted_tensor), dim=1)[0]).item()
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
"""
We track three metrics here:
1. dpd_acc, which is the percentage of the time that our best output action sequence is
in the set of action sequences provided by DPD. This is an easy-to-compute lower bound
on denotation accuracy for the set of examples where we actually have DPD output. We
only score dpd_acc on that subset.
2. denotation_acc, which is the percentage of examples where we get the correct
denotation. This is the typical "accuracy" metric, and it is what you should usually
report in an experimental result. You need to be careful, though, that you're
computing this on the full data, and not just the subset that has DPD output (make sure
you pass "keep_if_no_dpd=True" to the dataset reader, which we do for validation data,
but not training data).
3. lf_percent, which is the percentage of time that decoding actually produces a
finished logical form. We might not produce a valid logical form if the decoder gets
into a repetitive loop, or we're trying to produce a super long logical form and run
out of time steps, or something.
"""
return {
'dpd_acc': self._action_sequence_accuracy.get_metric(reset),
'denotation_acc': self._denotation_accuracy.get_metric(reset),
'lf_percent': self._has_logical_form.get_metric(reset),
}
def _create_grammar_state(self,
world: WikiTablesWorld,
possible_actions: List[ProductionRule],
linking_scores: torch.Tensor,
entity_types: torch.Tensor) -> LambdaGrammarStatelet:
"""
This method creates the LambdaGrammarStatelet object that's used for decoding. Part of
creating that is creating the `valid_actions` dictionary, which contains embedded
representations of all of the valid actions. So, we create that here as well.
The way we represent the valid expansions is a little complicated: we use a
dictionary of `action types`, where the key is the action type (like "global", "linked", or
whatever your model is expecting), and the value is a tuple representing all actions of
that type. The tuple is (input tensor, output tensor, action id). The input tensor has
the representation that is used when `selecting` actions, for all actions of this type.
The output tensor has the representation that is used when feeding the action to the next
step of the decoder (this could just be the same as the input tensor). The action ids are
a list of indices into the main action list for each batch instance.
The inputs to this method are for a `single instance in the batch`; none of the tensors we
create here are batched. We grab the global action ids from the input
``ProductionRules``, and we use those to embed the valid actions for every
non-terminal type. We use the input ``linking_scores`` for non-global actions.
Parameters
----------
world : ``WikiTablesWorld``
From the input to ``forward`` for a single batch instance.
possible_actions : ``List[ProductionRule]``
From the input to ``forward`` for a single batch instance.
linking_scores : ``torch.Tensor``
Assumed to have shape ``(num_entities, num_question_tokens)`` (i.e., there is no batch
dimension).
entity_types : ``torch.Tensor``
Assumed to have shape ``(num_entities,)`` (i.e., there is no batch dimension).
"""
# TODO(mattg): Move the "valid_actions" construction to another method.
action_map = {}
for action_index, action in enumerate(possible_actions):
action_string = action[0]
action_map[action_string] = action_index
entity_map = {}
for entity_index, entity in enumerate(world.table_graph.entities):
entity_map[entity] = entity_index
valid_actions = world.get_valid_actions()
translated_valid_actions: Dict[str, Dict[str, Tuple[torch.Tensor, torch.Tensor, List[int]]]] = {}
for key, action_strings in valid_actions.items():
translated_valid_actions[key] = {}
# `key` here is a non-terminal from the grammar, and `action_strings` are all the valid
# productions of that non-terminal. We'll first split those productions by global vs.
# linked action.
action_indices = [action_map[action_string] for action_string in action_strings]
production_rule_arrays = [(possible_actions[index], index) for index in action_indices]
global_actions = []
linked_actions = []
for production_rule_array, action_index in production_rule_arrays:
if production_rule_array[1]:
global_actions.append((production_rule_array[2], action_index))
else:
linked_actions.append((production_rule_array[0], action_index))
# Then we get the embedded representations of the global actions.
global_action_tensors, global_action_ids = zip(*global_actions)
global_action_tensor = torch.cat(global_action_tensors, dim=0)
global_input_embeddings = self._action_embedder(global_action_tensor)
if self._add_action_bias:
global_action_biases = self._action_biases(global_action_tensor)
global_input_embeddings = torch.cat([global_input_embeddings, global_action_biases], dim=-1)
global_output_embeddings = self._output_action_embedder(global_action_tensor)
translated_valid_actions[key]['global'] = (global_input_embeddings,
global_output_embeddings,
list(global_action_ids))
# Then the representations of the linked actions.
if linked_actions:
linked_rules, linked_action_ids = zip(*linked_actions)
entities = [rule.split(' -> ')[1] for rule in linked_rules]
entity_ids = [entity_map[entity] for entity in entities]
# (num_linked_actions, num_question_tokens)
entity_linking_scores = linking_scores[entity_ids]
# (num_linked_actions,)
entity_type_tensor = entity_types[entity_ids]
# (num_linked_actions, entity_type_embedding_dim)
entity_type_embeddings = self._entity_type_decoder_embedding(entity_type_tensor)
translated_valid_actions[key]['linked'] = (entity_linking_scores,
entity_type_embeddings,
list(linked_action_ids))
# Lastly, we need to also create embedded representations of context-specific actions. In
# this case, those are only variable productions, like "r -> x". Note that our language
# only permits one lambda at a time, so we don't need to worry about how nested lambdas
# might impact this.
context_actions = {}
for action_id, action in enumerate(possible_actions):
if action[0].endswith(" -> x"):
input_embedding = self._action_embedder(action[2])
if self._add_action_bias:
input_bias = self._action_biases(action[2])
input_embedding = torch.cat([input_embedding, input_bias], dim=-1)
output_embedding = self._output_action_embedder(action[2])
context_actions[action[0]] = (input_embedding, output_embedding, action_id)
return LambdaGrammarStatelet([START_SYMBOL],
{},
translated_valid_actions,
context_actions,
type_declaration.is_nonterminal)
def _compute_validation_outputs(self,
actions: List[List[ProductionRule]],
best_final_states: Mapping[int, Sequence[GrammarBasedState]],
world: List[WikiTablesWorld],
example_lisp_string: List[str],
metadata: List[Dict[str, Any]],
outputs: Dict[str, Any]) -> None:
"""
Does common things for validation time: computing logical form accuracy (which is expensive
and unnecessary during training), adding visualization info to the output dictionary, etc.
This doesn't return anything; instead it `modifies` the given ``outputs`` dictionary, and
calls metrics on ``self``.
"""
batch_size = len(actions)
action_mapping = {}
for batch_index, batch_actions in enumerate(actions):
for action_index, action in enumerate(batch_actions):
action_mapping[(batch_index, action_index)] = action[0]
outputs['action_mapping'] = action_mapping
outputs['best_action_sequence'] = []
outputs['debug_info'] = []
outputs['entities'] = []
outputs['logical_form'] = []
for i in range(batch_size):
# Decoding may not have terminated with any completed logical forms, if `num_steps`
# isn't long enough (or if the model is not trained enough and gets into an
# infinite action loop).
if i in best_final_states:
best_action_indices = best_final_states[i][0].action_history[0]
action_strings = [action_mapping[(i, action_index)] for action_index in best_action_indices]
try:
logical_form = world[i].get_logical_form(action_strings, add_var_function=False)
self._has_logical_form(1.0)
except ParsingError:
self._has_logical_form(0.0)
logical_form = 'Error producing logical form'
if example_lisp_string:
denotation_correct = self._executor.evaluate_logical_form(logical_form,
example_lisp_string[i])
self._denotation_accuracy(1.0 if denotation_correct else 0.0)
outputs['best_action_sequence'].append(action_strings)
outputs['logical_form'].append(logical_form)
outputs['debug_info'].append(best_final_states[i][0].debug_info[0]) # type: ignore
outputs['entities'].append(world[i].table_graph.entities)
else:
outputs['logical_form'].append('')
self._has_logical_form(0.0)
self._denotation_accuracy(0.0)
if metadata is not None:
outputs["question_tokens"] = [x["question_tokens"] for x in metadata]
outputs["original_table"] = [x["original_table"] for x in metadata]
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
This method overrides ``Model.decode``, which gets called after ``Model.forward``, at test
time, to finalize predictions. This is (confusingly) a separate notion from the "decoder"
in "encoder/decoder", where that decoder logic lives in the ``TransitionFunction``.
This method trims the output predictions to the first end symbol, replaces indices with
corresponding tokens, and adds a field called ``predicted_tokens`` to the ``output_dict``.
"""
action_mapping = output_dict['action_mapping']
best_actions = output_dict["best_action_sequence"]
debug_infos = output_dict['debug_info']
batch_action_info = []
for batch_index, (predicted_actions, debug_info) in enumerate(zip(best_actions, debug_infos)):
instance_action_info = []
for predicted_action, action_debug_info in zip(predicted_actions, debug_info):
action_info = {}
action_info['predicted_action'] = predicted_action
considered_actions = action_debug_info['considered_actions']
probabilities = action_debug_info['probabilities']
actions = []
for action, probability in zip(considered_actions, probabilities):
if action != -1:
actions.append((action_mapping[(batch_index, action)], probability))
actions.sort()
considered_actions, probabilities = zip(*actions)
action_info['considered_actions'] = considered_actions
action_info['action_probabilities'] = probabilities
action_info['question_attention'] = action_debug_info.get('question_attention', [])
instance_action_info.append(action_info)
batch_action_info.append(instance_action_info)
output_dict["predicted_actions"] = batch_action_info
return output_dict
class NlvrCoverageSemanticParser(NlvrSemanticParser):
"""
``NlvrSemanticCoverageParser`` is an ``NlvrSemanticParser`` that gets around the problem of lack
of annotated logical forms by maximizing coverage of the output sequences over a prespecified
agenda. In addition to the signal from coverage, we also compute the denotations given by the
logical forms and define a hybrid cost based on coverage and denotation errors. The training
process then minimizes the expected value of this cost over an approximate set of logical forms
produced by the parser, obtained by performing beam search.
Parameters
----------
vocab : ``Vocabulary``
Passed to super-class.
sentence_embedder : ``TextFieldEmbedder``
Passed to super-class.
action_embedding_dim : ``int``
Passed to super-class.
encoder : ``Seq2SeqEncoder``
Passed to super-class.
attention : ``Attention``
We compute an attention over the input question at each step of the decoder, using the
decoder hidden state as the query. Passed to the DecoderStep.
beam_size : ``int``
Beam size for the beam search used during training.
max_num_finished_states : ``int``, optional (default=None)
Maximum number of finished states the trainer should compute costs for.
normalize_beam_score_by_length : ``bool``, optional (default=False)
Should the log probabilities be normalized by length before renormalizing them? Edunov et
al. do this in their work, but we found that not doing it works better. It's possible they
did this because their task is NMT, and longer decoded sequences are not necessarily worse,
and shouldn't be penalized, while we will mostly want to penalize longer logical forms.
max_decoding_steps : ``int``
Maximum number of steps for the beam search during training.
dropout : ``float``, optional (default=0.0)
Probability of dropout to apply on encoder outputs, decoder outputs and predicted actions.
checklist_cost_weight : ``float``, optional (default=0.6)
Mixture weight (0-1) for combining coverage cost and denotation cost. As this increases, we
weigh the coverage cost higher, with a value of 1.0 meaning that we do not care about
denotation accuracy.
dynamic_cost_weight : ``Dict[str, Union[int, float]]``, optional (default=None)
A dict containing keys ``wait_num_epochs`` and ``rate`` indicating the number of steps
after which we should start decreasing the weight on checklist cost in favor of denotation
cost, and the rate at which we should do it. We will decrease the weight in the following
way - ``checklist_cost_weight = checklist_cost_weight - rate * checklist_cost_weight``
starting at the apropriate epoch. The weight will remain constant if this is not provided.
penalize_non_agenda_actions : ``bool``, optional (default=False)
Should we penalize the model for producing terminal actions that are outside the agenda?
initial_mml_model_file : ``str`` , optional (default=None)
If you want to initialize this model using weights from another model trained using MML,
pass the path to the ``model.tar.gz`` file of that model here.
"""
def __init__(self,
vocab: Vocabulary,
sentence_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
attention: Attention,
beam_size: int,
max_decoding_steps: int,
max_num_finished_states: int = None,
dropout: float = 0.0,
normalize_beam_score_by_length: bool = False,
checklist_cost_weight: float = 0.6,
dynamic_cost_weight: Dict[str, Union[int, float]] = None,
penalize_non_agenda_actions: bool = False,
initial_mml_model_file: str = None) -> None:
super(NlvrCoverageSemanticParser, self).__init__(vocab=vocab,
sentence_embedder=sentence_embedder,
action_embedding_dim=action_embedding_dim,
encoder=encoder,
dropout=dropout)
self._agenda_coverage = Average()
self._decoder_trainer: DecoderTrainer[Callable[[NlvrDecoderState], torch.Tensor]] = \
ExpectedRiskMinimization(beam_size=beam_size,
normalize_by_length=normalize_beam_score_by_length,
max_decoding_steps=max_decoding_steps,
max_num_finished_states=max_num_finished_states)
# Instantiating an empty NlvrWorld just to get the number of terminals.
self._terminal_productions = set(NlvrWorld([]).terminal_productions.values())
self._decoder_step = NlvrDecoderStep(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=attention,
dropout=dropout,
use_coverage=True)
self._checklist_cost_weight = checklist_cost_weight
self._dynamic_cost_wait_epochs = None
self._dynamic_cost_rate = None
if dynamic_cost_weight:
self._dynamic_cost_wait_epochs = dynamic_cost_weight["wait_num_epochs"]
self._dynamic_cost_rate = dynamic_cost_weight["rate"]
self._penalize_non_agenda_actions = penalize_non_agenda_actions
self._last_epoch_in_forward: int = None
# TODO (pradeep): Checking whether file exists here to avoid raising an error when we've
# copied a trained ERM model from a different machine and the original MML model that was
# used to initialize it does not exist on the current machine. This may not be the best
# solution for the problem.
if initial_mml_model_file is not None:
if os.path.isfile(initial_mml_model_file):
archive = load_archive(initial_mml_model_file)
self._initialize_weights_from_archive(archive)
else:
# A model file is passed, but it does not exist. This is expected to happen when
# you're using a trained ERM model to decode. But it may also happen if the path to
# the file is really just incorrect. So throwing a warning.
logger.warning("MML model file for initializing weights is passed, but does not exist."
" This is fine if you're just decoding.")
def _initialize_weights_from_archive(self, archive: Archive) -> None:
logger.info("Initializing weights from MML model.")
model_parameters = dict(self.named_parameters())
archived_parameters = dict(archive.model.named_parameters())
sentence_embedder_weight = "_sentence_embedder.token_embedder_tokens.weight"
if sentence_embedder_weight not in archived_parameters or \
sentence_embedder_weight not in model_parameters:
raise RuntimeError("When initializing model weights from an MML model, we need "
"the sentence embedder to be a TokenEmbedder using namespace called "
"tokens.")
for name, weights in archived_parameters.items():
if name in model_parameters:
if name == "_sentence_embedder.token_embedder_tokens.weight":
# The shapes of embedding weights will most likely differ between the two models
# because the vocabularies will most likely be different. We will get a mapping
# of indices from this model's token indices to the archived model's and copy
# the tensor accordingly.
vocab_index_mapping = self._get_vocab_index_mapping(archive.model.vocab)
archived_embedding_weights = weights.data
new_weights = model_parameters[name].data.clone()
for index, archived_index in vocab_index_mapping:
new_weights[index] = archived_embedding_weights[archived_index]
logger.info("Copied embeddings of %d out of %d tokens",
len(vocab_index_mapping), new_weights.size()[0])
else:
new_weights = weights.data
logger.info("Copying parameter %s", name)
model_parameters[name].data.copy_(new_weights)
def _get_vocab_index_mapping(self, archived_vocab: Vocabulary) -> List[Tuple[int, int]]:
vocab_index_mapping: List[Tuple[int, int]] = []
for index in range(self.vocab.get_vocab_size(namespace='tokens')):
token = self.vocab.get_token_from_index(index=index, namespace='tokens')
archived_token_index = archived_vocab.get_token_index(token, namespace='tokens')
# Checking if we got the UNK token index, because we don't want all new token
# representations initialized to UNK token's representation. We do that by checking if
# the two tokens are the same. They will not be if the token at the archived index is
# UNK.
if archived_vocab.get_token_from_index(archived_token_index, namespace="tokens") == token:
vocab_index_mapping.append((index, archived_token_index))
return vocab_index_mapping
@overrides
def forward(self, # type: ignore
sentence: Dict[str, torch.LongTensor],
worlds: List[List[NlvrWorld]],
actions: List[List[ProductionRuleArray]],
agenda: torch.LongTensor,
identifier: List[str] = None,
labels: torch.LongTensor = None,
epoch_num: List[int] = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Decoder logic for producing type constrained target sequences that maximize coverage of
their respective agendas, and minimize a denotation based loss.
"""
# We look at the epoch number and adjust the checklist cost weight if needed here.
instance_epoch_num = epoch_num[0] if epoch_num is not None else None
if self._dynamic_cost_rate is not None:
if self.training and instance_epoch_num is None:
raise RuntimeError("If you want a dynamic cost weight, use the "
"EpochTrackingBucketIterator!")
if instance_epoch_num != self._last_epoch_in_forward:
if instance_epoch_num >= self._dynamic_cost_wait_epochs:
decrement = self._checklist_cost_weight * self._dynamic_cost_rate
self._checklist_cost_weight -= decrement
logger.info("Checklist cost weight is now %f", self._checklist_cost_weight)
self._last_epoch_in_forward = instance_epoch_num
batch_size = len(worlds)
action_embeddings, action_indices = self._embed_actions(actions)
initial_rnn_state = self._get_initial_rnn_state(sentence)
initial_score_list = [next(iter(sentence.values())).new_zeros(1, dtype=torch.float)
for i in range(batch_size)]
# TODO (pradeep): Assuming all worlds give the same set of valid actions.
initial_grammar_state = [self._create_grammar_state(worlds[i][0], actions[i]) for i in
range(batch_size)]
label_strings = self._get_label_strings(labels) if labels is not None else None
# Each instance's agenda is of size (agenda_size, 1)
agenda_list = [agenda[i] for i in range(batch_size)]
initial_checklist_states = []
for instance_actions, instance_agenda in zip(actions, agenda_list):
checklist_info = self._get_checklist_info(instance_agenda, instance_actions)
checklist_target, terminal_actions, checklist_mask = checklist_info
initial_checklist = checklist_target.new_zeros(checklist_target.size())
initial_checklist_states.append(ChecklistState(terminal_actions=terminal_actions,
checklist_target=checklist_target,
checklist_mask=checklist_mask,
checklist=initial_checklist))
initial_state = NlvrDecoderState(batch_indices=list(range(batch_size)),
action_history=[[] for _ in range(batch_size)],
score=initial_score_list,
rnn_state=initial_rnn_state,
grammar_state=initial_grammar_state,
action_embeddings=action_embeddings,
action_indices=action_indices,
possible_actions=actions,
worlds=worlds,
label_strings=label_strings,
checklist_state=initial_checklist_states)
agenda_data = [agenda_[:, 0].cpu().data for agenda_ in agenda_list]
outputs = self._decoder_trainer.decode(initial_state,
self._decoder_step,
self._get_state_cost)
if identifier is not None:
outputs['identifier'] = identifier
best_action_sequences = outputs['best_action_sequences']
batch_action_strings = self._get_action_strings(actions, best_action_sequences)
batch_denotations = self._get_denotations(batch_action_strings, worlds)
if labels is not None:
# We're either training or validating.
self._update_metrics(action_strings=batch_action_strings,
worlds=worlds,
label_strings=label_strings,
possible_actions=actions,
agenda_data=agenda_data)
else:
# We're testing.
outputs["best_action_strings"] = batch_action_strings
outputs["denotations"] = batch_denotations
return outputs
def _get_checklist_info(self,
agenda: torch.LongTensor,
all_actions: List[ProductionRuleArray]) -> Tuple[torch.Tensor,
torch.Tensor,
torch.Tensor]:
"""
Takes an agenda and a list of all actions and returns a target checklist against which the
checklist at each state will be compared to compute a loss, indices of ``terminal_actions``,
and a ``checklist_mask`` that indicates which of the terminal actions are relevant for
checklist loss computation. If ``self.penalize_non_agenda_actions`` is set to``True``,
``checklist_mask`` will be all 1s (i.e., all terminal actions are relevant). If it is set to
``False``, indices of all terminals that are not in the agenda will be masked.
Parameters
----------
``agenda`` : ``torch.LongTensor``
Agenda of one instance of size ``(agenda_size, 1)``.
``all_actions`` : ``List[ProductionRuleArray]``
All actions for one instance.
"""
terminal_indices = []
target_checklist_list = []
agenda_indices_set = set([int(x) for x in agenda.squeeze(0).detach().cpu().numpy()])
for index, action in enumerate(all_actions):
# Each action is a ProductionRuleArray, a tuple where the first item is the production
# rule string.
if action[0] in self._terminal_productions:
terminal_indices.append([index])
if index in agenda_indices_set:
target_checklist_list.append([1])
else:
target_checklist_list.append([0])
# We want to return checklist target and terminal actions that are column vectors to make
# computing softmax over the difference between checklist and target easier.
# (num_terminals, 1)
terminal_actions = agenda.new_tensor(terminal_indices)
# (num_terminals, 1)
target_checklist = agenda.new_tensor(target_checklist_list, dtype=torch.float)
if self._penalize_non_agenda_actions:
# All terminal actions are relevant
checklist_mask = torch.ones_like(target_checklist)
else:
checklist_mask = (target_checklist != 0).float()
return target_checklist, terminal_actions, checklist_mask
def _update_metrics(self,
action_strings: List[List[List[str]]],
worlds: List[List[NlvrWorld]],
label_strings: List[List[str]],
possible_actions: List[List[ProductionRuleArray]],
agenda_data: List[List[int]]) -> None:
# TODO(pradeep): Move this to the base class.
# TODO(pradeep): action_strings contains k-best lists. This method only uses the top decoded
# sequence currently. Maybe define top-k metrics?
batch_size = len(worlds)
for i in range(batch_size):
# Using only the top decoded sequence per instance.
instance_action_strings = action_strings[i][0] if action_strings[i] else []
sequence_is_correct = [False]
in_agenda_ratio = 0.0
instance_possible_actions = possible_actions[i]
if instance_action_strings:
terminal_agenda_actions = []
for rule_id in agenda_data[i]:
if rule_id == -1:
continue
action_string = instance_possible_actions[rule_id][0]
right_side = action_string.split(" -> ")[1]
if right_side.isdigit() or ('[' not in right_side and len(right_side) > 1):
terminal_agenda_actions.append(action_string)
actions_in_agenda = [action in instance_action_strings for action in
terminal_agenda_actions]
in_agenda_ratio = sum(actions_in_agenda) / len(actions_in_agenda)
instance_label_strings = label_strings[i]
instance_worlds = worlds[i]
sequence_is_correct = self._check_denotation(instance_action_strings,
instance_label_strings,
instance_worlds)
for correct_in_world in sequence_is_correct:
self._denotation_accuracy(1 if correct_in_world else 0)
self._consistency(1 if all(sequence_is_correct) else 0)
self._agenda_coverage(in_agenda_ratio)
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
return {
'denotation_accuracy': self._denotation_accuracy.get_metric(reset),
'consistency': self._consistency.get_metric(reset),
'agenda_coverage': self._agenda_coverage.get_metric(reset)
}
def _get_state_cost(self, state: NlvrDecoderState) -> torch.Tensor:
"""
Return the costs a finished state. Since it is a finished state, the group size will be 1,
and hence we'll return just one cost.
"""
if not state.is_finished():
raise RuntimeError("_get_state_cost() is not defined for unfinished states!")
# Our checklist cost is a sum of squared error from where we want to be, making sure we
# take into account the mask.
checklist_balance = state.checklist_state[0].get_balance()
checklist_cost = torch.sum((checklist_balance) ** 2)
# This is the number of items on the agenda that we want to see in the decoded sequence.
# We use this as the denotation cost if the path is incorrect.
# Note: If we are penalizing the model for producing non-agenda actions, this is not the
# upper limit on the checklist cost. That would be the number of terminal actions.
denotation_cost = torch.sum(state.checklist_state[0].checklist_target.float())
checklist_cost = self._checklist_cost_weight * checklist_cost
# TODO (pradeep): The denotation based cost below is strict. May be define a cost based on
# how many worlds the logical form is correct in?
# label_strings being None happens when we are testing. We do not care about the cost then.
# TODO (pradeep): Make this cleaner.
if state.label_strings is None or all(self._check_state_denotations(state)):
cost = checklist_cost
else:
cost = checklist_cost + (1 - self._checklist_cost_weight) * denotation_cost
return cost
def _get_state_info(self, state) -> Dict[str, List]:
"""
This method is here for debugging purposes, in case you want to look at the what the model
is learning. It may be inefficient to call it while training the model on real data.
"""
if len(state.batch_indices) == 1 and state.is_finished():
costs = [float(self._get_state_cost(state).detach().cpu().numpy())]
else:
costs = []
model_scores = [float(score.detach().cpu().numpy()) for score in state.score]
all_actions = state.possible_actions[0]
action_sequences = [[self._get_action_string(all_actions[action]) for action in history]
for history in state.action_history]
agenda_sequences = []
all_agenda_indices = []
for agenda, checklist_target in zip(state.terminal_actions, state.checklist_target):
agenda_indices = []
for action, is_wanted in zip(agenda, checklist_target):
action_int = int(action.detach().cpu().numpy())
is_wanted_int = int(is_wanted.detach().cpu().numpy())
if is_wanted_int != 0:
agenda_indices.append(action_int)
agenda_sequences.append([self._get_action_string(all_actions[action])
for action in agenda_indices])
all_agenda_indices.append(agenda_indices)
return {"agenda": agenda_sequences,
"agenda_indices": all_agenda_indices,
"history": action_sequences,
"history_indices": state.action_history,
"costs": costs,
"scores": model_scores}
class Text2SqlParser(Model):
"""
Parameters
----------
vocab : ``Vocabulary``
utterance_embedder : ``TextFieldEmbedder``
Embedder for utterances.
action_embedding_dim : ``int``
Dimension to use for action embeddings.
encoder : ``Seq2SeqEncoder``
The encoder to use for the input utterance.
decoder_beam_search : ``BeamSearch``
Beam search used to retrieve best sequences after training.
max_decoding_steps : ``int``
When we're decoding with a beam search, what's the maximum number of steps we should take?
This only applies at evaluation time, not during training.
input_attention: ``Attention``
We compute an attention over the input utterance at each step of the decoder, using the
decoder hidden state as the query. Passed to the transition function.
add_action_bias : ``bool``, optional (default=True)
If ``True``, we will learn a bias weight for each action that gets used when predicting
that action, in addition to its embedding.
dropout : ``float``, optional (default=0)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer).
"""
def __init__(self,
vocab: Vocabulary,
utterance_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
input_attention: Attention,
add_action_bias: bool = True,
dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self._utterance_embedder = utterance_embedder
self._encoder = encoder
self._max_decoding_steps = max_decoding_steps
self._add_action_bias = add_action_bias
self._dropout = torch.nn.Dropout(p=dropout)
self._exact_match = Average()
self._valid_sql_query = Average()
self._action_similarity = Average()
self._denotation_accuracy = Average()
# the padding value used by IndexField
self._action_padding_index = -1
num_actions = vocab.get_vocab_size("rule_labels")
input_action_dim = action_embedding_dim
if self._add_action_bias:
input_action_dim += 1
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=input_action_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous utterance attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_utterance = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_utterance)
self._beam_search = decoder_beam_search
self._decoder_trainer = MaximumMarginalLikelihood(beam_size=1)
self._transition_function = BasicTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=input_attention,
predict_start_type_separately=False,
add_action_bias=self._add_action_bias,
dropout=dropout)
initializer(self)
@overrides
def forward(self, # type: ignore
tokens: Dict[str, torch.LongTensor],
valid_actions: List[List[ProductionRule]],
action_sequence: torch.LongTensor = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer,
if we're training, or a BeamSearch for inference, if we're not.
Parameters
----------
tokens : Dict[str, torch.LongTensor]
The output of ``TextField.as_array()`` applied on the tokens ``TextField``. This will
be passed through a ``TextFieldEmbedder`` and then through an encoder.
valid_actions : ``List[List[ProductionRule]]``
A list of all possible actions for each ``World`` in the batch, indexed into a
``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these
and use the embeddings to determine which action to take at each timestep in the
decoder.
target_action_sequence : torch.Tensor, optional (default=None)
The action sequence for the correct action sequence, where each action is an index into the list
of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the
trailing dimension.
sql_queries : List[List[str]], optional (default=None)
A list of the SQL queries that are given during training or validation.
"""
embedded_utterance = self._utterance_embedder(tokens)
mask = util.get_text_field_mask(tokens).float()
batch_size = embedded_utterance.size(0)
# (batch_size, num_tokens, encoder_output_dim)
encoder_outputs = self._dropout(self._encoder(embedded_utterance, mask))
initial_state = self._get_initial_state(encoder_outputs, mask, valid_actions)
if action_sequence is not None:
# Remove the trailing dimension (from ListField[ListField[IndexField]]).
action_sequence = action_sequence.squeeze(-1)
target_mask = action_sequence != self._action_padding_index
else:
target_mask = None
outputs: Dict[str, Any] = {}
if action_sequence is not None:
# target_action_sequence is of shape (batch_size, 1, target_sequence_length)
# here after we unsqueeze it for the MML trainer.
loss_output = self._decoder_trainer.decode(initial_state,
self._transition_function,
(action_sequence.unsqueeze(1),
target_mask.unsqueeze(1)))
outputs.update(loss_output)
if not self.training:
action_mapping = []
for batch_actions in valid_actions:
batch_action_mapping = {}
for action_index, action in enumerate(batch_actions):
batch_action_mapping[action_index] = action[0]
action_mapping.append(batch_action_mapping)
outputs['action_mapping'] = action_mapping
# This tells the state to start keeping track of debug info, which we'll pass along in
# our output dictionary.
initial_state.debug_info = [[] for _ in range(batch_size)]
best_final_states = self._beam_search.search(self._max_decoding_steps,
initial_state,
self._transition_function,
keep_final_unfinished_states=True)
outputs['best_action_sequence'] = []
outputs['debug_info'] = []
outputs['predicted_sql_query'] = []
outputs['sql_queries'] = []
for i in range(batch_size):
# Decoding may not have terminated with any completed valid SQL queries, if `num_steps`
# isn't long enough (or if the model is not trained enough and gets into an
# infinite action loop).
if i not in best_final_states:
self._exact_match(0)
self._denotation_accuracy(0)
self._valid_sql_query(0)
self._action_similarity(0)
outputs['predicted_sql_query'].append('')
continue
best_action_indices = best_final_states[i][0].action_history[0]
action_strings = [action_mapping[i][action_index]
for action_index in best_action_indices]
predicted_sql_query = action_sequence_to_sql(action_strings)
if action_sequence is not None:
# Use a Tensor, not a Variable, to avoid a memory leak.
targets = action_sequence[i].data
sequence_in_targets = 0
sequence_in_targets = self._action_history_match(best_action_indices, targets)
self._exact_match(sequence_in_targets)
similarity = difflib.SequenceMatcher(None, best_action_indices, targets)
self._action_similarity(similarity.ratio())
outputs['best_action_sequence'].append(action_strings)
outputs['predicted_sql_query'].append(sqlparse.format(predicted_sql_query, reindent=True))
outputs['debug_info'].append(best_final_states[i][0].debug_info[0]) # type: ignore
return outputs
def _get_initial_state(self,
encoder_outputs: torch.Tensor,
mask: torch.Tensor,
actions: List[List[ProductionRule]]) -> GrammarBasedState:
batch_size = encoder_outputs.size(0)
# This will be our initial hidden state and memory cell for the decoder LSTM.
final_encoder_output = util.get_final_encoder_states(encoder_outputs,
mask,
self._encoder.is_bidirectional())
memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder.get_output_dim())
initial_score = encoder_outputs.data.new_zeros(batch_size)
# To make grouping states together in the decoder easier, we convert the batch dimension in
# all of our tensors into an outer list. For instance, the encoder outputs have shape
# `(batch_size, utterance_length, encoder_output_dim)`. We need to convert this into a list
# of `batch_size` tensors, each of shape `(utterance_length, encoder_output_dim)`. Then we
# won't have to do any index selects, or anything, we'll just do some `torch.cat()`s.
initial_score_list = [initial_score[i] for i in range(batch_size)]
encoder_output_list = [encoder_outputs[i] for i in range(batch_size)]
utterance_mask_list = [mask[i] for i in range(batch_size)]
initial_rnn_state = []
for i in range(batch_size):
initial_rnn_state.append(RnnStatelet(final_encoder_output[i],
memory_cell[i],
self._first_action_embedding,
self._first_attended_utterance,
encoder_output_list,
utterance_mask_list))
initial_grammar_state = [self._create_grammar_state(actions[i]) for i in range(batch_size)]
initial_state = GrammarBasedState(batch_indices=list(range(batch_size)),
action_history=[[] for _ in range(batch_size)],
score=initial_score_list,
rnn_state=initial_rnn_state,
grammar_state=initial_grammar_state,
possible_actions=actions,
debug_info=None)
return initial_state
@staticmethod
def _action_history_match(predicted: List[int], targets: torch.LongTensor) -> int:
# TODO(mattg): this could probably be moved into a FullSequenceMatch metric, or something.
# Check if target is big enough to cover prediction (including start/end symbols)
if len(predicted) > targets.size(0):
return 0
predicted_tensor = targets.new_tensor(predicted)
targets_trimmed = targets[:len(predicted)]
# Return 1 if the predicted sequence is anywhere in the list of targets.
return predicted_tensor.equal(targets_trimmed)
@staticmethod
def is_nonterminal(token: str):
if token[0] == '"' and token[-1] == '"':
return False
return True
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
"""
We track four metrics here:
1. exact_match, which is the percentage of the time that our best output action sequence
matches the SQL query exactly.
2. denotation_acc, which is the percentage of examples where we get the correct
denotation. This is the typical "accuracy" metric, and it is what you should usually
report in an experimental result. You need to be careful, though, that you're
computing this on the full data, and not just the subset that can be parsed. (make sure
you pass "keep_if_unparseable=True" to the dataset reader, which we do for validation data,
but not training data).
3. valid_sql_query, which is the percentage of time that decoding actually produces a
valid SQL query. We might not produce a valid SQL query if the decoder gets
into a repetitive loop, or we're trying to produce a super long SQL query and run
out of time steps, or something.
4. action_similarity, which is how similar the action sequence predicted is to the actual
action sequence. This is basically a soft measure of exact_match.
"""
validation_correct = self._exact_match._total_value # pylint: disable=protected-access
validation_total = self._exact_match._count # pylint: disable=protected-access
return {
'_exact_match_count': validation_correct,
'_example_count': validation_total,
'exact_match': self._exact_match.get_metric(reset),
'denotation_acc': self._denotation_accuracy.get_metric(reset),
'valid_sql_query': self._valid_sql_query.get_metric(reset),
'action_similarity': self._action_similarity.get_metric(reset)
}
def _create_grammar_state(self, possible_actions: List[ProductionRule]) -> GrammarStatelet:
"""
This method creates the GrammarStatelet object that's used for decoding. Part of creating
that is creating the `valid_actions` dictionary, which contains embedded representations of
all of the valid actions. So, we create that here as well.
The inputs to this method are for a `single instance in the batch`; none of the tensors we
create here are batched. We grab the global action ids from the input
``ProductionRules``, and we use those to embed the valid actions for every
non-terminal type. We use the input ``linking_scores`` for non-global actions.
Parameters
----------
possible_actions : ``List[ProductionRule]``
From the input to ``forward`` for a single batch instance.
"""
device = util.get_device_of(self._action_embedder.weight)
# TODO(Mark): This type is pure \(- . ^)/
translated_valid_actions: Dict[str, Dict[str, Tuple[torch.Tensor, torch.Tensor, List[int]]]] = {}
actions_grouped_by_nonterminal: Dict[str, List[Tuple[ProductionRule, int]]] = defaultdict(list)
for i, action in enumerate(possible_actions):
if action.rule == "":
continue
if action.is_global_rule:
actions_grouped_by_nonterminal[action.nonterminal].append((action, i))
else:
raise ValueError("The sql parser doesn't support non-global actions yet.")
for key, production_rule_arrays in actions_grouped_by_nonterminal.items():
translated_valid_actions[key] = {}
# `key` here is a non-terminal from the grammar, and `action_strings` are all the valid
# productions of that non-terminal. We'll first split those productions by global vs.
# linked action.
global_actions = []
for production_rule_array, action_index in production_rule_arrays:
global_actions.append((production_rule_array.rule_id, action_index))
if global_actions:
global_action_tensors, global_action_ids = zip(*global_actions)
global_action_tensor = torch.cat(global_action_tensors, dim=0).long()
if device >= 0:
global_action_tensor = global_action_tensor.to(device)
global_input_embeddings = self._action_embedder(global_action_tensor)
global_output_embeddings = self._output_action_embedder(global_action_tensor)
translated_valid_actions[key]['global'] = (global_input_embeddings,
global_output_embeddings,
list(global_action_ids))
return GrammarStatelet(['statement'],
translated_valid_actions,
self.is_nonterminal,
reverse_productions=True)
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
This method overrides ``Model.decode``, which gets called after ``Model.forward``, at test
time, to finalize predictions. This is (confusingly) a separate notion from the "decoder"
in "encoder/decoder", where that decoder logic lives in ``TransitionFunction``.
This method trims the output predictions to the first end symbol, replaces indices with
corresponding tokens, and adds a field called ``predicted_actions`` to the ``output_dict``.
"""
action_mapping = output_dict['action_mapping']
best_actions = output_dict["best_action_sequence"]
debug_infos = output_dict['debug_info']
batch_action_info = []
for batch_index, (predicted_actions, debug_info) in enumerate(zip(best_actions, debug_infos)):
instance_action_info = []
for predicted_action, action_debug_info in zip(predicted_actions, debug_info):
action_info = {}
action_info['predicted_action'] = predicted_action
considered_actions = action_debug_info['considered_actions']
probabilities = action_debug_info['probabilities']
actions = []
for action, probability in zip(considered_actions, probabilities):
if action != -1:
actions.append((action_mapping[batch_index][action], probability))
actions.sort()
considered_actions, probabilities = zip(*actions)
action_info['considered_actions'] = considered_actions
action_info['action_probabilities'] = probabilities
action_info['utterance_attention'] = action_debug_info.get('question_attention', [])
instance_action_info.append(action_info)
batch_action_info.append(instance_action_info)
output_dict["predicted_actions"] = batch_action_info
return output_dict
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
phrase_layer: Seq2SeqEncoder,
residual_encoder: Seq2SeqEncoder,
span_start_encoder: Seq2SeqEncoder,
span_end_encoder: Seq2SeqEncoder,
initializer: InitializerApplicator,
dropout: float = 0.2,
num_context_answers: int = 0,
marker_embedding_dim: int = 10,
max_span_length: int = 30,
max_turn_length: int = 12) -> None:
super().__init__(vocab)
self._num_context_answers = num_context_answers
self._max_span_length = max_span_length
self._text_field_embedder = text_field_embedder
self._phrase_layer = phrase_layer
self._marker_embedding_dim = marker_embedding_dim
self._encoding_dim = phrase_layer.get_output_dim()
self._matrix_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._merge_atten = TimeDistributed(torch.nn.Linear(self._encoding_dim * 4, self._encoding_dim))
self._residual_encoder = residual_encoder
if num_context_answers > 0:
self._question_num_marker = torch.nn.Embedding(max_turn_length,
marker_embedding_dim * num_context_answers)
self._prev_ans_marker = torch.nn.Embedding((num_context_answers * 4) + 1, marker_embedding_dim)
self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._followup_lin = torch.nn.Linear(self._encoding_dim, 3)
self._merge_self_attention = TimeDistributed(torch.nn.Linear(self._encoding_dim * 3,
self._encoding_dim))
self._span_start_encoder = span_start_encoder
self._span_end_encoder = span_end_encoder
self._span_start_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1))
self._span_end_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1))
self._span_yesno_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 3))
self._span_followup_predictor = TimeDistributed(self._followup_lin)
check_dimensions_match(phrase_layer.get_input_dim(),
text_field_embedder.get_output_dim() +
marker_embedding_dim * num_context_answers,
"phrase layer input dim",
"embedding dim + marker dim * num context answers")
initializer(self)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_yesno_accuracy = CategoricalAccuracy()
self._span_followup_accuracy = CategoricalAccuracy()
self._span_gt_yesno_accuracy = CategoricalAccuracy()
self._span_gt_followup_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._official_f1 = Average()
self._variational_dropout = InputVariationalDropout(dropout)
def __init__(self,
vocab: Vocabulary,
utterance_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
input_attention: Attention,
add_action_bias: bool = True,
training_beam_size: int = None,
decoder_num_layers: int = 1,
dropout: float = 0.0,
rule_namespace: str = 'rule_labels',
database_file='/atis/atis.db') -> None:
# Atis semantic parser init
super().__init__(vocab)
self._utterance_embedder = utterance_embedder
self._encoder = encoder
self._max_decoding_steps = max_decoding_steps
self._add_action_bias = add_action_bias
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._exact_match = Average()
self._valid_sql_query = Average()
self._action_similarity = Average()
self._denotation_accuracy = Average()
self._executor = SqlExecutor(database_file)
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
if self._add_action_bias:
input_action_dim = action_embedding_dim + 1
else:
input_action_dim = action_embedding_dim
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=input_action_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous utterance attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_utterance = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_utterance)
self._num_entity_types = 2 # TODO(kevin): get this in a more principled way somehow?
self._entity_type_decoder_embedding = Embedding(self._num_entity_types, action_embedding_dim)
self._decoder_num_layers = decoder_num_layers
self._beam_search = decoder_beam_search
self._decoder_trainer = MaximumMarginalLikelihood(training_beam_size)
self._transition_function = LinkingTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=input_attention,
predict_start_type_separately=False,
add_action_bias=self._add_action_bias,
dropout=dropout,
num_layers=self._decoder_num_layers)
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
attention: Attention,
mixture_feedforward: FeedForward = None,
add_action_bias: bool = True,
dropout: float = 0.0,
num_linking_features: int = 0,
num_entity_bits: int = 0,
entity_bits_output: bool = True,
use_entities: bool = False,
denotation_only: bool = False,
# Deprecated parameter to load older models
entity_encoder: Seq2VecEncoder = None, # pylint: disable=unused-argument
entity_similarity_mode: str = "dot_product",
rule_namespace: str = 'rule_labels') -> None:
super(QuarelSemanticParser, self).__init__(vocab)
self._question_embedder = question_embedder
self._encoder = encoder
self._beam_search = decoder_beam_search
self._max_decoding_steps = max_decoding_steps
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._denotation_accuracy = Average()
self._action_sequence_accuracy = Average()
self._has_logical_form = Average()
self._embedding_dim = question_embedder.get_output_dim()
self._use_entities = use_entities
# Note: there's only one non-trivial entity type in QuaRel for now, so most of the
# entity_type stuff is irrelevant
self._num_entity_types = 4 # TODO(mattg): get this in a more principled way somehow?
self._num_start_types = 1 # Hardcoded until we feed lf syntax into the model
self._entity_type_encoder_embedding = Embedding(self._num_entity_types, self._embedding_dim)
self._entity_type_decoder_embedding = Embedding(self._num_entity_types, action_embedding_dim)
self._entity_similarity_layer = None
self._entity_similarity_mode = entity_similarity_mode
if self._entity_similarity_mode == "weighted_dot_product":
self._entity_similarity_layer = \
TimeDistributed(torch.nn.Linear(self._embedding_dim, 1, bias=False))
# Center initial values around unweighted dot product
self._entity_similarity_layer._module.weight.data += 1 # pylint: disable=protected-access
elif self._entity_similarity_mode == "dot_product":
pass
else:
raise ValueError("Invalid entity_similarity_mode: {}".format(self._entity_similarity_mode))
if num_linking_features > 0:
self._linking_params = torch.nn.Linear(num_linking_features, 1)
else:
self._linking_params = None
self._decoder_trainer = MaximumMarginalLikelihood()
self._encoder_output_dim = self._encoder.get_output_dim()
if entity_bits_output:
self._encoder_output_dim += num_entity_bits
self._entity_bits_output = entity_bits_output
self._debug_count = 10
self._num_denotation_cats = 2 # Hardcoded for simplicity
self._denotation_only = denotation_only
if self._denotation_only:
self._denotation_accuracy_cat = CategoricalAccuracy()
self._denotation_classifier = torch.nn.Linear(self._encoder_output_dim,
self._num_denotation_cats)
# Rest of init not needed for denotation only where no decoding to actions needed
return
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
self._num_actions = num_actions
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# We are tying the action embeddings used for input and output
# self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._output_action_embedder = self._action_embedder # tied weights
self._add_action_bias = add_action_bias
if self._add_action_bias:
self._action_biases = Embedding(num_embeddings=num_actions, embedding_dim=1)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous question attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_question = torch.nn.Parameter(torch.FloatTensor(self._encoder_output_dim))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_question)
self._decoder_step = LinkingTransitionFunction(encoder_output_dim=self._encoder_output_dim,
action_embedding_dim=action_embedding_dim,
input_attention=attention,
num_start_types=self._num_start_types,
predict_start_type_separately=False,
add_action_bias=self._add_action_bias,
mixture_feedforward=mixture_feedforward,
dropout=dropout)
class AtisSemanticParser(Model):
"""
Parameters
----------
vocab : ``Vocabulary``
utterance_embedder : ``TextFieldEmbedder``
Embedder for utterances.
action_embedding_dim : ``int``
Dimension to use for action embeddings.
encoder : ``Seq2SeqEncoder``
The encoder to use for the input utterance.
decoder_beam_search : ``BeamSearch``
Beam search used to retrieve best sequences after training.
max_decoding_steps : ``int``
When we're decoding with a beam search, what's the maximum number of steps we should take?
This only applies at evaluation time, not during training.
input_attention: ``Attention``
We compute an attention over the input utterance at each step of the decoder, using the
decoder hidden state as the query. Passed to the transition function.
add_action_bias : ``bool``, optional (default=True)
If ``True``, we will learn a bias weight for each action that gets used when predicting
that action, in addition to its embedding.
dropout : ``float``, optional (default=0)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer).
rule_namespace : ``str``, optional (default=rule_labels)
The vocabulary namespace to use for production rules. The default corresponds to the
default used in the dataset reader, so you likely don't need to modify this.
database_file: ``str``, optional (default=/atis/atis.db)
The path of the SQLite database when evaluating SQL queries. SQLite is disk based, so we need
the file location to connect to it.
"""
def __init__(self,
vocab: Vocabulary,
utterance_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
input_attention: Attention,
add_action_bias: bool = True,
training_beam_size: int = None,
decoder_num_layers: int = 1,
dropout: float = 0.0,
rule_namespace: str = 'rule_labels',
database_file='/atis/atis.db') -> None:
# Atis semantic parser init
super().__init__(vocab)
self._utterance_embedder = utterance_embedder
self._encoder = encoder
self._max_decoding_steps = max_decoding_steps
self._add_action_bias = add_action_bias
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._exact_match = Average()
self._valid_sql_query = Average()
self._action_similarity = Average()
self._denotation_accuracy = Average()
self._executor = SqlExecutor(database_file)
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
if self._add_action_bias:
input_action_dim = action_embedding_dim + 1
else:
input_action_dim = action_embedding_dim
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=input_action_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous utterance attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_utterance = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_utterance)
self._num_entity_types = 2 # TODO(kevin): get this in a more principled way somehow?
self._entity_type_decoder_embedding = Embedding(self._num_entity_types, action_embedding_dim)
self._decoder_num_layers = decoder_num_layers
self._beam_search = decoder_beam_search
self._decoder_trainer = MaximumMarginalLikelihood(training_beam_size)
self._transition_function = LinkingTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=input_attention,
predict_start_type_separately=False,
add_action_bias=self._add_action_bias,
dropout=dropout,
num_layers=self._decoder_num_layers)
@overrides
def forward(self, # type: ignore
utterance: Dict[str, torch.LongTensor],
world: List[AtisWorld],
actions: List[List[ProductionRule]],
linking_scores: torch.Tensor,
target_action_sequence: torch.LongTensor = None,
sql_queries: List[List[str]] = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
We set up the initial state for the decoder, and pass that state off to either a DecoderTrainer,
if we're training, or a BeamSearch for inference, if we're not.
Parameters
----------
utterance : Dict[str, torch.LongTensor]
The output of ``TextField.as_array()`` applied on the utterance ``TextField``. This will
be passed through a ``TextFieldEmbedder`` and then through an encoder.
world : ``List[AtisWorld]``
We use a ``MetadataField`` to get the ``World`` for each input instance. Because of
how ``MetadataField`` works, this gets passed to us as a ``List[AtisWorld]``,
actions : ``List[List[ProductionRule]]``
A list of all possible actions for each ``World`` in the batch, indexed into a
``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these
and use the embeddings to determine which action to take at each timestep in the
decoder.
linking_scores: ``torch.Tensor``
A matrix of the linking the utterance tokens and the entities. This is a binary matrix that
is deterministically generated where each entry indicates whether a token generated an entity.
This tensor has shape ``(batch_size, num_entities, num_utterance_tokens)``.
target_action_sequence : torch.Tensor, optional (default=None)
The action sequence for the correct action sequence, where each action is an index into the list
of possible actions. This tensor has shape ``(batch_size, sequence_length, 1)``. We remove the
trailing dimension.
sql_queries : List[List[str]], optional (default=None)
A list of the SQL queries that are given during training or validation.
"""
initial_state = self._get_initial_state(utterance, world, actions, linking_scores)
batch_size = linking_scores.shape[0]
if target_action_sequence is not None:
# Remove the trailing dimension (from ListField[ListField[IndexField]]).
target_action_sequence = target_action_sequence.squeeze(-1)
target_mask = target_action_sequence != self._action_padding_index
else:
target_mask = None
if self.training:
# target_action_sequence is of shape (batch_size, 1, sequence_length) here after we unsqueeze it for
# the MML trainer.
return self._decoder_trainer.decode(initial_state,
self._transition_function,
(target_action_sequence.unsqueeze(1), target_mask.unsqueeze(1)))
else:
# TODO(kevin) Move some of this functionality to a separate method for computing validation outputs.
action_mapping = {}
for batch_index, batch_actions in enumerate(actions):
for action_index, action in enumerate(batch_actions):
action_mapping[(batch_index, action_index)] = action[0]
outputs: Dict[str, Any] = {'action_mapping': action_mapping}
outputs['linking_scores'] = linking_scores
if target_action_sequence is not None:
outputs['loss'] = self._decoder_trainer.decode(initial_state,
self._transition_function,
(target_action_sequence.unsqueeze(1),
target_mask.unsqueeze(1)))['loss']
num_steps = self._max_decoding_steps
# This tells the state to start keeping track of debug info, which we'll pass along in
# our output dictionary.
initial_state.debug_info = [[] for _ in range(batch_size)]
best_final_states = self._beam_search.search(num_steps,
initial_state,
self._transition_function,
keep_final_unfinished_states=False)
outputs['best_action_sequence'] = []
outputs['debug_info'] = []
outputs['entities'] = []
outputs['predicted_sql_query'] = []
outputs['sql_queries'] = []
outputs['utterance'] = []
outputs['tokenized_utterance'] = []
for i in range(batch_size):
# Decoding may not have terminated with any completed valid SQL queries, if `num_steps`
# isn't long enough (or if the model is not trained enough and gets into an
# infinite action loop).
if i not in best_final_states:
self._exact_match(0)
self._denotation_accuracy(0)
self._valid_sql_query(0)
self._action_similarity(0)
outputs['predicted_sql_query'].append('')
continue
best_action_indices = best_final_states[i][0].action_history[0]
action_strings = [action_mapping[(i, action_index)]
for action_index in best_action_indices]
predicted_sql_query = action_sequence_to_sql(action_strings)
if target_action_sequence is not None:
# Use a Tensor, not a Variable, to avoid a memory leak.
targets = target_action_sequence[i].data
sequence_in_targets = 0
sequence_in_targets = self._action_history_match(best_action_indices, targets)
self._exact_match(sequence_in_targets)
similarity = difflib.SequenceMatcher(None, best_action_indices, targets)
self._action_similarity(similarity.ratio())
if sql_queries and sql_queries[i]:
denotation_correct = self._executor.evaluate_sql_query(predicted_sql_query, sql_queries[i])
self._denotation_accuracy(denotation_correct)
outputs['sql_queries'].append(sql_queries[i])
outputs['utterance'].append(world[i].utterances[-1])
outputs['tokenized_utterance'].append([token.text
for token in world[i].tokenized_utterances[-1]])
outputs['entities'].append(world[i].entities)
outputs['best_action_sequence'].append(action_strings)
outputs['predicted_sql_query'].append(sqlparse.format(predicted_sql_query, reindent=True))
outputs['debug_info'].append(best_final_states[i][0].debug_info[0]) # type: ignore
return outputs
def _get_initial_state(self,
utterance: Dict[str, torch.LongTensor],
worlds: List[AtisWorld],
actions: List[List[ProductionRule]],
linking_scores: torch.Tensor) -> GrammarBasedState:
embedded_utterance = self._utterance_embedder(utterance)
utterance_mask = util.get_text_field_mask(utterance).float()
batch_size = embedded_utterance.size(0)
num_entities = max([len(world.entities) for world in worlds])
# entity_types: tensor with shape (batch_size, num_entities)
entity_types, _ = self._get_type_vector(worlds, num_entities, embedded_utterance)
# (batch_size, num_utterance_tokens, embedding_dim)
encoder_input = embedded_utterance
# (batch_size, utterance_length, encoder_output_dim)
encoder_outputs = self._dropout(self._encoder(encoder_input, utterance_mask))
# This will be our initial hidden state and memory cell for the decoder LSTM.
final_encoder_output = util.get_final_encoder_states(encoder_outputs,
utterance_mask,
self._encoder.is_bidirectional())
memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder.get_output_dim())
initial_score = embedded_utterance.data.new_zeros(batch_size)
# To make grouping states together in the decoder easier, we convert the batch dimension in
# all of our tensors into an outer list. For instance, the encoder outputs have shape
# `(batch_size, utterance_length, encoder_output_dim)`. We need to convert this into a list
# of `batch_size` tensors, each of shape `(utterance_length, encoder_output_dim)`. Then we
# won't have to do any index selects, or anything, we'll just do some `torch.cat()`s.
initial_score_list = [initial_score[i] for i in range(batch_size)]
encoder_output_list = [encoder_outputs[i] for i in range(batch_size)]
utterance_mask_list = [utterance_mask[i] for i in range(batch_size)]
initial_rnn_state = []
for i in range(batch_size):
if self._decoder_num_layers > 1:
initial_rnn_state.append(RnnStatelet(final_encoder_output[i].repeat(self._decoder_num_layers, 1),
memory_cell[i].repeat(self._decoder_num_layers, 1),
self._first_action_embedding,
self._first_attended_utterance,
encoder_output_list,
utterance_mask_list))
else:
initial_rnn_state.append(RnnStatelet(final_encoder_output[i],
memory_cell[i],
self._first_action_embedding,
self._first_attended_utterance,
encoder_output_list,
utterance_mask_list))
initial_grammar_state = [self._create_grammar_state(worlds[i],
actions[i],
linking_scores[i],
entity_types[i])
for i in range(batch_size)]
initial_state = GrammarBasedState(batch_indices=list(range(batch_size)),
action_history=[[] for _ in range(batch_size)],
score=initial_score_list,
rnn_state=initial_rnn_state,
grammar_state=initial_grammar_state,
possible_actions=actions,
debug_info=None)
return initial_state
@staticmethod
def _get_type_vector(worlds: List[AtisWorld],
num_entities: int,
tensor: torch.Tensor = None) -> Tuple[torch.LongTensor, Dict[int, int]]:
"""
Produces the encoding for each entity's type. In addition, a map from a flattened entity
index to type is returned to combine entity type operations into one method.
Parameters
----------
worlds : ``List[AtisWorld]``
num_entities : ``int``
tensor : ``torch.Tensor``
Used for copying the constructed list onto the right device.
Returns
-------
A ``torch.LongTensor`` with shape ``(batch_size, num_entities, num_types)``.
entity_types : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
"""
entity_types = {}
batch_types = []
for batch_index, world in enumerate(worlds):
types = []
entities = [('number', entity)
if any([entity.startswith(numeric_nonterminal)
for numeric_nonterminal in NUMERIC_NONTERMINALS])
else ('string', entity)
for entity in world.entities]
for entity_index, entity in enumerate(entities):
# We need numbers to be first, then strings, since our entities are going to be
# sorted. We do a split by type and then a merge later, and it relies on this sorting.
if entity[0] == 'number':
entity_type = 1
else:
entity_type = 0
types.append(entity_type)
# For easier lookups later, we're actually using a _flattened_ version
# of (batch_index, entity_index) for the key, because this is how the
# linking scores are stored.
flattened_entity_index = batch_index * num_entities + entity_index
entity_types[flattened_entity_index] = entity_type
padded = pad_sequence_to_length(types, num_entities, lambda: 0)
batch_types.append(padded)
return tensor.new_tensor(batch_types, dtype=torch.long), entity_types
@staticmethod
def _action_history_match(predicted: List[int], targets: torch.LongTensor) -> int:
# TODO(mattg): this could probably be moved into a FullSequenceMatch metric, or something.
# Check if target is big enough to cover prediction (including start/end symbols)
if len(predicted) > targets.size(0):
return 0
predicted_tensor = targets.new_tensor(predicted)
targets_trimmed = targets[:len(predicted)]
# Return 1 if the predicted sequence is anywhere in the list of targets.
return predicted_tensor.equal(targets_trimmed)
@staticmethod
def is_nonterminal(token: str):
if token[0] == '"' and token[-1] == '"':
return False
return True
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
"""
We track four metrics here:
1. exact_match, which is the percentage of the time that our best output action sequence
matches the SQL query exactly.
2. denotation_acc, which is the percentage of examples where we get the correct
denotation. This is the typical "accuracy" metric, and it is what you should usually
report in an experimental result. You need to be careful, though, that you're
computing this on the full data, and not just the subset that can be parsed. (make sure
you pass "keep_if_unparseable=True" to the dataset reader, which we do for validation data,
but not training data).
3. valid_sql_query, which is the percentage of time that decoding actually produces a
valid SQL query. We might not produce a valid SQL query if the decoder gets
into a repetitive loop, or we're trying to produce a super long SQL query and run
out of time steps, or something.
4. action_similarity, which is how similar the action sequence predicted is to the actual
action sequence. This is basically a soft measure of exact_match.
"""
return {
'exact_match': self._exact_match.get_metric(reset),
'denotation_acc': self._denotation_accuracy.get_metric(reset),
'valid_sql_query': self._valid_sql_query.get_metric(reset),
'action_similarity': self._action_similarity.get_metric(reset)
}
def _create_grammar_state(self,
world: AtisWorld,
possible_actions: List[ProductionRule],
linking_scores: torch.Tensor,
entity_types: torch.Tensor) -> GrammarStatelet:
"""
This method creates the GrammarStatelet object that's used for decoding. Part of creating
that is creating the `valid_actions` dictionary, which contains embedded representations of
all of the valid actions. So, we create that here as well.
The inputs to this method are for a `single instance in the batch`; none of the tensors we
create here are batched. We grab the global action ids from the input
``ProductionRules``, and we use those to embed the valid actions for every
non-terminal type. We use the input ``linking_scores`` for non-global actions.
Parameters
----------
world : ``AtisWorld``
From the input to ``forward`` for a single batch instance.
possible_actions : ``List[ProductionRule]``
From the input to ``forward`` for a single batch instance.
linking_scores : ``torch.Tensor``
Assumed to have shape ``(num_entities, num_utterance_tokens)`` (i.e., there is no batch
dimension).
entity_types : ``torch.Tensor``
Assumed to have shape ``(num_entities,)`` (i.e., there is no batch dimension).
"""
action_map = {}
for action_index, action in enumerate(possible_actions):
action_string = action[0]
action_map[action_string] = action_index
valid_actions = world.valid_actions
entity_map = {}
entities = world.entities
for entity_index, entity in enumerate(entities):
entity_map[entity] = entity_index
translated_valid_actions: Dict[str, Dict[str, Tuple[torch.Tensor, torch.Tensor, List[int]]]] = {}
for key, action_strings in valid_actions.items():
translated_valid_actions[key] = {}
# `key` here is a non-terminal from the grammar, and `action_strings` are all the valid
# productions of that non-terminal. We'll first split those productions by global vs.
# linked action.
action_indices = [action_map[action_string] for action_string in action_strings]
production_rule_arrays = [(possible_actions[index], index) for index in action_indices]
global_actions = []
linked_actions = []
for production_rule_array, action_index in production_rule_arrays:
if production_rule_array[1]:
global_actions.append((production_rule_array[2], action_index))
else:
linked_actions.append((production_rule_array[0], action_index))
if global_actions:
global_action_tensors, global_action_ids = zip(*global_actions)
global_action_tensor = torch.cat(global_action_tensors, dim=0).to(entity_types.device).long()
global_input_embeddings = self._action_embedder(global_action_tensor)
global_output_embeddings = self._output_action_embedder(global_action_tensor)
translated_valid_actions[key]['global'] = (global_input_embeddings,
global_output_embeddings,
list(global_action_ids))
if linked_actions:
linked_rules, linked_action_ids = zip(*linked_actions)
entities = linked_rules
entity_ids = [entity_map[entity] for entity in entities]
entity_linking_scores = linking_scores[entity_ids]
entity_type_tensor = entity_types[entity_ids]
entity_type_embeddings = (self._entity_type_decoder_embedding(entity_type_tensor)
.to(entity_types.device)
.float())
translated_valid_actions[key]['linked'] = (entity_linking_scores,
entity_type_embeddings,
list(linked_action_ids))
return GrammarStatelet(['statement'],
translated_valid_actions,
self.is_nonterminal)
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
This method overrides ``Model.decode``, which gets called after ``Model.forward``, at test
time, to finalize predictions. This is (confusingly) a separate notion from the "decoder"
in "encoder/decoder", where that decoder logic lives in ``TransitionFunction``.
This method trims the output predictions to the first end symbol, replaces indices with
corresponding tokens, and adds a field called ``predicted_actions`` to the ``output_dict``.
"""
action_mapping = output_dict['action_mapping']
best_actions = output_dict["best_action_sequence"]
debug_infos = output_dict['debug_info']
batch_action_info = []
for batch_index, (predicted_actions, debug_info) in enumerate(zip(best_actions, debug_infos)):
instance_action_info = []
for predicted_action, action_debug_info in zip(predicted_actions, debug_info):
action_info = {}
action_info['predicted_action'] = predicted_action
considered_actions = action_debug_info['considered_actions']
probabilities = action_debug_info['probabilities']
actions = []
for action, probability in zip(considered_actions, probabilities):
if action != -1:
actions.append((action_mapping[(batch_index, action)], probability))
actions.sort()
considered_actions, probabilities = zip(*actions)
action_info['considered_actions'] = considered_actions
action_info['action_probabilities'] = probabilities
action_info['utterance_attention'] = action_debug_info.get('question_attention', [])
instance_action_info.append(action_info)
batch_action_info.append(instance_action_info)
output_dict["predicted_actions"] = batch_action_info
return output_dict
class WikiTablesSemanticParser(Model):
"""
A ``WikiTablesSemanticParser`` is a :class:`Model` which takes as input a table and a question,
and produces a logical form that answers the question when executed over the table. The
logical form is generated by a `type-constrained`, `transition-based` parser. This is an
abstract class that defines most of the functionality related to the transition-based parser. It
does not contain the implementation for actually training the parser. You may want to train it
using a learning-to-search algorithm, in which case you will want to use
``WikiTablesErmSemanticParser``, or if you have a set of approximate logical forms that give the
correct denotation, you will want to use ``WikiTablesMmlSemanticParser``.
Parameters
----------
vocab : ``Vocabulary``
question_embedder : ``TextFieldEmbedder``
Embedder for questions.
action_embedding_dim : ``int``
Dimension to use for action embeddings.
encoder : ``Seq2SeqEncoder``
The encoder to use for the input question.
entity_encoder : ``Seq2VecEncoder``
The encoder to used for averaging the words of an entity.
max_decoding_steps : ``int``
When we're decoding with a beam search, what's the maximum number of steps we should take?
This only applies at evaluation time, not during training.
use_neighbor_similarity_for_linking : ``bool``, optional (default=False)
If ``True``, we will compute a max similarity between a question token and the `neighbors`
of an entity as a component of the linking scores. This is meant to capture the same kind
of information as the ``related_column`` feature.
dropout : ``float``, optional (default=0)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer).
num_linking_features : ``int``, optional (default=10)
We need to construct a parameter vector for the linking features, so we need to know how
many there are. The default of 8 here matches the default in the ``KnowledgeGraphField``,
which is to use all eight defined features. If this is 0, another term will be added to the
linking score. This term contains the maximum similarity value from the entity's neighbors
and the question.
rule_namespace : ``str``, optional (default=rule_labels)
The vocabulary namespace to use for production rules. The default corresponds to the
default used in the dataset reader, so you likely don't need to modify this.
tables_directory : ``str``, optional (default=/wikitables/)
The directory to find tables when evaluating logical forms. We rely on a call to SEMPRE to
evaluate logical forms, and SEMPRE needs to read the table from disk itself. This tells
SEMPRE where to find the tables.
"""
# pylint: disable=abstract-method
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
entity_encoder: Seq2VecEncoder,
max_decoding_steps: int,
use_neighbor_similarity_for_linking: bool = False,
dropout: float = 0.0,
num_linking_features: int = 10,
rule_namespace: str = 'rule_labels',
tables_directory: str = '/wikitables/') -> None:
super(WikiTablesSemanticParser, self).__init__(vocab)
self._question_embedder = question_embedder
self._encoder = encoder
self._entity_encoder = TimeDistributed(entity_encoder)
self._max_decoding_steps = max_decoding_steps
self._use_neighbor_similarity_for_linking = use_neighbor_similarity_for_linking
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._denotation_accuracy = WikiTablesAccuracy(tables_directory)
self._action_sequence_accuracy = Average()
self._has_logical_form = Average()
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._action_biases = Embedding(num_embeddings=num_actions, embedding_dim=1)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous question attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_question = torch.nn.Parameter(torch.FloatTensor(encoder.get_output_dim()))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_question)
check_dimensions_match(entity_encoder.get_output_dim(), question_embedder.get_output_dim(),
"entity word average embedding dim", "question embedding dim")
self._num_entity_types = 4 # TODO(mattg): get this in a more principled way somehow?
self._num_start_types = 5 # TODO(mattg): get this in a more principled way somehow?
self._embedding_dim = question_embedder.get_output_dim()
self._type_params = torch.nn.Linear(self._num_entity_types, self._embedding_dim)
self._neighbor_params = torch.nn.Linear(self._embedding_dim, self._embedding_dim)
if num_linking_features > 0:
self._linking_params = torch.nn.Linear(num_linking_features, 1)
else:
self._linking_params = None
if self._use_neighbor_similarity_for_linking:
self._question_entity_params = torch.nn.Linear(1, 1)
self._question_neighbor_params = torch.nn.Linear(1, 1)
else:
self._question_entity_params = None
self._question_neighbor_params = None
def _get_initial_state_and_scores(self,
question: Dict[str, torch.LongTensor],
table: Dict[str, torch.LongTensor],
world: List[WikiTablesWorld],
actions: List[List[ProductionRuleArray]],
example_lisp_string: List[str] = None,
add_world_to_initial_state: bool = False,
checklist_states: List[ChecklistState] = None) -> Dict:
"""
Does initial preparation and creates an intiial state for both the semantic parsers. Note
that the checklist state is optional, and the ``WikiTablesMmlParser`` is not expected to
pass it.
"""
table_text = table['text']
# (batch_size, question_length, embedding_dim)
embedded_question = self._question_embedder(question)
question_mask = util.get_text_field_mask(question).float()
# (batch_size, num_entities, num_entity_tokens, embedding_dim)
embedded_table = self._question_embedder(table_text, num_wrapping_dims=1)
table_mask = util.get_text_field_mask(table_text, num_wrapping_dims=1).float()
batch_size, num_entities, num_entity_tokens, _ = embedded_table.size()
num_question_tokens = embedded_question.size(1)
# (batch_size, num_entities, embedding_dim)
encoded_table = self._entity_encoder(embedded_table, table_mask)
# (batch_size, num_entities, num_neighbors)
neighbor_indices = self._get_neighbor_indices(world, num_entities, encoded_table)
# Neighbor_indices is padded with -1 since 0 is a potential neighbor index.
# Thus, the absolute value needs to be taken in the index_select, and 1 needs to
# be added for the mask since that method expects 0 for padding.
# (batch_size, num_entities, num_neighbors, embedding_dim)
embedded_neighbors = util.batched_index_select(encoded_table, torch.abs(neighbor_indices))
neighbor_mask = util.get_text_field_mask({'ignored': neighbor_indices + 1},
num_wrapping_dims=1).float()
# Encoder initialized to easily obtain a masked average.
neighbor_encoder = TimeDistributed(BagOfEmbeddingsEncoder(self._embedding_dim, averaged=True))
# (batch_size, num_entities, embedding_dim)
embedded_neighbors = neighbor_encoder(embedded_neighbors, neighbor_mask)
# entity_types: one-hot tensor with shape (batch_size, num_entities, num_types)
# entity_type_dict: Dict[int, int], mapping flattened_entity_index -> type_index
# These encode the same information, but for efficiency reasons later it's nice
# to have one version as a tensor and one that's accessible on the cpu.
entity_types, entity_type_dict = self._get_type_vector(world, num_entities, encoded_table)
entity_type_embeddings = self._type_params(entity_types.float())
projected_neighbor_embeddings = self._neighbor_params(embedded_neighbors.float())
# (batch_size, num_entities, embedding_dim)
entity_embeddings = torch.nn.functional.tanh(entity_type_embeddings + projected_neighbor_embeddings)
# Compute entity and question word similarity. We tried using cosine distance here, but
# because this similarity is the main mechanism that the model can use to push apart logit
# scores for certain actions (like "n -> 1" and "n -> -1"), this needs to have a larger
# output range than [-1, 1].
question_entity_similarity = torch.bmm(embedded_table.view(batch_size,
num_entities * num_entity_tokens,
self._embedding_dim),
torch.transpose(embedded_question, 1, 2))
question_entity_similarity = question_entity_similarity.view(batch_size,
num_entities,
num_entity_tokens,
num_question_tokens)
# (batch_size, num_entities, num_question_tokens)
question_entity_similarity_max_score, _ = torch.max(question_entity_similarity, 2)
# (batch_size, num_entities, num_question_tokens, num_features)
linking_features = table['linking']
linking_scores = question_entity_similarity_max_score
if self._use_neighbor_similarity_for_linking:
# The linking score is computed as a linear projection of two terms. The first is the
# maximum similarity score over the entity's words and the question token. The second
# is the maximum similarity over the words in the entity's neighbors and the question
# token.
#
# The second term, projected_question_neighbor_similarity, is useful when a column
# needs to be selected. For example, the question token might have no similarity with
# the column name, but is similar with the cells in the column.
#
# Note that projected_question_neighbor_similarity is intended to capture the same
# information as the related_column feature.
#
# Also note that this block needs to be _before_ the `linking_params` block, because
# we're overwriting `linking_scores`, not adding to it.
# (batch_size, num_entities, num_neighbors, num_question_tokens)
question_neighbor_similarity = util.batched_index_select(question_entity_similarity_max_score,
torch.abs(neighbor_indices))
# (batch_size, num_entities, num_question_tokens)
question_neighbor_similarity_max_score, _ = torch.max(question_neighbor_similarity, 2)
projected_question_entity_similarity = self._question_entity_params(
question_entity_similarity_max_score.unsqueeze(-1)).squeeze(-1)
projected_question_neighbor_similarity = self._question_neighbor_params(
question_neighbor_similarity_max_score.unsqueeze(-1)).squeeze(-1)
linking_scores = projected_question_entity_similarity + projected_question_neighbor_similarity
feature_scores = None
if self._linking_params is not None:
feature_scores = self._linking_params(linking_features).squeeze(3)
linking_scores = linking_scores + feature_scores
# (batch_size, num_question_tokens, num_entities)
linking_probabilities = self._get_linking_probabilities(world, linking_scores.transpose(1, 2),
question_mask, entity_type_dict)
# (batch_size, num_question_tokens, embedding_dim)
link_embedding = util.weighted_sum(entity_embeddings, linking_probabilities)
encoder_input = torch.cat([link_embedding, embedded_question], 2)
# (batch_size, question_length, encoder_output_dim)
encoder_outputs = self._dropout(self._encoder(encoder_input, question_mask))
# This will be our initial hidden state and memory cell for the decoder LSTM.
final_encoder_output = util.get_final_encoder_states(encoder_outputs,
question_mask,
self._encoder.is_bidirectional())
memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder.get_output_dim())
initial_score = embedded_question.data.new_zeros(batch_size)
action_embeddings, output_action_embeddings, action_biases, action_indices = self._embed_actions(actions)
_, num_entities, num_question_tokens = linking_scores.size()
flattened_linking_scores, actions_to_entities = self._map_entity_productions(linking_scores,
world,
actions)
# To make grouping states together in the decoder easier, we convert the batch dimension in
# all of our tensors into an outer list. For instance, the encoder outputs have shape
# `(batch_size, question_length, encoder_output_dim)`. We need to convert this into a list
# of `batch_size` tensors, each of shape `(question_length, encoder_output_dim)`. Then we
# won't have to do any index selects, or anything, we'll just do some `torch.cat()`s.
initial_score_list = [initial_score[i] for i in range(batch_size)]
encoder_output_list = [encoder_outputs[i] for i in range(batch_size)]
question_mask_list = [question_mask[i] for i in range(batch_size)]
initial_rnn_state = []
for i in range(batch_size):
initial_rnn_state.append(RnnState(final_encoder_output[i],
memory_cell[i],
self._first_action_embedding,
self._first_attended_question,
encoder_output_list,
question_mask_list))
initial_grammar_state = [self._create_grammar_state(world[i], actions[i])
for i in range(batch_size)]
initial_state_world = world if add_world_to_initial_state else None
initial_state = WikiTablesDecoderState(batch_indices=list(range(batch_size)),
action_history=[[] for _ in range(batch_size)],
score=initial_score_list,
rnn_state=initial_rnn_state,
grammar_state=initial_grammar_state,
action_embeddings=action_embeddings,
output_action_embeddings=output_action_embeddings,
action_biases=action_biases,
action_indices=action_indices,
possible_actions=actions,
flattened_linking_scores=flattened_linking_scores,
actions_to_entities=actions_to_entities,
entity_types=entity_type_dict,
world=initial_state_world,
example_lisp_string=example_lisp_string,
checklist_state=checklist_states,
debug_info=None)
return {"initial_state": initial_state,
"linking_scores": linking_scores,
"feature_scores": feature_scores,
"similarity_scores": question_entity_similarity_max_score}
@staticmethod
def _get_neighbor_indices(worlds: List[WikiTablesWorld],
num_entities: int,
tensor: torch.Tensor) -> torch.LongTensor:
"""
This method returns the indices of each entity's neighbors. A tensor
is accepted as a parameter for copying purposes.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
num_entities : ``int``
tensor : ``torch.Tensor``
Used for copying the constructed list onto the right device.
Returns
-------
A ``torch.LongTensor`` with shape ``(batch_size, num_entities, num_neighbors)``. It is padded
with -1 instead of 0, since 0 is a valid neighbor index.
"""
num_neighbors = 0
for world in worlds:
for entity in world.table_graph.entities:
if len(world.table_graph.neighbors[entity]) > num_neighbors:
num_neighbors = len(world.table_graph.neighbors[entity])
batch_neighbors = []
for world in worlds:
# Each batch instance has its own world, which has a corresponding table.
entities = world.table_graph.entities
entity2index = {entity: i for i, entity in enumerate(entities)}
entity2neighbors = world.table_graph.neighbors
neighbor_indexes = []
for entity in entities:
entity_neighbors = [entity2index[n] for n in entity2neighbors[entity]]
# Pad with -1 instead of 0, since 0 represents a neighbor index.
padded = pad_sequence_to_length(entity_neighbors, num_neighbors, lambda: -1)
neighbor_indexes.append(padded)
neighbor_indexes = pad_sequence_to_length(neighbor_indexes,
num_entities,
lambda: [-1] * num_neighbors)
batch_neighbors.append(neighbor_indexes)
return tensor.new_tensor(batch_neighbors, dtype=torch.long)
@staticmethod
def _get_type_vector(worlds: List[WikiTablesWorld],
num_entities: int,
tensor: torch.Tensor) -> Tuple[torch.LongTensor, Dict[int, int]]:
"""
Produces the one hot encoding for each entity's type. In addition,
a map from a flattened entity index to type is returned to combine
entity type operations into one method.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
num_entities : ``int``
tensor : ``torch.Tensor``
Used for copying the constructed list onto the right device.
Returns
-------
A ``torch.LongTensor`` with shape ``(batch_size, num_entities, num_types)``.
entity_types : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
"""
entity_types = {}
batch_types = []
for batch_index, world in enumerate(worlds):
types = []
for entity_index, entity in enumerate(world.table_graph.entities):
one_hot_vectors = [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]
# We need numbers to be first, then cells, then parts, then row, because our
# entities are going to be sorted. We do a split by type and then a merge later,
# and it relies on this sorting.
if entity.startswith('fb:cell'):
entity_type = 1
elif entity.startswith('fb:part'):
entity_type = 2
elif entity.startswith('fb:row'):
entity_type = 3
else:
entity_type = 0
types.append(one_hot_vectors[entity_type])
# For easier lookups later, we're actually using a _flattened_ version
# of (batch_index, entity_index) for the key, because this is how the
# linking scores are stored.
flattened_entity_index = batch_index * num_entities + entity_index
entity_types[flattened_entity_index] = entity_type
padded = pad_sequence_to_length(types, num_entities, lambda: [0, 0, 0, 0])
batch_types.append(padded)
return tensor.new_tensor(batch_types), entity_types
def _get_linking_probabilities(self,
worlds: List[WikiTablesWorld],
linking_scores: torch.FloatTensor,
question_mask: torch.LongTensor,
entity_type_dict: Dict[int, int]) -> torch.FloatTensor:
"""
Produces the probability of an entity given a question word and type. The logic below
separates the entities by type since the softmax normalization term sums over entities
of a single type.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
linking_scores : ``torch.FloatTensor``
Has shape (batch_size, num_question_tokens, num_entities).
question_mask: ``torch.LongTensor``
Has shape (batch_size, num_question_tokens).
entity_type_dict : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
Returns
-------
batch_probabilities : ``torch.FloatTensor``
Has shape ``(batch_size, num_question_tokens, num_entities)``.
Contains all the probabilities for an entity given a question word.
"""
_, num_question_tokens, num_entities = linking_scores.size()
batch_probabilities = []
for batch_index, world in enumerate(worlds):
all_probabilities = []
num_entities_in_instance = 0
# NOTE: The way that we're doing this here relies on the fact that entities are
# implicitly sorted by their types when we sort them by name, and that numbers come
# before "fb:cell", and "fb:cell" comes before "fb:row". This is not a great
# assumption, and could easily break later, but it should work for now.
for type_index in range(self._num_entity_types):
# This index of 0 is for the null entity for each type, representing the case where a
# word doesn't link to any entity.
entity_indices = [0]
entities = world.table_graph.entities
for entity_index, _ in enumerate(entities):
if entity_type_dict[batch_index * num_entities + entity_index] == type_index:
entity_indices.append(entity_index)
if len(entity_indices) == 1:
# No entities of this type; move along...
continue
# We're subtracting one here because of the null entity we added above.
num_entities_in_instance += len(entity_indices) - 1
# We separate the scores by type, since normalization is done per type. There's an
# extra "null" entity per type, also, so we have `num_entities_per_type + 1`. We're
# selecting from a (num_question_tokens, num_entities) linking tensor on _dimension 1_,
# so we get back something of shape (num_question_tokens,) for each index we're
# selecting. All of the selected indices together then make a tensor of shape
# (num_question_tokens, num_entities_per_type + 1).
indices = linking_scores.new_tensor(entity_indices, dtype=torch.long)
entity_scores = linking_scores[batch_index].index_select(1, indices)
# We used index 0 for the null entity, so this will actually have some values in it.
# But we want the null entity's score to be 0, so we set that here.
entity_scores[:, 0] = 0
# No need for a mask here, as this is done per batch instance, with no padding.
type_probabilities = torch.nn.functional.softmax(entity_scores, dim=1)
all_probabilities.append(type_probabilities[:, 1:])
# We need to add padding here if we don't have the right number of entities.
if num_entities_in_instance != num_entities:
zeros = linking_scores.new_zeros(num_question_tokens,
num_entities - num_entities_in_instance)
all_probabilities.append(zeros)
# (num_question_tokens, num_entities)
probabilities = torch.cat(all_probabilities, dim=1)
batch_probabilities.append(probabilities)
batch_probabilities = torch.stack(batch_probabilities, dim=0)
return batch_probabilities * question_mask.unsqueeze(-1).float()
@staticmethod
def _action_history_match(predicted: List[int], targets: torch.LongTensor) -> int:
# TODO(mattg): this could probably be moved into a FullSequenceMatch metric, or something.
# Check if target is big enough to cover prediction (including start/end symbols)
if len(predicted) > targets.size(1):
return 0
predicted_tensor = targets.new_tensor(predicted)
targets_trimmed = targets[:, :len(predicted)]
# Return 1 if the predicted sequence is anywhere in the list of targets.
return torch.max(torch.min(targets_trimmed.eq(predicted_tensor), dim=1)[0]).item()
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
"""
We track three metrics here:
1. dpd_acc, which is the percentage of the time that our best output action sequence is
in the set of action sequences provided by DPD. This is an easy-to-compute lower bound
on denotation accuracy for the set of examples where we actually have DPD output. We
only score dpd_acc on that subset.
2. denotation_acc, which is the percentage of examples where we get the correct
denotation. This is the typical "accuracy" metric, and it is what you should usually
report in an experimental result. You need to be careful, though, that you're
computing this on the full data, and not just the subset that has DPD output (make sure
you pass "keep_if_no_dpd=True" to the dataset reader, which we do for validation data,
but not training data).
3. lf_percent, which is the percentage of time that decoding actually produces a
finished logical form. We might not produce a valid logical form if the decoder gets
into a repetitive loop, or we're trying to produce a super long logical form and run
out of time steps, or something.
"""
return {
'dpd_acc': self._action_sequence_accuracy.get_metric(reset),
'denotation_acc': self._denotation_accuracy.get_metric(reset),
'lf_percent': self._has_logical_form.get_metric(reset),
}
@staticmethod
def _create_grammar_state(world: WikiTablesWorld,
possible_actions: List[ProductionRuleArray]) -> GrammarState:
valid_actions = world.get_valid_actions()
action_mapping = {}
for i, action in enumerate(possible_actions):
action_string = action[0]
action_mapping[action_string] = i
translated_valid_actions = {}
for key, action_strings in valid_actions.items():
translated_valid_actions[key] = [action_mapping[action_string]
for action_string in action_strings]
return GrammarState([START_SYMBOL],
{},
translated_valid_actions,
action_mapping,
type_declaration.is_nonterminal)
def _embed_actions(self, actions: List[List[ProductionRuleArray]]) -> Tuple[torch.Tensor,
torch.Tensor,
torch.Tensor,
Dict[Tuple[int, int], int]]:
"""
Given all of the possible actions for all batch instances, produce an embedding for them.
There will be significant overlap in this list, as the production rules from the grammar
are shared across all batch instances. Our returned tensor has an embedding for each
`unique` action, so we also need to return a mapping from the original ``(batch_index,
action_index)`` to our new ``global_action_index``, so that we can get the right action
embedding during decoding.
Returns
-------
action_embeddings : ``torch.Tensor``
Has shape ``(num_unique_actions, action_embedding_dim)``.
output_action_embeddings : ``torch.Tensor``
Has shape ``(num_unique_actions, action_embedding_dim)``.
action_biases : ``torch.Tensor``
Has shape ``(num_unique_actions, 1)``.
action_map : ``Dict[Tuple[int, int], int]``
Maps ``(batch_index, action_index)`` in the input action list to ``action_index`` in
the ``action_embeddings`` tensor. All non-embeddable actions get mapped to `-1` here.
"""
# TODO(mattg): This whole action pipeline might be a whole lot more complicated than it
# needs to be. We used to embed actions differently (using some crazy ideas about
# embedding the LHS and RHS separately); we could probably get away with simplifying things
# further now that we're just doing a simple embedding for global actions. But I'm leaving
# it like this for now to have a minimal change to go from the LHS/RHS embedding to a
# single action embedding.
embedded_actions = self._action_embedder.weight
output_embedded_actions = self._output_action_embedder.weight
action_biases = self._action_biases.weight
# Now we just need to make a map from `(batch_index, action_index)` to
# `global_action_index`. global_action_ids has the list of all unique actions; here we're
# going over all of the actions for each batch instance so we can map them to the global
# action ids.
action_vocab = self.vocab.get_token_to_index_vocabulary(self._rule_namespace)
action_map: Dict[Tuple[int, int], int] = {}
for batch_index, instance_actions in enumerate(actions):
for action_index, action in enumerate(instance_actions):
if not action[0]:
# This rule is padding.
continue
global_action_id = action_vocab.get(action[0], -1)
action_map[(batch_index, action_index)] = global_action_id
return embedded_actions, output_embedded_actions, action_biases, action_map
@staticmethod
def _map_entity_productions(linking_scores: torch.FloatTensor,
worlds: List[WikiTablesWorld],
actions: List[List[ProductionRuleArray]]) -> Tuple[torch.Tensor,
Dict[Tuple[int, int], int]]:
"""
Constructs a map from ``(batch_index, action_index)`` to ``(batch_index * entity_index)``.
That is, some actions correspond to terminal productions of entities from our table. We
need to find those actions and map them to their corresponding entity indices, where the
entity index is its position in the list of entities returned by the ``world``. This list
is what defines the second dimension of the ``linking_scores`` tensor, so we can use this
index to look up linking scores for each action in that tensor.
For easier processing later, the mapping that we return is `flattened` - we really want to
map ``(batch_index, action_index)`` to ``(batch_index, entity_index)``, but we are going to
have to use the result of this mapping to do ``index_selects`` on the ``linking_scores``
tensor. You can't do ``index_select`` with tuples, so we flatten ``linking_scores`` to
have shape ``(batch_size * num_entities, num_question_tokens)``, and return shifted indices
into this flattened tensor.
Parameters
----------
linking_scores : ``torch.Tensor``
A tensor representing linking scores between each table entity and each question token.
Has shape ``(batch_size, num_entities, num_question_tokens)``.
worlds : ``List[WikiTablesWorld]``
The ``World`` for each batch instance. The ``World`` contains a reference to the
``TableKnowledgeGraph`` that defines the set of entities in the linking.
actions : ``List[List[ProductionRuleArray]]``
The list of possible actions for each batch instance. Our action indices are defined
in terms of this list, so we'll find entity productions in this list and map them to
entity indices from the entity list we get from the ``World``.
Returns
-------
flattened_linking_scores : ``torch.Tensor``
A flattened version of ``linking_scores``, with shape ``(batch_size * num_entities,
num_question_tokens)``.
actions_to_entities : ``Dict[Tuple[int, int], int]``
A mapping from ``(batch_index, action_index)`` to ``(batch_size * num_entities)``,
representing which action indices correspond to which entity indices in the returned
``flattened_linking_scores`` tensor.
"""
batch_size, num_entities, num_question_tokens = linking_scores.size()
entity_map: Dict[Tuple[int, str], int] = {}
for batch_index, world in enumerate(worlds):
for entity_index, entity in enumerate(world.table_graph.entities):
entity_map[(batch_index, entity)] = batch_index * num_entities + entity_index
actions_to_entities: Dict[Tuple[int, int], int] = {}
for batch_index, action_list in enumerate(actions):
for action_index, action in enumerate(action_list):
if not action[0]:
# This action is padding.
continue
_, production = action[0].split(' -> ')
entity_index = entity_map.get((batch_index, production), None)
if entity_index is not None:
actions_to_entities[(batch_index, action_index)] = entity_index
flattened_linking_scores = linking_scores.view(batch_size * num_entities, num_question_tokens)
return flattened_linking_scores, actions_to_entities
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
This method overrides ``Model.decode``, which gets called after ``Model.forward``, at test
time, to finalize predictions. This is (confusingly) a separate notion from the "decoder"
in "encoder/decoder", where that decoder logic lives in ``WikiTablesDecoderStep``.
This method trims the output predictions to the first end symbol, replaces indices with
corresponding tokens, and adds a field called ``predicted_tokens`` to the ``output_dict``.
"""
action_mapping = output_dict['action_mapping']
best_actions = output_dict["best_action_sequence"]
debug_infos = output_dict['debug_info']
batch_action_info = []
for batch_index, (predicted_actions, debug_info) in enumerate(zip(best_actions, debug_infos)):
instance_action_info = []
for predicted_action, action_debug_info in zip(predicted_actions, debug_info):
action_info = {}
action_info['predicted_action'] = predicted_action
considered_actions = action_debug_info['considered_actions']
probabilities = action_debug_info['probabilities']
actions = []
for action, probability in zip(considered_actions, probabilities):
if action != -1:
actions.append((action_mapping[(batch_index, action)], probability))
actions.sort()
considered_actions, probabilities = zip(*actions)
action_info['considered_actions'] = considered_actions
action_info['action_probabilities'] = probabilities
action_info['question_attention'] = action_debug_info.get('question_attention', [])
instance_action_info.append(action_info)
batch_action_info.append(instance_action_info)
output_dict["predicted_actions"] = batch_action_info
return output_dict
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
entity_encoder: Seq2VecEncoder,
attention: Attention,
decoder_beam_size: int,
decoder_num_finished_states: int,
max_decoding_steps: int,
mixture_feedforward: FeedForward = None,
add_action_bias: bool = True,
normalize_beam_score_by_length: bool = False,
checklist_cost_weight: float = 0.6,
use_neighbor_similarity_for_linking: bool = False,
dropout: float = 0.0,
num_linking_features: int = 10,
rule_namespace: str = 'rule_labels',
tables_directory: str = '/wikitables/',
mml_model_file: str = None) -> None:
use_similarity = use_neighbor_similarity_for_linking
super().__init__(vocab=vocab,
question_embedder=question_embedder,
action_embedding_dim=action_embedding_dim,
encoder=encoder,
entity_encoder=entity_encoder,
max_decoding_steps=max_decoding_steps,
add_action_bias=add_action_bias,
use_neighbor_similarity_for_linking=use_similarity,
dropout=dropout,
num_linking_features=num_linking_features,
rule_namespace=rule_namespace,
tables_directory=tables_directory)
# Not sure why mypy needs a type annotation for this!
self._decoder_trainer: ExpectedRiskMinimization = \
ExpectedRiskMinimization(beam_size=decoder_beam_size,
normalize_by_length=normalize_beam_score_by_length,
max_decoding_steps=self._max_decoding_steps,
max_num_finished_states=decoder_num_finished_states)
unlinked_terminals_global_indices = []
global_vocab = self.vocab.get_token_to_index_vocabulary(rule_namespace)
for production, index in global_vocab.items():
right_side = production.split(" -> ")[1]
if right_side in types.COMMON_NAME_MAPPING:
# This is a terminal production.
unlinked_terminals_global_indices.append(index)
self._num_unlinked_terminals = len(unlinked_terminals_global_indices)
self._decoder_step = LinkingCoverageTransitionFunction(encoder_output_dim=self._encoder.get_output_dim(),
action_embedding_dim=action_embedding_dim,
input_attention=attention,
num_start_types=self._num_start_types,
predict_start_type_separately=True,
add_action_bias=self._add_action_bias,
mixture_feedforward=mixture_feedforward,
dropout=dropout)
self._checklist_cost_weight = checklist_cost_weight
self._agenda_coverage = Average()
# TODO (pradeep): Checking whether file exists here to avoid raising an error when we've
# copied a trained ERM model from a different machine and the original MML model that was
# used to initialize it does not exist on the current machine. This may not be the best
# solution for the problem.
if mml_model_file is not None:
if os.path.isfile(mml_model_file):
archive = load_archive(mml_model_file)
self._initialize_weights_from_archive(archive)
else:
# A model file is passed, but it does not exist. This is expected to happen when
# you're using a trained ERM model to decode. But it may also happen if the path to
# the file is really just incorrect. So throwing a warning.
logger.warning("MML model file for initializing weights is passed, but does not exist."
" This is fine if you're just decoding.")
class STS14Task(Task):
'''
Task class for Sentence Textual Similarity 14.
Training data is STS12 and STS13 data, as provided in the dataset.
'''
def __init__(self, path, max_seq_len, name="sts14"):
''' '''
super(STS14Task, self).__init__(name, 1)
self.name = name
self.pair_input = 1
self.categorical = 0
#self.val_metric = "%s_accuracy" % self.name
self.val_metric = "%s_accuracy" % self.name
self.val_metric_decreases = False
self.scorer = Average()
self.load_data(path, max_seq_len)
def load_data(self, path, max_seq_len):
'''
Process the dataset located at path.
TODO: preprocess and store data so don't have to wait?
Args:
- path (str): path to data
'''
def load_year_split(path):
sents1, sents2, targs = [], [], []
input_files = glob.glob('%s/STS.input.*.txt' % path)
targ_files = glob.glob('%s/STS.gs.*.txt' % path)
input_files.sort()
targ_files.sort()
for inp, targ in zip(input_files, targ_files):
topic_sents1, topic_sents2, topic_targs = \
load_file(path, inp, targ)
sents1 += topic_sents1
sents2 += topic_sents2
targs += topic_targs
assert len(sents1) == len(sents2) == len(targs)
return sents1, sents2, targs
def load_file(path, inp, targ):
sents1, sents2, targs = [], [], []
with open(inp) as fh, open(targ) as gh:
for raw_sents, raw_targ in zip(fh, gh):
raw_sents = raw_sents.split('\t')
sent1 = process_sentence(raw_sents[0], max_seq_len)
sent2 = process_sentence(raw_sents[1], max_seq_len)
if not sent1 or not sent2:
continue
sents1.append(sent1)
sents2.append(sent2)
targs.append(float(raw_targ) / 5) # rescale for cosine
return sents1, sents2, targs
sort_data = lambda s1, s2, t: \
sorted(zip(s1, s2, t), key=lambda x: (len(x[0]), len(x[1])))
unpack = lambda x: [l for l in map(list, zip(*x))]
sts2topics = {
12: ['MSRpar', 'MSRvid', 'SMTeuroparl', 'surprise.OnWN', \
'surprise.SMTnews'],
13: ['FNWN', 'headlines', 'OnWN'],
14: ['deft-forum', 'deft-news', 'headlines', 'images', \
'OnWN', 'tweet-news']
}
sents1, sents2, targs = [], [], []
train_dirs = ['STS2012-train', 'STS2012-test', 'STS2013-test']
for train_dir in train_dirs:
res = load_year_split(path + train_dir + '/')
sents1 += res[0]
sents2 += res[1]
targs += res[2]
data = [(s1, s2, t) for s1, s2, t in zip(sents1, sents2, targs)]
random.shuffle(data)
sents1, sents2, targs = unpack(data)
split_pt = int(.8 * len(sents1))
tr_data = sort_data(sents1[:split_pt], sents2[:split_pt],
targs[:split_pt])
val_data = sort_data(sents1[split_pt:], sents2[split_pt:],
targs[split_pt:])
te_data = sort_data(*load_year_split(path))
self.train_data_text = unpack(tr_data)
self.val_data_text = unpack(val_data)
self.test_data_text = unpack(te_data)
log.info("\tFinished loading STS14 data.")
def get_metrics(self, reset=False):
return {'accuracy': self.scorer.get_metric(reset)}
class QuarelSemanticParser(Model):
"""
A ``QuarelSemanticParser`` is a variant of ``WikiTablesSemanticParser`` with various
tweaks and changes.
Parameters
----------
vocab : ``Vocabulary``
question_embedder : ``TextFieldEmbedder``
Embedder for questions.
action_embedding_dim : ``int``
Dimension to use for action embeddings.
encoder : ``Seq2SeqEncoder``
The encoder to use for the input question.
decoder_beam_search : ``BeamSearch``
When we're not training, this is how we will do decoding.
max_decoding_steps : ``int``
When we're decoding with a beam search, what's the maximum number of steps we should take?
This only applies at evaluation time, not during training.
attention : ``Attention``
We compute an attention over the input question at each step of the decoder, using the
decoder hidden state as the query. Passed to the transition function.
dropout : ``float``, optional (default=0)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer).
num_linking_features : ``int``, optional (default=10)
We need to construct a parameter vector for the linking features, so we need to know how
many there are. The default of 8 here matches the default in the ``KnowledgeGraphField``,
which is to use all eight defined features. If this is 0, another term will be added to the
linking score. This term contains the maximum similarity value from the entity's neighbors
and the question.
use_entities : ``bool``, optional (default=False)
Whether dynamic entities are part of the action space
num_entity_bits : ``int``, optional (default=0)
Whether any bits are added to encoder input/output to represent tagged entities
entity_bits_output : ``bool``, optional (default=False)
Whether entity bits are added to the encoder output or input
denotation_only : ``bool``, optional (default=False)
Whether to only predict target denotation, skipping the the whole logical form decoder
entity_similarity_mode : ``str``, optional (default="dot_product")
How to compute vector similarity between question and entity tokens, can take values
"dot_product" or "weighted_dot_product" (learned weights on each dimension)
rule_namespace : ``str``, optional (default=rule_labels)
The vocabulary namespace to use for production rules. The default corresponds to the
default used in the dataset reader, so you likely don't need to modify this.
"""
def __init__(self,
vocab: Vocabulary,
question_embedder: TextFieldEmbedder,
action_embedding_dim: int,
encoder: Seq2SeqEncoder,
decoder_beam_search: BeamSearch,
max_decoding_steps: int,
attention: Attention,
mixture_feedforward: FeedForward = None,
add_action_bias: bool = True,
dropout: float = 0.0,
num_linking_features: int = 0,
num_entity_bits: int = 0,
entity_bits_output: bool = True,
use_entities: bool = False,
denotation_only: bool = False,
# Deprecated parameter to load older models
entity_encoder: Seq2VecEncoder = None, # pylint: disable=unused-argument
entity_similarity_mode: str = "dot_product",
rule_namespace: str = 'rule_labels') -> None:
super(QuarelSemanticParser, self).__init__(vocab)
self._question_embedder = question_embedder
self._encoder = encoder
self._beam_search = decoder_beam_search
self._max_decoding_steps = max_decoding_steps
if dropout > 0:
self._dropout = torch.nn.Dropout(p=dropout)
else:
self._dropout = lambda x: x
self._rule_namespace = rule_namespace
self._denotation_accuracy = Average()
self._action_sequence_accuracy = Average()
self._has_logical_form = Average()
self._embedding_dim = question_embedder.get_output_dim()
self._use_entities = use_entities
# Note: there's only one non-trivial entity type in QuaRel for now, so most of the
# entity_type stuff is irrelevant
self._num_entity_types = 4 # TODO(mattg): get this in a more principled way somehow?
self._num_start_types = 1 # Hardcoded until we feed lf syntax into the model
self._entity_type_encoder_embedding = Embedding(self._num_entity_types, self._embedding_dim)
self._entity_type_decoder_embedding = Embedding(self._num_entity_types, action_embedding_dim)
self._entity_similarity_layer = None
self._entity_similarity_mode = entity_similarity_mode
if self._entity_similarity_mode == "weighted_dot_product":
self._entity_similarity_layer = \
TimeDistributed(torch.nn.Linear(self._embedding_dim, 1, bias=False))
# Center initial values around unweighted dot product
self._entity_similarity_layer._module.weight.data += 1 # pylint: disable=protected-access
elif self._entity_similarity_mode == "dot_product":
pass
else:
raise ValueError("Invalid entity_similarity_mode: {}".format(self._entity_similarity_mode))
if num_linking_features > 0:
self._linking_params = torch.nn.Linear(num_linking_features, 1)
else:
self._linking_params = None
self._decoder_trainer = MaximumMarginalLikelihood()
self._encoder_output_dim = self._encoder.get_output_dim()
if entity_bits_output:
self._encoder_output_dim += num_entity_bits
self._entity_bits_output = entity_bits_output
self._debug_count = 10
self._num_denotation_cats = 2 # Hardcoded for simplicity
self._denotation_only = denotation_only
if self._denotation_only:
self._denotation_accuracy_cat = CategoricalAccuracy()
self._denotation_classifier = torch.nn.Linear(self._encoder_output_dim,
self._num_denotation_cats)
# Rest of init not needed for denotation only where no decoding to actions needed
return
self._action_padding_index = -1 # the padding value used by IndexField
num_actions = vocab.get_vocab_size(self._rule_namespace)
self._num_actions = num_actions
self._action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
# We are tying the action embeddings used for input and output
# self._output_action_embedder = Embedding(num_embeddings=num_actions, embedding_dim=action_embedding_dim)
self._output_action_embedder = self._action_embedder # tied weights
self._add_action_bias = add_action_bias
if self._add_action_bias:
self._action_biases = Embedding(num_embeddings=num_actions, embedding_dim=1)
# This is what we pass as input in the first step of decoding, when we don't have a
# previous action, or a previous question attention.
self._first_action_embedding = torch.nn.Parameter(torch.FloatTensor(action_embedding_dim))
self._first_attended_question = torch.nn.Parameter(torch.FloatTensor(self._encoder_output_dim))
torch.nn.init.normal_(self._first_action_embedding)
torch.nn.init.normal_(self._first_attended_question)
self._decoder_step = LinkingTransitionFunction(encoder_output_dim=self._encoder_output_dim,
action_embedding_dim=action_embedding_dim,
input_attention=attention,
num_start_types=self._num_start_types,
predict_start_type_separately=False,
add_action_bias=self._add_action_bias,
mixture_feedforward=mixture_feedforward,
dropout=dropout)
@overrides
def forward(self, # type: ignore
question: Dict[str, torch.LongTensor],
table: Dict[str, torch.LongTensor],
world: List[QuarelWorld],
actions: List[List[ProductionRule]],
entity_bits: torch.Tensor = None,
denotation_target: torch.Tensor = None,
target_action_sequences: torch.LongTensor = None,
metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
# pylint: disable=unused-argument
"""
In this method we encode the table entities, link them to words in the question, then
encode the question. Then we set up the initial state for the decoder, and pass that
state off to either a DecoderTrainer, if we're training, or a BeamSearch for inference,
if we're not.
Parameters
----------
question : Dict[str, torch.LongTensor]
The output of ``TextField.as_array()`` applied on the question ``TextField``. This will
be passed through a ``TextFieldEmbedder`` and then through an encoder.
table : ``Dict[str, torch.LongTensor]``
The output of ``KnowledgeGraphField.as_array()`` applied on the table
``KnowledgeGraphField``. This output is similar to a ``TextField`` output, where each
entity in the table is treated as a "token", and we will use a ``TextFieldEmbedder`` to
get embeddings for each entity.
world : ``List[QuarelWorld]``
We use a ``MetadataField`` to get the ``World`` for each input instance. Because of
how ``MetadataField`` works, this gets passed to us as a ``List[QuarelWorld]``,
actions : ``List[List[ProductionRule]]``
A list of all possible actions for each ``World`` in the batch, indexed into a
``ProductionRule`` using a ``ProductionRuleField``. We will embed all of these
and use the embeddings to determine which action to take at each timestep in the
decoder.
target_action_sequences : torch.Tensor, optional (default=None)
A list of possibly valid action sequences, where each action is an index into the list
of possible actions. This tensor has shape ``(batch_size, num_action_sequences,
sequence_length)``.
"""
table_text = table['text']
self._debug_count -= 1
# (batch_size, question_length, embedding_dim)
embedded_question = self._question_embedder(question)
question_mask = util.get_text_field_mask(question).float()
num_question_tokens = embedded_question.size(1)
# (batch_size, num_entities, num_entity_tokens, embedding_dim)
embedded_table = self._question_embedder(table_text, num_wrapping_dims=1)
batch_size, num_entities, num_entity_tokens, _ = embedded_table.size()
# entity_types: one-hot tensor with shape (batch_size, num_entities, num_types)
# entity_type_dict: Dict[int, int], mapping flattened_entity_index -> type_index
# These encode the same information, but for efficiency reasons later it's nice
# to have one version as a tensor and one that's accessible on the cpu.
entity_types, entity_type_dict = self._get_type_vector(world, num_entities, embedded_table)
if self._use_entities:
if self._entity_similarity_mode == "dot_product":
# Compute entity and question word cosine similarity. Need to add a small value to
# to the table norm since there are padding values which cause a divide by 0.
embedded_table = embedded_table / (embedded_table.norm(dim=-1, keepdim=True) + 1e-13)
embedded_question = embedded_question / (embedded_question.norm(dim=-1, keepdim=True) + 1e-13)
question_entity_similarity = torch.bmm(embedded_table.view(batch_size,
num_entities * num_entity_tokens,
self._embedding_dim),
torch.transpose(embedded_question, 1, 2))
question_entity_similarity = question_entity_similarity.view(batch_size,
num_entities,
num_entity_tokens,
num_question_tokens)
# (batch_size, num_entities, num_question_tokens)
question_entity_similarity_max_score, _ = torch.max(question_entity_similarity, 2)
linking_scores = question_entity_similarity_max_score
elif self._entity_similarity_mode == "weighted_dot_product":
embedded_table = embedded_table / (embedded_table.norm(dim=-1, keepdim=True) + 1e-13)
embedded_question = embedded_question / (embedded_question.norm(dim=-1, keepdim=True) + 1e-13)
eqe = embedded_question.unsqueeze(1).expand(-1, num_entities*num_entity_tokens, -1, -1)
ete = embedded_table.view(batch_size, num_entities*num_entity_tokens, self._embedding_dim)
ete = ete.unsqueeze(2).expand(-1, -1, num_question_tokens, -1)
product = torch.mul(eqe, ete)
product = product.view(batch_size,
num_question_tokens*num_entities*num_entity_tokens,
self._embedding_dim)
question_entity_similarity = self._entity_similarity_layer(product)
question_entity_similarity = question_entity_similarity.view(batch_size,
num_entities,
num_entity_tokens,
num_question_tokens)
# (batch_size, num_entities, num_question_tokens)
question_entity_similarity_max_score, _ = torch.max(question_entity_similarity, 2)
linking_scores = question_entity_similarity_max_score
# (batch_size, num_entities, num_question_tokens, num_features)
linking_features = table['linking']
if self._linking_params is not None:
feature_scores = self._linking_params(linking_features).squeeze(3)
linking_scores = linking_scores + feature_scores
# (batch_size, num_question_tokens, num_entities)
linking_probabilities = self._get_linking_probabilities(world, linking_scores.transpose(1, 2),
question_mask, entity_type_dict)
encoder_input = embedded_question
else:
if entity_bits is not None and not self._entity_bits_output:
encoder_input = torch.cat([embedded_question, entity_bits], 2)
else:
encoder_input = embedded_question
# Fake linking_scores added for downstream code to not object
linking_scores = question_mask.clone().fill_(0).unsqueeze(1)
linking_probabilities = None
# (batch_size, question_length, encoder_output_dim)
encoder_outputs = self._dropout(self._encoder(encoder_input, question_mask))
if self._entity_bits_output and entity_bits is not None:
encoder_outputs = torch.cat([encoder_outputs, entity_bits], 2)
# This will be our initial hidden state and memory cell for the decoder LSTM.
final_encoder_output = util.get_final_encoder_states(encoder_outputs,
question_mask,
self._encoder.is_bidirectional())
# For predicting a categorical denotation directly
if self._denotation_only:
denotation_logits = self._denotation_classifier(final_encoder_output)
loss = torch.nn.functional.cross_entropy(denotation_logits, denotation_target.view(-1))
self._denotation_accuracy_cat(denotation_logits, denotation_target)
return {"loss": loss}
memory_cell = encoder_outputs.new_zeros(batch_size, self._encoder_output_dim)
_, num_entities, num_question_tokens = linking_scores.size()
if target_action_sequences is not None:
# Remove the trailing dimension (from ListField[ListField[IndexField]]).
target_action_sequences = target_action_sequences.squeeze(-1)
target_mask = target_action_sequences != self._action_padding_index
else:
target_mask = None
# To make grouping states together in the decoder easier, we convert the batch dimension in
# all of our tensors into an outer list. For instance, the encoder outputs have shape
# `(batch_size, question_length, encoder_output_dim)`. We need to convert this into a list
# of `batch_size` tensors, each of shape `(question_length, encoder_output_dim)`. Then we
# won't have to do any index selects, or anything, we'll just do some `torch.cat()`s.
encoder_output_list = [encoder_outputs[i] for i in range(batch_size)]
question_mask_list = [question_mask[i] for i in range(batch_size)]
initial_rnn_state = []
for i in range(batch_size):
initial_rnn_state.append(RnnStatelet(final_encoder_output[i],
memory_cell[i],
self._first_action_embedding,
self._first_attended_question,
encoder_output_list,
question_mask_list))
initial_grammar_state = [self._create_grammar_state(world[i], actions[i],
linking_scores[i], entity_types[i])
for i in range(batch_size)]
initial_score = initial_rnn_state[0].hidden_state.new_zeros(batch_size)
initial_score_list = [initial_score[i] for i in range(batch_size)]
initial_state = GrammarBasedState(batch_indices=list(range(batch_size)),
action_history=[[] for _ in range(batch_size)],
score=initial_score_list,
rnn_state=initial_rnn_state,
grammar_state=initial_grammar_state,
possible_actions=actions,
extras=None,
debug_info=None)
if self.training:
outputs = self._decoder_trainer.decode(initial_state,
self._decoder_step,
(target_action_sequences, target_mask))
return outputs
else:
action_mapping = {}
for batch_index, batch_actions in enumerate(actions):
for action_index, action in enumerate(batch_actions):
action_mapping[(batch_index, action_index)] = action[0]
outputs = {'action_mapping': action_mapping}
if target_action_sequences is not None:
outputs['loss'] = self._decoder_trainer.decode(initial_state,
self._decoder_step,
(target_action_sequences, target_mask))['loss']
num_steps = self._max_decoding_steps
# This tells the state to start keeping track of debug info, which we'll pass along in
# our output dictionary.
initial_state.debug_info = [[] for _ in range(batch_size)]
best_final_states = self._beam_search.search(num_steps,
initial_state,
self._decoder_step,
keep_final_unfinished_states=False)
outputs['best_action_sequence'] = []
outputs['debug_info'] = []
outputs['entities'] = []
if self._linking_params is not None:
outputs['linking_scores'] = linking_scores
outputs['feature_scores'] = feature_scores
outputs['linking_features'] = linking_features
if self._use_entities:
outputs['linking_probabilities'] = linking_probabilities
if entity_bits is not None:
outputs['entity_bits'] = entity_bits
# outputs['similarity_scores'] = question_entity_similarity_max_score
outputs['logical_form'] = []
outputs['denotation_acc'] = []
outputs['score'] = []
outputs['parse_acc'] = []
outputs['answer_index'] = []
if metadata is not None:
outputs['question_tokens'] = []
outputs['world_extractions'] = []
for i in range(batch_size):
if metadata is not None:
outputs['question_tokens'].append(metadata[i].get('question_tokens', []))
if metadata is not None:
outputs['world_extractions'].append(metadata[i].get('world_extractions', {}))
outputs['entities'].append(world[i].table_graph.entities)
# Decoding may not have terminated with any completed logical forms, if `num_steps`
# isn't long enough (or if the model is not trained enough and gets into an
# infinite action loop).
if i in best_final_states:
best_action_indices = best_final_states[i][0].action_history[0]
sequence_in_targets = 0
if target_action_sequences is not None:
targets = target_action_sequences[i].data
sequence_in_targets = self._action_history_match(best_action_indices, targets)
self._action_sequence_accuracy(sequence_in_targets)
action_strings = [action_mapping[(i, action_index)] for action_index in best_action_indices]
try:
self._has_logical_form(1.0)
logical_form = world[i].get_logical_form(action_strings, add_var_function=False)
except ParsingError:
self._has_logical_form(0.0)
logical_form = 'Error producing logical form'
denotation_accuracy = 0.0
predicted_answer_index = world[i].execute(logical_form)
if metadata is not None and 'answer_index' in metadata[i]:
answer_index = metadata[i]['answer_index']
denotation_accuracy = self._denotation_match(predicted_answer_index, answer_index)
self._denotation_accuracy(denotation_accuracy)
score = math.exp(best_final_states[i][0].score[0].data.cpu().item())
outputs['answer_index'].append(predicted_answer_index)
outputs['score'].append(score)
outputs['parse_acc'].append(sequence_in_targets)
outputs['best_action_sequence'].append(action_strings)
outputs['logical_form'].append(logical_form)
outputs['denotation_acc'].append(denotation_accuracy)
outputs['debug_info'].append(best_final_states[i][0].debug_info[0]) # type: ignore
else:
outputs['parse_acc'].append(0)
outputs['logical_form'].append('')
outputs['denotation_acc'].append(0)
outputs['score'].append(0)
outputs['answer_index'].append(-1)
outputs['best_action_sequence'].append([])
outputs['debug_info'].append([])
self._has_logical_form(0.0)
return outputs
@staticmethod
def _get_type_vector(worlds: List[QuarelWorld],
num_entities: int,
tensor: torch.Tensor) -> Tuple[torch.LongTensor, Dict[int, int]]:
"""
Produces a tensor with shape ``(batch_size, num_entities)`` that encodes each entity's
type. In addition, a map from a flattened entity index to type is returned to combine
entity type operations into one method.
Parameters
----------
worlds : ``List[WikiTablesWorld]``
num_entities : ``int``
tensor : ``torch.Tensor``
Used for copying the constructed list onto the right device.
Returns
-------
A ``torch.LongTensor`` with shape ``(batch_size, num_entities)``.
entity_types : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
"""
entity_types = {}
batch_types = []
for batch_index, world in enumerate(worlds):
types = []
for entity_index, entity in enumerate(world.table_graph.entities):
# We need numbers to be first, then cells, then parts, then row, because our
# entities are going to be sorted. We do a split by type and then a merge later,
# and it relies on this sorting.
if entity.startswith('fb:cell'):
entity_type = 1
elif entity.startswith('fb:part'):
entity_type = 2
elif entity.startswith('fb:row'):
entity_type = 3
else:
entity_type = 0
types.append(entity_type)
# For easier lookups later, we're actually using a _flattened_ version
# of (batch_index, entity_index) for the key, because this is how the
# linking scores are stored.
flattened_entity_index = batch_index * num_entities + entity_index
entity_types[flattened_entity_index] = entity_type
padded = pad_sequence_to_length(types, num_entities, lambda: 0)
batch_types.append(padded)
return tensor.new_tensor(batch_types, dtype=torch.long), entity_types
def _get_linking_probabilities(self,
worlds: List[QuarelWorld],
linking_scores: torch.FloatTensor,
question_mask: torch.LongTensor,
entity_type_dict: Dict[int, int]) -> torch.FloatTensor:
"""
Produces the probability of an entity given a question word and type. The logic below
separates the entities by type since the softmax normalization term sums over entities
of a single type.
Parameters
----------
worlds : ``List[QuarelWorld]``
linking_scores : ``torch.FloatTensor``
Has shape (batch_size, num_question_tokens, num_entities).
question_mask: ``torch.LongTensor``
Has shape (batch_size, num_question_tokens).
entity_type_dict : ``Dict[int, int]``
This is a mapping from ((batch_index * num_entities) + entity_index) to entity type id.
Returns
-------
batch_probabilities : ``torch.FloatTensor``
Has shape ``(batch_size, num_question_tokens, num_entities)``.
Contains all the probabilities for an entity given a question word.
"""
_, num_question_tokens, num_entities = linking_scores.size()
batch_probabilities = []
for batch_index, world in enumerate(worlds):
all_probabilities = []
num_entities_in_instance = 0
# NOTE: The way that we're doing this here relies on the fact that entities are
# implicitly sorted by their types when we sort them by name, and that numbers come
# before "fb:cell", and "fb:cell" comes before "fb:row". This is not a great
# assumption, and could easily break later, but it should work for now.
for type_index in range(self._num_entity_types):
# This index of 0 is for the null entity for each type, representing the case where a
# word doesn't link to any entity.
entity_indices = [0]
entities = world.table_graph.entities
for entity_index, _ in enumerate(entities):
if entity_type_dict[batch_index * num_entities + entity_index] == type_index:
entity_indices.append(entity_index)
if len(entity_indices) == 1:
# No entities of this type; move along...
continue
# We're subtracting one here because of the null entity we added above.
num_entities_in_instance += len(entity_indices) - 1
# We separate the scores by type, since normalization is done per type. There's an
# extra "null" entity per type, also, so we have `num_entities_per_type + 1`. We're
# selecting from a (num_question_tokens, num_entities) linking tensor on _dimension 1_,
# so we get back something of shape (num_question_tokens,) for each index we're
# selecting. All of the selected indices together then make a tensor of shape
# (num_question_tokens, num_entities_per_type + 1).
indices = linking_scores.new_tensor(entity_indices, dtype=torch.long)
entity_scores = linking_scores[batch_index].index_select(1, indices)
# We used index 0 for the null entity, so this will actually have some values in it.
# But we want the null entity's score to be 0, so we set that here.
entity_scores[:, 0] = 0
# No need for a mask here, as this is done per batch instance, with no padding.
type_probabilities = torch.nn.functional.softmax(entity_scores, dim=1)
all_probabilities.append(type_probabilities[:, 1:])
# We need to add padding here if we don't have the right number of entities.
if num_entities_in_instance != num_entities:
zeros = linking_scores.new_zeros(num_question_tokens,
num_entities - num_entities_in_instance)
all_probabilities.append(zeros)
# (num_question_tokens, num_entities)
probabilities = torch.cat(all_probabilities, dim=1)
batch_probabilities.append(probabilities)
batch_probabilities = torch.stack(batch_probabilities, dim=0)
return batch_probabilities * question_mask.unsqueeze(-1).float()
@staticmethod
def _action_history_match(predicted: List[int], targets: torch.LongTensor) -> int:
# TODO(mattg): this could probably be moved into a FullSequenceMatch metric, or something.
# Check if target is big enough to cover prediction (including start/end symbols)
if len(predicted) > targets.size(1):
return 0
predicted_tensor = targets.new_tensor(predicted)
targets_trimmed = targets[:, :len(predicted)]
# Return 1 if the predicted sequence is anywhere in the list of targets.
return torch.max(torch.min(targets_trimmed.eq(predicted_tensor), dim=1)[0]).item()
def _denotation_match(self, predicted_answer_index: int, target_answer_index: int) -> float:
if predicted_answer_index < 0:
# Logical form doesn't properly resolve, we do random guess with appropriate credit
return 1.0/self._num_denotation_cats
elif predicted_answer_index == target_answer_index:
return 1.0
return 0.0
@overrides
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
"""
We track three metrics here:
1. parse_acc, which is the percentage of the time that our best output action sequence
corresponds to a correct logical form
2. denotation_acc, which is the percentage of examples where we get the correct
denotation, including spurious correct answers using the wrong logical form
3. lf_percent, which is the percentage of time that decoding actually produces a
finished logical form. We might not produce a valid logical form if the decoder gets
into a repetitive loop, or we're trying to produce a super long logical form and run
out of time steps, or something.
"""
if self._denotation_only:
metrics = {'denotation_acc': self._denotation_accuracy_cat.get_metric(reset)}
else:
metrics = {
'parse_acc': self._action_sequence_accuracy.get_metric(reset),
'denotation_acc': self._denotation_accuracy.get_metric(reset),
'lf_percent': self._has_logical_form.get_metric(reset),
}
return metrics
def _create_grammar_state(self,
world: QuarelWorld,
possible_actions: List[ProductionRule],
linking_scores: torch.Tensor,
entity_types: torch.Tensor) -> GrammarStatelet:
"""
This method creates the GrammarStatelet object that's used for decoding. Part of creating
that is creating the `valid_actions` dictionary, which contains embedded representations of
all of the valid actions. So, we create that here as well.
The inputs to this method are for a `single instance in the batch`; none of the tensors we
create here are batched. We grab the global action ids from the input
``ProductionRules``, and we use those to embed the valid actions for every
non-terminal type. We use the input ``linking_scores`` for non-global actions.
Parameters
----------
world : ``QuarelWorld``
From the input to ``forward`` for a single batch instance.
possible_actions : ``List[ProductionRule]``
From the input to ``forward`` for a single batch instance.
linking_scores : ``torch.Tensor``
Assumed to have shape ``(num_entities, num_question_tokens)`` (i.e., there is no batch
dimension).
entity_types : ``torch.Tensor``
Assumed to have shape ``(num_entities,)`` (i.e., there is no batch dimension).
"""
action_map = {}
for action_index, action in enumerate(possible_actions):
action_string = action[0]
action_map[action_string] = action_index
entity_map = {}
for entity_index, entity in enumerate(world.table_graph.entities):
entity_map[entity] = entity_index
valid_actions = world.get_valid_actions()
translated_valid_actions: Dict[str, Dict[str, Tuple[torch.Tensor, torch.Tensor, List[int]]]] = {}
for key, action_strings in valid_actions.items():
translated_valid_actions[key] = {}
# `key` here is a non-terminal from the grammar, and `action_strings` are all the valid
# productions of that non-terminal. We'll first split those productions by global vs.
# linked action.
action_indices = [action_map[action_string] for action_string in action_strings]
production_rule_arrays = [(possible_actions[index], index) for index in action_indices]
global_actions = []
linked_actions = []
for production_rule_array, action_index in production_rule_arrays:
if production_rule_array[1]:
global_actions.append((production_rule_array[2], action_index))
else:
linked_actions.append((production_rule_array[0], action_index))
# Then we get the embedded representations of the global actions.
global_action_tensors, global_action_ids = zip(*global_actions)
global_action_tensor = torch.cat(global_action_tensors, dim=0)
global_input_embeddings = self._action_embedder(global_action_tensor)
if self._add_action_bias:
global_action_biases = self._action_biases(global_action_tensor)
global_input_embeddings = torch.cat([global_input_embeddings, global_action_biases], dim=-1)
global_output_embeddings = self._output_action_embedder(global_action_tensor)
translated_valid_actions[key]['global'] = (global_input_embeddings,
global_output_embeddings,
list(global_action_ids))
# Then the representations of the linked actions.
if linked_actions:
linked_rules, linked_action_ids = zip(*linked_actions)
entities = [rule.split(' -> ')[1] for rule in linked_rules]
entity_ids = [entity_map[entity] for entity in entities]
# (num_linked_actions, num_question_tokens)
entity_linking_scores = linking_scores[entity_ids]
# (num_linked_actions,)
entity_type_tensor = entity_types[entity_ids]
# (num_linked_actions, entity_type_embedding_dim)
entity_type_embeddings = self._entity_type_decoder_embedding(entity_type_tensor)
translated_valid_actions[key]['linked'] = (entity_linking_scores,
entity_type_embeddings,
list(linked_action_ids))
return GrammarStatelet([START_SYMBOL],
translated_valid_actions,
type_declaration.is_nonterminal)
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
This method overrides ``Model.decode``, which gets called after ``Model.forward``, at test
time, to finalize predictions. This is (confusingly) a separate notion from the "decoder"
in "encoder/decoder", where that decoder logic lives in ``FrictionQDecoderStep``.
This method trims the output predictions to the first end symbol, replaces indices with
corresponding tokens, and adds a field called ``predicted_tokens`` to the ``output_dict``.
"""
action_mapping = output_dict['action_mapping']
best_actions = output_dict["best_action_sequence"]
debug_infos = output_dict['debug_info']
batch_action_info = []
for batch_index, (predicted_actions, debug_info) in enumerate(zip(best_actions, debug_infos)):
instance_action_info = []
for predicted_action, action_debug_info in zip(predicted_actions, debug_info):
action_info = {}
action_info['predicted_action'] = predicted_action
considered_actions = action_debug_info['considered_actions']
probabilities = action_debug_info['probabilities']
actions = []
for action, probability in zip(considered_actions, probabilities):
if action != -1:
actions.append((action_mapping[(batch_index, action)], probability))
actions.sort()
considered_actions, probabilities = zip(*actions)
action_info['considered_actions'] = considered_actions
action_info['action_probabilities'] = probabilities
action_info['question_attention'] = action_debug_info.get('question_attention', [])
instance_action_info.append(action_info)
batch_action_info.append(instance_action_info)
output_dict["predicted_actions"] = batch_action_info
return output_dict
class DialogQA(Model):
"""
This class implements modified version of BiDAF
(with self attention and residual layer, from Clark and Gardner ACL 17 paper) model as used in
Question Answering in Context (EMNLP 2018) paper [https://arxiv.org/pdf/1808.07036.pdf].
In this set-up, a single instance is a dialog, list of question answer pairs.
Parameters
----------
vocab : ``Vocabulary``
text_field_embedder : ``TextFieldEmbedder``
Used to embed the ``question`` and ``passage`` ``TextFields`` we get as input to the model.
phrase_layer : ``Seq2SeqEncoder``
The encoder (with its own internal stacking) that we will use in between embedding tokens
and doing the bidirectional attention.
span_start_encoder : ``Seq2SeqEncoder``
The encoder that we will use to incorporate span start predictions into the passage state
before predicting span end.
span_end_encoder : ``Seq2SeqEncoder``
The encoder that we will use to incorporate span end predictions into the passage state.
dropout : ``float``, optional (default=0.2)
If greater than 0, we will apply dropout with this probability after all encoders (pytorch
LSTMs do not apply dropout to their last layer).
num_context_answers : ``int``, optional (default=0)
If greater than 0, the model will consider previous question answering context.
max_span_length: ``int``, optional (default=0)
Maximum token length of the output span.
max_turn_length: ``int``, optional (default=12)
Maximum length of an interaction.
"""
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
phrase_layer: Seq2SeqEncoder,
residual_encoder: Seq2SeqEncoder,
span_start_encoder: Seq2SeqEncoder,
span_end_encoder: Seq2SeqEncoder,
initializer: InitializerApplicator,
dropout: float = 0.2,
num_context_answers: int = 0,
marker_embedding_dim: int = 10,
max_span_length: int = 30,
max_turn_length: int = 12) -> None:
super().__init__(vocab)
self._num_context_answers = num_context_answers
self._max_span_length = max_span_length
self._text_field_embedder = text_field_embedder
self._phrase_layer = phrase_layer
self._marker_embedding_dim = marker_embedding_dim
self._encoding_dim = phrase_layer.get_output_dim()
self._matrix_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._merge_atten = TimeDistributed(torch.nn.Linear(self._encoding_dim * 4, self._encoding_dim))
self._residual_encoder = residual_encoder
if num_context_answers > 0:
self._question_num_marker = torch.nn.Embedding(max_turn_length,
marker_embedding_dim * num_context_answers)
self._prev_ans_marker = torch.nn.Embedding((num_context_answers * 4) + 1, marker_embedding_dim)
self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y')
self._followup_lin = torch.nn.Linear(self._encoding_dim, 3)
self._merge_self_attention = TimeDistributed(torch.nn.Linear(self._encoding_dim * 3,
self._encoding_dim))
self._span_start_encoder = span_start_encoder
self._span_end_encoder = span_end_encoder
self._span_start_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1))
self._span_end_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1))
self._span_yesno_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 3))
self._span_followup_predictor = TimeDistributed(self._followup_lin)
check_dimensions_match(phrase_layer.get_input_dim(),
text_field_embedder.get_output_dim() +
marker_embedding_dim * num_context_answers,
"phrase layer input dim",
"embedding dim + marker dim * num context answers")
initializer(self)
self._span_start_accuracy = CategoricalAccuracy()
self._span_end_accuracy = CategoricalAccuracy()
self._span_yesno_accuracy = CategoricalAccuracy()
self._span_followup_accuracy = CategoricalAccuracy()
self._span_gt_yesno_accuracy = CategoricalAccuracy()
self._span_gt_followup_accuracy = CategoricalAccuracy()
self._span_accuracy = BooleanAccuracy()
self._official_f1 = Average()
self._variational_dropout = InputVariationalDropout(dropout)
def forward(self, # type: ignore
question: Dict[str, torch.LongTensor],
passage: Dict[str, torch.LongTensor],
span_start: torch.IntTensor = None,
span_end: torch.IntTensor = None,
p1_answer_marker: torch.IntTensor = None,
p2_answer_marker: torch.IntTensor = None,
p3_answer_marker: torch.IntTensor = None,
yesno_list: torch.IntTensor = None,
followup_list: torch.IntTensor = None,
metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Parameters
----------
question : Dict[str, torch.LongTensor]
From a ``TextField``.
passage : Dict[str, torch.LongTensor]
From a ``TextField``. The model assumes that this passage contains the answer to the
question, and predicts the beginning and ending positions of the answer within the
passage.
span_start : ``torch.IntTensor``, optional
From an ``IndexField``. This is one of the things we are trying to predict - the
beginning position of the answer with the passage. This is an `inclusive` token index.
If this is given, we will compute a loss that gets included in the output dictionary.
span_end : ``torch.IntTensor``, optional
From an ``IndexField``. This is one of the things we are trying to predict - the
ending position of the answer with the passage. This is an `inclusive` token index.
If this is given, we will compute a loss that gets included in the output dictionary.
p1_answer_marker : ``torch.IntTensor``, optional
This is one of the inputs, but only when num_context_answers > 0.
This is a tensor that has a shape [batch_size, max_qa_count, max_passage_length].
Most passage token will have assigned 'O', except the passage tokens belongs to the previous answer
in the dialog, which will be assigned labels such as <1_start>, <1_in>, <1_end>.
For more details, look into dataset_readers/util/make_reading_comprehension_instance_quac
p2_answer_marker : ``torch.IntTensor``, optional
This is one of the inputs, but only when num_context_answers > 1.
It is similar to p1_answer_marker, but marking previous previous answer in passage.
p3_answer_marker : ``torch.IntTensor``, optional
This is one of the inputs, but only when num_context_answers > 2.
It is similar to p1_answer_marker, but marking previous previous previous answer in passage.
yesno_list : ``torch.IntTensor``, optional
This is one of the outputs that we are trying to predict.
Three way classification (the yes/no/not a yes no question).
followup_list : ``torch.IntTensor``, optional
This is one of the outputs that we are trying to predict.
Three way classification (followup / maybe followup / don't followup).
metadata : ``List[Dict[str, Any]]``, optional
If present, this should contain the question ID, original passage text, and token
offsets into the passage for each instance in the batch. We use this for computing
official metrics using the official SQuAD evaluation script. The length of this list
should be the batch size, and each dictionary should have the keys ``id``,
``original_passage``, and ``token_offsets``. If you only want the best span string and
don't care about official metrics, you can omit the ``id`` key.
Returns
-------
An output dictionary consisting of the followings.
Each of the followings is a nested list because first iterates over dialog, then questions in dialog.
qid : List[List[str]]
A list of list, consisting of question ids.
followup : List[List[int]]
A list of list, consisting of continuation marker prediction index.
(y :yes, m: maybe follow up, n: don't follow up)
yesno : List[List[int]]
A list of list, consisting of affirmation marker prediction index.
(y :yes, x: not a yes/no question, n: np)
best_span_str : List[List[str]]
If sufficient metadata was provided for the instances in the batch, we also return the
string from the original passage that the model thinks is the best answer to the
question.
loss : torch.FloatTensor, optional
A scalar loss to be optimised.
"""
batch_size, max_qa_count, max_q_len, _ = question['token_characters'].size()
total_qa_count = batch_size * max_qa_count
qa_mask = torch.ge(followup_list, 0).view(total_qa_count)
embedded_question = self._text_field_embedder(question, num_wrapping_dims=1)
embedded_question = embedded_question.reshape(total_qa_count, max_q_len,
self._text_field_embedder.get_output_dim())
embedded_question = self._variational_dropout(embedded_question)
embedded_passage = self._variational_dropout(self._text_field_embedder(passage))
passage_length = embedded_passage.size(1)
question_mask = util.get_text_field_mask(question, num_wrapping_dims=1).float()
question_mask = question_mask.reshape(total_qa_count, max_q_len)
passage_mask = util.get_text_field_mask(passage).float()
repeated_passage_mask = passage_mask.unsqueeze(1).repeat(1, max_qa_count, 1)
repeated_passage_mask = repeated_passage_mask.view(total_qa_count, passage_length)
if self._num_context_answers > 0:
# Encode question turn number inside the dialog into question embedding.
question_num_ind = util.get_range_vector(max_qa_count, util.get_device_of(embedded_question))
question_num_ind = question_num_ind.unsqueeze(-1).repeat(1, max_q_len)
question_num_ind = question_num_ind.unsqueeze(0).repeat(batch_size, 1, 1)
question_num_ind = question_num_ind.reshape(total_qa_count, max_q_len)
question_num_marker_emb = self._question_num_marker(question_num_ind)
embedded_question = torch.cat([embedded_question, question_num_marker_emb], dim=-1)
# Encode the previous answers in passage embedding.
repeated_embedded_passage = embedded_passage.unsqueeze(1).repeat(1, max_qa_count, 1, 1). \
view(total_qa_count, passage_length, self._text_field_embedder.get_output_dim())
# batch_size * max_qa_count, passage_length, word_embed_dim
p1_answer_marker = p1_answer_marker.view(total_qa_count, passage_length)
p1_answer_marker_emb = self._prev_ans_marker(p1_answer_marker)
repeated_embedded_passage = torch.cat([repeated_embedded_passage, p1_answer_marker_emb], dim=-1)
if self._num_context_answers > 1:
p2_answer_marker = p2_answer_marker.view(total_qa_count, passage_length)
p2_answer_marker_emb = self._prev_ans_marker(p2_answer_marker)
repeated_embedded_passage = torch.cat([repeated_embedded_passage, p2_answer_marker_emb], dim=-1)
if self._num_context_answers > 2:
p3_answer_marker = p3_answer_marker.view(total_qa_count, passage_length)
p3_answer_marker_emb = self._prev_ans_marker(p3_answer_marker)
repeated_embedded_passage = torch.cat([repeated_embedded_passage, p3_answer_marker_emb],
dim=-1)
repeated_encoded_passage = self._variational_dropout(self._phrase_layer(repeated_embedded_passage,
repeated_passage_mask))
else:
encoded_passage = self._variational_dropout(self._phrase_layer(embedded_passage, passage_mask))
repeated_encoded_passage = encoded_passage.unsqueeze(1).repeat(1, max_qa_count, 1, 1)
repeated_encoded_passage = repeated_encoded_passage.view(total_qa_count,
passage_length,
self._encoding_dim)
encoded_question = self._variational_dropout(self._phrase_layer(embedded_question, question_mask))
# Shape: (batch_size * max_qa_count, passage_length, question_length)
passage_question_similarity = self._matrix_attention(repeated_encoded_passage, encoded_question)
# Shape: (batch_size * max_qa_count, passage_length, question_length)
passage_question_attention = util.masked_softmax(passage_question_similarity, question_mask)
# Shape: (batch_size * max_qa_count, passage_length, encoding_dim)
passage_question_vectors = util.weighted_sum(encoded_question, passage_question_attention)
# We replace masked values with something really negative here, so they don't affect the
# max below.
masked_similarity = util.replace_masked_values(passage_question_similarity,
question_mask.unsqueeze(1),
-1e7)
question_passage_similarity = masked_similarity.max(dim=-1)[0].squeeze(-1)
question_passage_attention = util.masked_softmax(question_passage_similarity, repeated_passage_mask)
# Shape: (batch_size * max_qa_count, encoding_dim)
question_passage_vector = util.weighted_sum(repeated_encoded_passage, question_passage_attention)
tiled_question_passage_vector = question_passage_vector.unsqueeze(1).expand(total_qa_count,
passage_length,
self._encoding_dim)
# Shape: (batch_size * max_qa_count, passage_length, encoding_dim * 4)
final_merged_passage = torch.cat([repeated_encoded_passage,
passage_question_vectors,
repeated_encoded_passage * passage_question_vectors,
repeated_encoded_passage * tiled_question_passage_vector],
dim=-1)
final_merged_passage = F.relu(self._merge_atten(final_merged_passage))
residual_layer = self._variational_dropout(self._residual_encoder(final_merged_passage,
repeated_passage_mask))
self_attention_matrix = self._self_attention(residual_layer, residual_layer)
mask = repeated_passage_mask.reshape(total_qa_count, passage_length, 1) \
* repeated_passage_mask.reshape(total_qa_count, 1, passage_length)
self_mask = torch.eye(passage_length, passage_length, device=self_attention_matrix.device)
self_mask = self_mask.reshape(1, passage_length, passage_length)
mask = mask * (1 - self_mask)
self_attention_probs = util.masked_softmax(self_attention_matrix, mask)
# (batch, passage_len, passage_len) * (batch, passage_len, dim) -> (batch, passage_len, dim)
self_attention_vecs = torch.matmul(self_attention_probs, residual_layer)
self_attention_vecs = torch.cat([self_attention_vecs, residual_layer,
residual_layer * self_attention_vecs],
dim=-1)
residual_layer = F.relu(self._merge_self_attention(self_attention_vecs))
final_merged_passage = final_merged_passage + residual_layer
# batch_size * maxqa_pair_len * max_passage_len * 200
final_merged_passage = self._variational_dropout(final_merged_passage)
start_rep = self._span_start_encoder(final_merged_passage, repeated_passage_mask)
span_start_logits = self._span_start_predictor(start_rep).squeeze(-1)
end_rep = self._span_end_encoder(torch.cat([final_merged_passage, start_rep], dim=-1),
repeated_passage_mask)
span_end_logits = self._span_end_predictor(end_rep).squeeze(-1)
span_yesno_logits = self._span_yesno_predictor(end_rep).squeeze(-1)
span_followup_logits = self._span_followup_predictor(end_rep).squeeze(-1)
span_start_logits = util.replace_masked_values(span_start_logits, repeated_passage_mask, -1e7)
# batch_size * maxqa_len_pair, max_document_len
span_end_logits = util.replace_masked_values(span_end_logits, repeated_passage_mask, -1e7)
best_span = self._get_best_span_yesno_followup(span_start_logits, span_end_logits,
span_yesno_logits, span_followup_logits,
self._max_span_length)
output_dict: Dict[str, Any] = {}
# Compute the loss.
if span_start is not None:
loss = nll_loss(util.masked_log_softmax(span_start_logits, repeated_passage_mask), span_start.view(-1),
ignore_index=-1)
self._span_start_accuracy(span_start_logits, span_start.view(-1), mask=qa_mask)
loss += nll_loss(util.masked_log_softmax(span_end_logits,
repeated_passage_mask), span_end.view(-1), ignore_index=-1)
self._span_end_accuracy(span_end_logits, span_end.view(-1), mask=qa_mask)
self._span_accuracy(best_span[:, 0:2],
torch.stack([span_start, span_end], -1).view(total_qa_count, 2),
mask=qa_mask.unsqueeze(1).expand(-1, 2).long())
# add a select for the right span to compute loss
gold_span_end_loc = []
span_end = span_end.view(total_qa_count).squeeze().data.cpu().numpy()
for i in range(0, total_qa_count):
gold_span_end_loc.append(max(span_end[i] * 3 + i * passage_length * 3, 0))
gold_span_end_loc.append(max(span_end[i] * 3 + i * passage_length * 3 + 1, 0))
gold_span_end_loc.append(max(span_end[i] * 3 + i * passage_length * 3 + 2, 0))
gold_span_end_loc = span_start.new(gold_span_end_loc)
pred_span_end_loc = []
for i in range(0, total_qa_count):
pred_span_end_loc.append(max(best_span[i][1] * 3 + i * passage_length * 3, 0))
pred_span_end_loc.append(max(best_span[i][1] * 3 + i * passage_length * 3 + 1, 0))
pred_span_end_loc.append(max(best_span[i][1] * 3 + i * passage_length * 3 + 2, 0))
predicted_end = span_start.new(pred_span_end_loc)
_yesno = span_yesno_logits.view(-1).index_select(0, gold_span_end_loc).view(-1, 3)
_followup = span_followup_logits.view(-1).index_select(0, gold_span_end_loc).view(-1, 3)
loss += nll_loss(F.log_softmax(_yesno, dim=-1), yesno_list.view(-1), ignore_index=-1)
loss += nll_loss(F.log_softmax(_followup, dim=-1), followup_list.view(-1), ignore_index=-1)
_yesno = span_yesno_logits.view(-1).index_select(0, predicted_end).view(-1, 3)
_followup = span_followup_logits.view(-1).index_select(0, predicted_end).view(-1, 3)
self._span_yesno_accuracy(_yesno, yesno_list.view(-1), mask=qa_mask)
self._span_followup_accuracy(_followup, followup_list.view(-1), mask=qa_mask)
output_dict["loss"] = loss
# Compute F1 and preparing the output dictionary.
output_dict['best_span_str'] = []
output_dict['qid'] = []
output_dict['followup'] = []
output_dict['yesno'] = []
best_span_cpu = best_span.detach().cpu().numpy()
for i in range(batch_size):
passage_str = metadata[i]['original_passage']
offsets = metadata[i]['token_offsets']
f1_score = 0.0
per_dialog_best_span_list = []
per_dialog_yesno_list = []
per_dialog_followup_list = []
per_dialog_query_id_list = []
for per_dialog_query_index, (iid, answer_texts) in enumerate(
zip(metadata[i]["instance_id"], metadata[i]["answer_texts_list"])):
predicted_span = tuple(best_span_cpu[i * max_qa_count + per_dialog_query_index])
start_offset = offsets[predicted_span[0]][0]
end_offset = offsets[predicted_span[1]][1]
yesno_pred = predicted_span[2]
followup_pred = predicted_span[3]
per_dialog_yesno_list.append(yesno_pred)
per_dialog_followup_list.append(followup_pred)
per_dialog_query_id_list.append(iid)
best_span_string = passage_str[start_offset:end_offset]
per_dialog_best_span_list.append(best_span_string)
if answer_texts:
if len(answer_texts) > 1:
t_f1 = []
# Compute F1 over N-1 human references and averages the scores.
for answer_index in range(len(answer_texts)):
idxes = list(range(len(answer_texts)))
idxes.pop(answer_index)
refs = [answer_texts[z] for z in idxes]
t_f1.append(squad_eval.metric_max_over_ground_truths(squad_eval.f1_score,
best_span_string,
refs))
f1_score = 1.0 * sum(t_f1) / len(t_f1)
else:
f1_score = squad_eval.metric_max_over_ground_truths(squad_eval.f1_score,
best_span_string,
answer_texts)
self._official_f1(100 * f1_score)
output_dict['qid'].append(per_dialog_query_id_list)
output_dict['best_span_str'].append(per_dialog_best_span_list)
output_dict['yesno'].append(per_dialog_yesno_list)
output_dict['followup'].append(per_dialog_followup_list)
return output_dict
@overrides
def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, Any]:
yesno_tags = [[self.vocab.get_token_from_index(x, namespace="yesno_labels") for x in yn_list] \
for yn_list in output_dict.pop("yesno")]
followup_tags = [[self.vocab.get_token_from_index(x, namespace="followup_labels") for x in followup_list] \
for followup_list in output_dict.pop("followup")]
output_dict['yesno'] = yesno_tags
output_dict['followup'] = followup_tags
return output_dict
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
return {'start_acc': self._span_start_accuracy.get_metric(reset),
'end_acc': self._span_end_accuracy.get_metric(reset),
'span_acc': self._span_accuracy.get_metric(reset),
'yesno': self._span_yesno_accuracy.get_metric(reset),
'followup': self._span_followup_accuracy.get_metric(reset),
'f1': self._official_f1.get_metric(reset), }
@staticmethod
def _get_best_span_yesno_followup(span_start_logits: torch.Tensor,
span_end_logits: torch.Tensor,
span_yesno_logits: torch.Tensor,
span_followup_logits: torch.Tensor,
max_span_length: int) -> torch.Tensor:
# Returns the index of highest-scoring span that is not longer than 30 tokens, as well as
# yesno prediction bit and followup prediction bit from the predicted span end token.
if span_start_logits.dim() != 2 or span_end_logits.dim() != 2:
raise ValueError("Input shapes must be (batch_size, passage_length)")
batch_size, passage_length = span_start_logits.size()
max_span_log_prob = [-1e20] * batch_size
span_start_argmax = [0] * batch_size
best_word_span = span_start_logits.new_zeros((batch_size, 4), dtype=torch.long)
span_start_logits = span_start_logits.data.cpu().numpy()
span_end_logits = span_end_logits.data.cpu().numpy()
span_yesno_logits = span_yesno_logits.data.cpu().numpy()
span_followup_logits = span_followup_logits.data.cpu().numpy()
for b_i in range(batch_size): # pylint: disable=invalid-name
for j in range(passage_length):
val1 = span_start_logits[b_i, span_start_argmax[b_i]]
if val1 < span_start_logits[b_i, j]:
span_start_argmax[b_i] = j
val1 = span_start_logits[b_i, j]
val2 = span_end_logits[b_i, j]
if val1 + val2 > max_span_log_prob[b_i]:
if j - span_start_argmax[b_i] > max_span_length:
continue
best_word_span[b_i, 0] = span_start_argmax[b_i]
best_word_span[b_i, 1] = j
max_span_log_prob[b_i] = val1 + val2
for b_i in range(batch_size):
j = best_word_span[b_i, 1]
yesno_pred = np.argmax(span_yesno_logits[b_i, j])
followup_pred = np.argmax(span_followup_logits[b_i, j])
best_word_span[b_i, 2] = int(yesno_pred)
best_word_span[b_i, 3] = int(followup_pred)
return best_word_span
