def predict_zero_shot_2afc(self, sentence, model1, model2):
"""
Yield zero-shot predictions on a 2AFC sentence, marginalizing over possible
novel lexical entries required to parse the sentence.
TODO explain marginalization process in more detail
Args:
sentence: List of token strings
models:
Returns:
model_scores: `Distribution` over scene models (with support `models`),
`p(referred scene | sentence)`
"""
parser = chart.WeightedCCGChartParser(self.lexicon)
weighted_results = parser.parse(sentence, True)
if len(weighted_results) == 0:
L.warning("Parse failed for sentence '%s'", " ".join(sentence))
aug_lexicon = self.do_lexical_induction(
sentence, (model1, model2),
augment_lexicon_fn=augment_lexicon_2afc,
queue_limit=50)
parser = chart.WeightedCCGChartParser(aug_lexicon)
weighted_results = parser.parse(sentence, True)
dist = Distribution()
for result, score, _ in weighted_results:
semantics = result.label()[0].semantics()
try:
model1_pass = model1.evaluate(semantics) == True
except:
pass
else:
if model1_pass:
dist[model1] += np.exp(score)
try:
model2_pass = model2.evaluate(semantics) == True
except:
pass
else:
if model2_pass:
dist[model2] += np.exp(score)
return dist.ensure_support((model1, model2)).normalize()
def score_cat_assignment(cat_assignment):
"""
Assign a log-probability to a joint assignment of categories to tokens.
"""
for token, category in zip(tokens, cat_assignment):
lex.set_entries(token, [(category, None, 0.001)])
# Attempt a parse.
results = chart.WeightedCCGChartParser(lex, chart.DefaultRuleSet) \
.parse(sentence, return_aux=True)
if results:
# Prior weight for category comes from lexicon.
#
# Might also downweight categories which require type-lifting parses by
# default?
score = 0
for token, category in zip(tokens, cat_assignment):
score += np.log(category_prior[category])
# Likelihood weight comes from parse score
score += np.log(sum(np.exp(weight) for _, weight, _ in results))
return score
return -np.inf
def attempt_candidate_parse(lexicon, tokens, candidate_categories,
sentence, scorer, dummy_vars, sentence_meta=None):
"""
Attempt to parse a sentence, mapping `tokens` to new candidate
lexical entries.
Arguments:
lexicon: Current lexicon. Will modify in place -- send a copy.
tokens: List of string token(s) to be attempted.
candidate_categories: List of candidate categories for each token (one per
token).
sentence: Sentence which we are attempting to parse.
"""
get_arity = (lexicon.ontology and lexicon.ontology.get_expr_arity) \
or get_semantic_arity
# Prepare dummy variable which will be inserted into parse checks.
sub_exprs = {token: l.FunctionVariableExpression(dummy_vars[token])
for token in tokens}
lexicon = lexicon.clone()
for token, syntax in zip(tokens, candidate_categories):
lexicon.set_entries(token, [(syntax, sub_exprs[token], 1.0)])
# Reconstruct scorer with this modified lexicon.
scorer = scorer.clone_with_lexicon(lexicon)
parse_results = []
# First attempt a parse with only function application rules.
parser = chart.WeightedCCGChartParser(lexicon, scorer=scorer, ruleset=chart.ApplicationRuleSet)
results = parser.parse(sentence, sentence_meta=sentence_meta)
if True:#results or not allow_composition:
return results
def attempt_candidate_parse(lexicon, tokens, candidate_categories, sentence,
dummy_vars):
"""
Attempt to parse a sentence, mapping `tokens` to new candidate
lexical entries.
Arguments:
lexicon: Current lexicon. Will modify in place -- send a copy.
tokens: List of string token(s) to be attempted.
candidate_categories: List of candidate categories for each token (one per
token).
sentence: Sentence which we are attempting to parse.
"""
get_arity = (lexicon.ontology and lexicon.ontology.get_expr_arity) \
or get_semantic_arity
# Prepare dummy variable which will be inserted into parse checks.
token_vars = {token: copy(dummy_vars[token]) for token in tokens}
var_to_token = {var.name: token for token, var in token_vars.items()}
sub_exprs = {
token: l.FunctionVariableExpression(token_var)
for token, token_var in token_vars.items()
}
lexicon = lexicon.clone()
for token, syntax in zip(tokens, candidate_categories):
lexicon.set_entries(token, [(syntax, sub_exprs[token], 1.0)])
parse_results = []
# First attempt a parse with only function application rules.
results = chart.WeightedCCGChartParser(lexicon, ruleset=chart.ApplicationRuleSet) \
.parse(sentence)
for result in results:
# Retrieve apparent types of each token's dummy var.
apparent_types = defaultdict(list)
def visit(expr):
if isinstance(expr, l.FunctionVariableExpression):
if expr.variable.name in var_to_token:
apparent_types[var_to_token[expr.variable.name]].append(
expr.type)
elif isinstance(expr, l.ApplicationExpression):
visit(expr.pred)
for arg in expr.args:
visit(arg)
elif isinstance(expr, l.LambdaExpression):
visit(expr.variable)
visit(expr.term)
sem = result.label()[0].semantics()
visit(sem)
# Make sure we inferred a type for every dummy var.
assert (set(apparent_types.keys()) == set(token_vars.keys()))
yield result, apparent_types
def predict_zero_shot_nscl(self, sentence, model, sentence_meta, answer,
augment_lexicon_args):
"""Yield expected zero_shot accuracy on a novel sentence, marginalizing over
possible novel lexical entries required to parse the sentence.
"""
kwargs = {"answer": answer}
augment_lexicon_args.update(kwargs)
parser = self.make_parser()
weighted_results = parser.parse(sentence,
sentence_meta=sentence_meta,
return_aux=True)
if len(weighted_results) == 0:
L.warning("Parse failed for sentence '%s'", " ".join(sentence))
aug_lexicon = self.do_lexical_induction(
sentence,
model=model,
augment_lexicon_fn=augment_lexicon_nscl,
sentence_meta=sentence_meta,
**augment_lexicon_args)
# NB, we don't re-instantiate the scorer with the novel lexicon. That's
# actually okay for the frame-based inference, since the frame--meaning
# knowledge was baked into the initial weights of the lexicon during
# induction. But this is hacky, and should be fixed when the rest of the
# induction--learning dissociation is finally addressed.
parser = chart.WeightedCCGChartParser(lexicon=aug_lexicon,
scorer=self.scorer,
ruleset=chart.DefaultRuleSet)
weighted_results = parser.parse(sentence,
sentence_meta=sentence_meta,
return_aux=True)
else:
aug_lexicon = self.lexicon
# Marginalize over expected weighted results. # Try only top N?
if len(weighted_results) == 0:
return aug_lexicon, {0: 0.0}
rewards = [
_update_distant_success_fn(result, model, answer)[1]
for result, _, _ in weighted_results
]
success = T.stack([T.tensor(reward) for reward in rewards])
probs = T.exp(
T.stack([logp.detach() for _, logp, _ in weighted_results]))
probs = probs / T.sum(probs)
probs, success = probs.numpy(), success.numpy()
# Consider only the top N
def top_n_expected(top_n):
top_n_probs, top_n_success = probs[np.argsort(
-probs)][:top_n], success[np.argsort(-probs)][:top_n]
top_n_probs /= np.sum(top_n_probs)
return np.sum((top_n_probs * top_n_success))
top_n = {n: top_n_expected(n) for n in [5, 10, 20, 50, 100, 1000]}
return aug_lexicon, top_n
def make_parser(self, lexicon=None, ruleset=chart.DefaultRuleSet):
"""
Construct a CCG parser from the current learner state.
"""
if lexicon is None:
lexicon = self.lexicon
return chart.WeightedCCGChartParser(lexicon=lexicon,
scorer=self.scorer,
ruleset=ruleset)
def score_cat_assignment(cat_assignment):
"""
Calculate a log-probability for a joint assignment of categories to tokens.
"""
for token, category in zip(tokens, cat_assignment):
lex.set_entries(token, [(category, None, 0.001)])
new_scorer = scorer.clone_with_lexicon(lex)
# Attempt a parse.
parser = chart.WeightedCCGChartParser(lex, scorer=new_scorer, ruleset=chart.DefaultRuleSet)
results = parser.parse(sentence, sentence_meta=sentence_meta, return_aux=True)
if len(results) == 0:
return FAIL
# Get total probability mass of legal parses.
logp = T.logsumexp(T.stack([logp for _, logp, _ in results]), 0)
return logp
def filter_lexicon_entry(lexicon, entry, sentence, lf):
"""
Filter possible syntactic/semantic mappings for a given lexicon entry s.t.
the given sentence renders the given LF, holding everything else
constant.
This process is of course not fail-safe -- the rest of the lexicon must
provide the necessary definitions to guarantee that any valid parse can
result.
Args:
lexicon: CCGLexicon
entry: string word
sentence: list of word tokens, must contain `entry`
lf: logical form string
"""
if entry not in sentence:
raise ValueError("Sentence does not contain given entry")
entry_idxs = [i for i, val in enumerate(sentence) if val == entry]
parse_results = chart.WeightedCCGChartParser(lexicon).parse(sentence, True)
valid_cands = [set() for _ in entry_idxs]
for _, _, edge_cands in parse_results:
for entry_idx, valid_cands_set in zip(entry_idxs, valid_cands):
valid_cands_set.add(edge_cands[entry_idx])
# Find valid interpretations across all uses of the word in the
# sentence.
valid_cands = list(reduce(lambda x, y: x & y, valid_cands))
if not valid_cands:
raise ValueError("no consistent interpretations of word found.")
new_lex = lexicon.clone()
new_lex.set_entries(entry, [
(cand.token().categ(), cand.token().semantics(), cand.token().weight())
for cand in valid_cands
])
return new_lex
def update_perceptron_batch(lexicon, data, learning_rate=0.1, parser=None):
"""
Execute a batch perceptron weight update with the given training data.
Args:
lexicon: CCGLexicon with weights
data: List of `(x, y)` tuples, where `x` is a list of string
tokens and `y` is an LF string.
learning_rate:
Returns:
l2 norm of total weight updates
"""
if parser is None:
parser = chart.WeightedCCGChartParser(lexicon)
norm = 0.0
for x, y in data:
weighted_results = parser.parse(x, return_aux=True)
max_result, max_score, _ = weighted_results[0]
correct_result, correct_score = None, None
for result, score, _ in weighted_results:
root_token, _ = result.label()
if str(root_token.semantics()) == y:
correct_result, correct_score = result, score
break
else:
raise NoParsesError("no valid parse derived", x)
if correct_score < max_score:
for result, sign in zip([correct_result, max_result], [1, -1]):
for _, leaf_token in result.pos():
delta = sign * learning_rate
norm += delta**2
leaf_token._weight += delta
return norm
def make_parser(self, ruleset=chart.DefaultRuleSet):
"""
Construct a CCG parser from the current learner state.
"""
return chart.WeightedCCGChartParser(self.lexicon, ruleset=ruleset)
def update_perceptron(lexicon,
sentence,
model,
success_fn,
learning_rate=10,
parser=None,
update_method="perceptron"):
if parser is None:
parser = chart.WeightedCCGChartParser(lexicon,
ruleset=chart.DefaultRuleSet)
norm = 0.0
weighted_results = parser.parse(sentence, return_aux=True)
if not weighted_results:
raise NoParsesError("No successful parses computed.", sentence)
max_score, max_incorrect_score = -np.inf, -np.inf
correct_results, incorrect_results = [], []
for result, score, _ in weighted_results:
success, answer_score = success_fn(result, model)
if success:
score += answer_score
if score > max_score:
max_score = score
correct_results = [(score, result)]
elif score == max_score:
correct_results.append((score, result))
else:
if score > max_incorrect_score:
max_incorrect_score = score
incorrect_results = [(score, result)]
elif score == max_incorrect_score:
incorrect_results.append((score, result))
if not correct_results:
raise NoParsesError("No parses derived are successful.", sentence)
elif not incorrect_results:
L.warning("No incorrect parses. Skipping update.")
return weighted_results, 0.0
# Sort results by descending parse score.
correct_results = sorted(correct_results, key=lambda r: r[0], reverse=True)
incorrect_results = sorted(incorrect_results,
key=lambda r: r[0],
reverse=True)
if update_method == "perceptron":
correct_results = correct_results[:1]
incorrect_results = incorrect_results[:1]
# TODO margin?
# Update to separate max-scoring parse from max-scoring correct parse if
# necessary.
positive_mass = 1 / len(correct_results)
negative_mass = 1 / len(incorrect_results)
token_deltas = Counter()
observed_leaf_sequences = set()
for results, delta in zip([correct_results, incorrect_results],
[positive_mass, -negative_mass]):
parse_results = [r[1] for r in results]
if update_method == "reinforce":
parse_scores = delta * softmax(np.array([r[0] for r in results]))
else:
parse_scores = np.repeat(delta, len(results))
for score_delta, result in zip(parse_scores, parse_results):
leaf_seq = tuple(leaf_token for _, leaf_token in result.pos())
if leaf_seq not in observed_leaf_sequences:
observed_leaf_sequences.add(leaf_seq)
for leaf_token in leaf_seq:
token_deltas[leaf_token] += score_delta
for token, delta in token_deltas.items():
delta *= learning_rate
norm += delta**2
L.info("Applying delta: %+.03f %s", delta, token)
token._weight += delta
return weighted_results, norm
