def construct_requests(self, doc, ctx):
""" Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param ctx: str
The context string, generated by fewshot_context. This includes the natural
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
problem = self.last_problem(doc)
ll_choices = [
rf.loglikelihood(ctx, " " + problem['options'][i])[0]
for i in range(4)
]
return ll_choices
def construct_requests(self, doc, ctx):
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param ctx: str
The context string, generated by fewshot_context. This includes the natural
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
target = self.partial_target(doc)
lls = []
for option in [doc["option1"], doc["option2"]]:
partial_ctx = self.partial_context(doc, option)
full_ctx = self.append_context(ctx, partial_ctx)
lls.append(rf.loglikelihood(full_ctx, target)[0])
return lls
def construct_requests(self, doc, ctx):
"""Uses RequestFactory to construct Requests and returns an iterable of
Requests which will be sent to the LM.
:param doc:
The document as returned from training_docs, validation_docs, or test_docs.
:param ctx: str
The context string, generated by fewshot_context. This includes the natural
language description, as well as the few shot examples, and the question
part of the document for `doc`.
"""
lls = []
for option in doc["options"]:
# Following Section 4.4 "Recurrent Language Models" in the CBT paper:
# "we rank candidate [option] c based on p(q1 . . . qk−1, c, qk+1 . . . ql)
# rather than simply p(q1 . . . qk−1, c)."
lls.append(rf.loglikelihood("", ctx.replace("XXXXX", option))[0])
return lls
def construct_requests(self, doc, ctx):
ll_1, _ = rf.loglikelihood(ctx, " " + doc['sol1'])
ll_2, _ = rf.loglikelihood(ctx, " " + doc['sol2'])
return ll_1, ll_2
def construct_requests(self, doc, ctx):
ll, is_greedy = rf.loglikelihood(ctx, self.doc_to_target(doc))
return ll, is_greedy
def construct_requests(self, doc, ctx):
ll, is_prediction = rf.loglikelihood(ctx, doc.completion)
return is_prediction
def construct_requests(self, doc, ctx):
ret = []
for alias in self._remove_prefixes(doc["answer"]["aliases"]):
_, is_prediction = rf.loglikelihood(ctx, " " + alias)
ret.append(is_prediction)
return ret
def construct_requests(self, doc, ctx):
ll_positive, _ = rf.loglikelihood(ctx, " Positive")
ll_negative, _ = rf.loglikelihood(ctx, " Negative")
return ll_positive, ll_negative
def construct_requests(self, doc, ctx):
ll_yes, _ = rf.loglikelihood(ctx, " yes")
ll_no, _ = rf.loglikelihood(ctx, " no")
return ll_yes, ll_no
def get_example(self, doc, ctx):
gold_req, _ = rf.loglikelihood(
ctx, " False") if doc["label"] else rf.loglikelihood(ctx, " True")
example = {"context": gold_req.args[0], "completion": gold_req.args[1]}
return example
def construct_requests(self, doc, ctx):
ll_true, _ = rf.loglikelihood(ctx, ' True')
ll_neither, _ = rf.loglikelihood(ctx, ' Neither')
ll_false, _ = rf.loglikelihood(ctx, ' False')
return ll_true, ll_neither, ll_false
def construct_requests(self, doc, ctx):
requests = [
rf.loglikelihood(ctx, self.format_answer(query=doc["query"], entity=entity))
for entity in doc["entities"]
]
return requests
def construct_requests(self, doc, ctx):
ll_yes, _ = rf.loglikelihood(ctx, " reasonable")
ll_no, _ = rf.loglikelihood(ctx, " unreasonable")
return ll_yes, ll_no
def construct_requests(self, doc, ctx):
lls = []
for option in doc["options"]:
lls.append(rf.loglikelihood(ctx, f" {self.detokenize(option)}")[0])
return lls
def construct_requests(self, doc, ctx):
ll_true_choice, _ = rf.loglikelihood(ctx, f' yes')
ll_false_choice, _ = rf.loglikelihood(ctx, f' no')
return ll_false_choice, ll_true_choice
def construct_requests(self, doc, ctx):
ll_positive, _ = rf.loglikelihood(ctx, " great")
ll_negative, _ = rf.loglikelihood(ctx, " terrible")
return ll_positive, ll_negative
def construct_requests(self, doc, ctx):
ll_true, _ = rf.loglikelihood(ctx + " can say that",
f" {self._convert_completion(doc)}")
ll_false, _ = rf.loglikelihood(ctx + " can't say that",
f" {self._convert_completion(doc)}")
return ll_true, ll_false
def construct_requests(self, doc, ctx):
ll_yes, _ = rf.loglikelihood(ctx, ' yes')
ll_no, _ = rf.loglikelihood(ctx, ' no')
return ll_yes, ll_no
def get_lls(targets):
return [rf.loglikelihood(ctx, " " + t)[0] for t in targets]
def construct_requests(self, doc, ctx):
ll_true, _ = rf.loglikelihood(ctx, " True")
ll_false, _ = rf.loglikelihood(ctx, " False")
return ll_true, ll_false
def construct_requests(self, doc, ctx):
ll_true, _ = rf.loglikelihood(ctx, " true")
ll_neither, _ = rf.loglikelihood(ctx, " neither")
ll_false, _ = rf.loglikelihood(ctx, " false")
return ll_true, ll_neither, ll_false
