def construct_requests(self, doc, ctx): """ Uses RequestFactory to construct Requests and returns an iterable of Requests which will be sent to the LM. :param doc: The document as returned from training_docs, validation_docs, or test_docs. :param ctx: str The context string, generated by fewshot_context. This includes the natural language description, as well as the few shot examples, and the question part of the document for `doc`. """ problem = self.last_problem(doc) ll_choices = [ rf.loglikelihood(ctx, " " + problem['options'][i])[0] for i in range(4) ] return ll_choices
def construct_requests(self, doc, ctx): """Uses RequestFactory to construct Requests and returns an iterable of Requests which will be sent to the LM. :param doc: The document as returned from training_docs, validation_docs, or test_docs. :param ctx: str The context string, generated by fewshot_context. This includes the natural language description, as well as the few shot examples, and the question part of the document for `doc`. """ target = self.partial_target(doc) lls = [] for option in [doc["option1"], doc["option2"]]: partial_ctx = self.partial_context(doc, option) full_ctx = self.append_context(ctx, partial_ctx) lls.append(rf.loglikelihood(full_ctx, target)[0]) return lls
def construct_requests(self, doc, ctx): """Uses RequestFactory to construct Requests and returns an iterable of Requests which will be sent to the LM. :param doc: The document as returned from training_docs, validation_docs, or test_docs. :param ctx: str The context string, generated by fewshot_context. This includes the natural language description, as well as the few shot examples, and the question part of the document for `doc`. """ lls = [] for option in doc["options"]: # Following Section 4.4 "Recurrent Language Models" in the CBT paper: # "we rank candidate [option] c based on p(q1 . . . qk−1, c, qk+1 . . . ql) # rather than simply p(q1 . . . qk−1, c)." lls.append(rf.loglikelihood("", ctx.replace("XXXXX", option))[0]) return lls
def construct_requests(self, doc, ctx): ll_1, _ = rf.loglikelihood(ctx, " " + doc['sol1']) ll_2, _ = rf.loglikelihood(ctx, " " + doc['sol2']) return ll_1, ll_2
def construct_requests(self, doc, ctx): ll, is_greedy = rf.loglikelihood(ctx, self.doc_to_target(doc)) return ll, is_greedy
def construct_requests(self, doc, ctx): ll, is_prediction = rf.loglikelihood(ctx, doc.completion) return is_prediction
def construct_requests(self, doc, ctx): ret = [] for alias in self._remove_prefixes(doc["answer"]["aliases"]): _, is_prediction = rf.loglikelihood(ctx, " " + alias) ret.append(is_prediction) return ret
def construct_requests(self, doc, ctx): ll_positive, _ = rf.loglikelihood(ctx, " Positive") ll_negative, _ = rf.loglikelihood(ctx, " Negative") return ll_positive, ll_negative
def construct_requests(self, doc, ctx): ll_yes, _ = rf.loglikelihood(ctx, " yes") ll_no, _ = rf.loglikelihood(ctx, " no") return ll_yes, ll_no
def get_example(self, doc, ctx): gold_req, _ = rf.loglikelihood( ctx, " False") if doc["label"] else rf.loglikelihood(ctx, " True") example = {"context": gold_req.args[0], "completion": gold_req.args[1]} return example
def construct_requests(self, doc, ctx): ll_true, _ = rf.loglikelihood(ctx, ' True') ll_neither, _ = rf.loglikelihood(ctx, ' Neither') ll_false, _ = rf.loglikelihood(ctx, ' False') return ll_true, ll_neither, ll_false
def construct_requests(self, doc, ctx): requests = [ rf.loglikelihood(ctx, self.format_answer(query=doc["query"], entity=entity)) for entity in doc["entities"] ] return requests
def construct_requests(self, doc, ctx): ll_yes, _ = rf.loglikelihood(ctx, " reasonable") ll_no, _ = rf.loglikelihood(ctx, " unreasonable") return ll_yes, ll_no
def construct_requests(self, doc, ctx): lls = [] for option in doc["options"]: lls.append(rf.loglikelihood(ctx, f" {self.detokenize(option)}")[0]) return lls
def construct_requests(self, doc, ctx): ll_true_choice, _ = rf.loglikelihood(ctx, f' yes') ll_false_choice, _ = rf.loglikelihood(ctx, f' no') return ll_false_choice, ll_true_choice
def construct_requests(self, doc, ctx): ll_positive, _ = rf.loglikelihood(ctx, " great") ll_negative, _ = rf.loglikelihood(ctx, " terrible") return ll_positive, ll_negative
def construct_requests(self, doc, ctx): ll_true, _ = rf.loglikelihood(ctx + " can say that", f" {self._convert_completion(doc)}") ll_false, _ = rf.loglikelihood(ctx + " can't say that", f" {self._convert_completion(doc)}") return ll_true, ll_false
def construct_requests(self, doc, ctx): ll_yes, _ = rf.loglikelihood(ctx, ' yes') ll_no, _ = rf.loglikelihood(ctx, ' no') return ll_yes, ll_no
def get_lls(targets): return [rf.loglikelihood(ctx, " " + t)[0] for t in targets]
def construct_requests(self, doc, ctx): ll_true, _ = rf.loglikelihood(ctx, " True") ll_false, _ = rf.loglikelihood(ctx, " False") return ll_true, ll_false
def construct_requests(self, doc, ctx): ll_true, _ = rf.loglikelihood(ctx, " true") ll_neither, _ = rf.loglikelihood(ctx, " neither") ll_false, _ = rf.loglikelihood(ctx, " false") return ll_true, ll_neither, ll_false