def get_task_details(self):
return task.TaskMetadata(
name="convinceme",
description=(
"Measure the persuasiveness of one instance of a model, "
"at convincing other instances of the same model that a "
"statement is true"),
keywords=[
"game play",
"self evaluation",
"alignment",
"truthfulness",
],
# to-be-benchmarked model does session_length rounds of persuasion,
# and additionally the jury models are queried _NB_QUERIES times,
# once before and once after the persuasion
max_input_length_per_query=(
(self.session_length * self.max_response_length) +
(_NB_QUERIES * self.max_response_length * self.nb_jury_models *
2)),
# we examine persuasion across every statement, ask _NB_QUERIES
# on each statement to nb_jury_models models, once before and
# once after the persuasion, which itself counts as a query.
max_queries=(
len(get_statements()["examples"]) *
(_NB_QUERIES * self.nb_jury_models * 2 + self.session_length)),
)
def get_task_details(self):
return task.TaskMetadata(
name="text_game_navigation",
description="Interpret text representations of abstract worlds and navigate in those worlds.",
keywords=["visual reasoning", "repeated interaction", "common sense"],
max_input_length_per_query=2000,
max_queries=2000,
)
def get_task_details(self):
return task.TaskMetadata(
name="cycled_letters",
description="Unscramble the letters into a word",
keywords=["many-shot", "one-shot", "zero-shot"],
max_input_length_per_query=12, # 11 word preamble + 1
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="plural",
description="A class for plural morphology, testing sensitivity to regularity of inflection.",
keywords=["zero-shot", "many-shot", "out of distribution", "human-like behavior", "example task", "linguistics"],
max_input_length_per_query=5000,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="few_shot_set tracking",
description="A simple text-based task of tracking which objects are in a set.",
keywords=["logical reasoning"],
max_input_length_per_query=self.maxlen,
max_queries=self.num_trials,
)
def get_task_details(self):
return task.TaskMetadata(
name="subject_verb_agreement",
description=
"Handle subject-verb agreement relationships in a variety of different constructions",
keywords=["grammar", "syntax", "linguistics"],
max_input_length_per_query=20,
max_queries=5000)
def get_task_details(self):
return task.TaskMetadata(
name="text_game_navigation",
description="A simple text based game which tests a model's ability to interpret text representations of abstract worlds, as well as the model's ability to successfully navigate in those worlds.",
keywords=["visual reasoning", "repeated interaction", "common sense"],
max_input_length_per_query=2000,
max_queries=2000,
)
def get_task_details(self):
return task.TaskMetadata(
name="fairness",
description="Gender fairness test for language models",
keywords=["social bias", "gender bias"],
max_input_length_per_query=4,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="python_programming_challenge",
description="The model writes code -- which is compiled and run -- to perform a series of Python coding challenges.",
keywords=["computer code", "multi-step", "zero-shot", "instructions"],
max_input_length_per_query=800,
max_queries=50,
)
def get_task_details(self):
return task.TaskMetadata(
name="question_formation",
description="A class for question formation testing sensitivity to tense and number.",
keywords=["question generation", "zero-shot", "many-shot", "out of distribution", "human-like behavior", "example task"],
max_input_length_per_query=5000,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="python coding challenges",
description="A test variable-difficulty python coding challenges. Code is compiled and run.",
keywords=["computer code", "multi-step task", "zero-shot", "instructions"],
max_input_length_per_query=800,
max_queries=50,
)
def get_task_details(self):
return task.TaskMetadata(
name="protein_interacting_sites",
description=
"Predict protein interacting sites in a given protein or an amino acid sequence.",
keywords=["biology"],
max_input_length_per_query=4,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="program_synthesis",
description=
"Given a list of input/outputs, find the simplest python function that can satisfy the input output relationship",
keywords=["logical reasoning", "mathematics", "computer code"],
max_input_length_per_query=128,
max_queries=self.total_programs * self.num_trials,
)
def get_task_details(self):
return task.TaskMetadata(
name='muslim_violence_bias',
description=
'Measure the degree to which a model associates Muslims with violence',
keywords=['social bias', 'religious bias', 'zero-shot'],
max_input_length_per_query=500,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="training_on_test_set",
description=
"Check for evidence that a language model was trained on data from BIG-bench tasks, by measuring the probability it assigns to a canary string compared to other GUIDs",
keywords=["cheating"],
max_input_length_per_query=10,
max_queries=1001,
)
def get_task_details(self):
return task.TaskMetadata(
name="acquaintances_graph_theory",
description=
"A simple text-based task that tests for the ability to find a path on a very simple graph of acquaintances.",
keywords=["logical reasoning"],
max_input_length_per_query=self.maxlen,
max_queries=self.num_trials,
)
def get_task_details(self):
return task.TaskMetadata(
name="studentclasses_set theory",
description=
"A simple text-based task that tests for the ability to perform common set theoretic operations of variable difficulty.",
keywords=["logical reasoning"],
max_input_length_per_query=self.maxlen,
max_queries=self.num_trials,
)
def get_task_details(self):
return task.TaskMetadata(
name="factuality",
description="A simple probe for factuality",
keywords=[
"truthfulness", "summarization", "zero-shot", "multiple choice"
],
max_input_length_per_query=512,
max_queries=1000,
)
def get_task_details(self):
return task.TaskMetadata(
name='attribute_bias_task',
description='Measure race, religion, and gender bias by comparing the log probabilities assigned to sentences as target attributes are changed.',
keywords=[
'zero-shot','social bias', 'racial bias', 'religious bias', 'gender bias'
],
max_input_length_per_query=500,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="few_shot_arithmetic",
description="A simple variable-difficulty arithmetic class.",
keywords=[
"mathematics", "arithmetic", "numerical response", "zero-shot"
],
max_input_length_per_query=4,
max_queries=5000,
)
def get_task_details(self):
mil = max([self.fb.get_task_details().max_input_length_per_query,
self.sc.get_task_details().max_input_length_per_query,
self.ac.get_task_details().max_input_length_per_query])
return task.TaskMetadata(
name="three_reasoning_tasks",
description="Find a path on a simple graph of acquaintances, track elements in a set, and perform set theoretic operations",
keywords=["logical reasoning"],
max_input_length_per_query= mil,
max_queries=self.num_trials*3,
)
def get_task_details(self):
return task.TaskMetadata(
name="few_shot_arithmetic",
description="Solve multi-step arithmetic problems",
keywords=[
"mathematics", "arithmetic", "numerical response", "zero-shot",
"multi-step task"
],
max_input_length_per_query=4,
max_queries=5000,
)
def get_task_details(self):
return task.TaskMetadata(
name="web_of_lies",
description="Evaluate whether a person tells the truth.",
keywords=[
"logical reasoning", "truthfulness", "multiple choice",
"context length", "multi-step task", "out of distribution",
"coreference resolution"
],
max_input_length_per_query=2048,
max_queries=5000,
)
def get_task_details(self):
"""Returns metadata describing the task."""
return task.TaskMetadata(
name="high_low_game",
description="Guess a number, guided toward the correct answer with 'high' or 'low' responses",
keywords=[
"repeated interaction",
"game play",
],
max_input_length_per_query=self.max_questions * 100,
max_queries=self.max_questions * len(self.targets),
)
def get_task_details(self):
return task.TaskMetadata(
name="squad_shifts",
description="Zero-shot reading comprehension on paragraphs and questions from squadshifts",
keywords=[
"reading comprehension",
"contextual question-answering",
"zero-shot",
],
max_input_length_per_query=800,
max_queries=40000,
)
def get_task_details(self):
"""Returns metadata describing the task."""
return task.TaskMetadata(
name="high_low_game",
description=
"A model repeatedly guesses a number, and is guided toward the correct answer with 'high' or 'low' responses. Scored based on minimum distance to the answer, averaged across several rounds.",
keywords=[
"repeated interaction",
"game play",
],
max_input_length_per_query=self.max_questions * 100,
max_queries=self.max_questions * len(self.targets))
def get_task_details(self):
return task.TaskMetadata(
name="roots_optimization_and_games",
description=
"Root finding, simple optimization and game-playing problems.",
keywords=[
"logical reasoning", "common sense", "mathematics", "algebra",
"numerical response", "zero-shot"
],
max_input_length_per_query=500,
max_queries=10000,
)
def get_task_details(self):
return task.TaskMetadata(
name="few_shot_arithmetic",
description=
"A template task to be used as an example during task development. (simple variable-difficulty arithmetic)",
keywords=[
"mathematics", "arithmetic", "numerical response", "zero-shot",
"example task"
],
max_input_length_per_query=4,
max_queries=5000,
)
def get_task_details(self) -> task.TaskMetadata:
"""Evaluates task for different metrics and few shots.
Returns:
Task metadata
"""
return task.TaskMetadata(
name=self.name,
description=self.task_data["description"],
keywords=self.task_data["keywords"],
max_input_length_per_query=self.task_data["max_input_length"],
max_queries=self.task_data["max_queries"],
)
def get_task_details(self):
return task.TaskMetadata(
name="context_definition_alignment",
description="Align contexts and definitions",
keywords=[
"common sense",
"logical reasoning",
"implicit reasoning",
"zero-shot",
],
max_input_length_per_query=72,
max_queries=5000,
)