def test_get_task_names(self):
self.assertListEqual(
bigbench_json_paths.get_task_names(bb_json_paths=TEST_JSON_PATHS),
[
'combination_composite_task', 'combination_simple_task',
'composite_gen_task', 'composite_mul_task', 'simple_gen_task',
'simple_mul_task', 'task_with_alternate_path'
])
def test_simple_combination(self, shots, vocab, max_examples):
test_json_paths = {
'test_simple_combination': '/foo3',
}
test_json_metadata = {
'test_simple_combination': {
'subtask_names': [],
'num_generate_text': 100,
'num_multiple_choice': 100,
},
}
test_json_subtask_names = {}
self.mock_bb_json_paths_utils(test_json_paths, test_json_metadata,
test_json_subtask_names)
# Register BIG-bench tasks and mixtures, similar to tasks.py.
task_names = bb_json_paths.get_task_names()
task_rates = {}
for task_name in task_names:
_, rates = task_api.register_composite_task(
task_name=task_name,
shots=shots,
vocab=vocab,
max_examples=max_examples,
**TEST_PREPROCESSING_SETTINGS)
task_rates.update(rates)
task_rates = sorted(task_rates.items())
# Verify registered tasks and mixtures.
registered_task_names = sorted(
[n for n in seqio.TaskRegistry.names() if 'bigbench:test_' in n])
registered_mixture_names = sorted([
n for n in seqio.MixtureRegistry.names() if 'bigbench:test_' in n
])
expected_task_rates = sorted([
self.get_seqio_name_and_rate('test_simple_combination', GEN, None,
shots, vocab, max_examples),
self.get_seqio_name_and_rate('test_simple_combination', MUL, None,
shots, vocab, max_examples),
self.get_seqio_name_and_rate('test_simple_combination', SCR, None,
shots, vocab, max_examples),
])
expected_mixture_rates = [
self.get_seqio_name_and_rate('test_simple_combination', MIX, None,
shots, vocab, max_examples),
]
expected_task_names = sorted(
[name_and_rate[0] for name_and_rate in expected_task_rates])
expected_mixture_names = sorted(
[name_and_rate[0] for name_and_rate in expected_mixture_rates])
self.assertListEqual(expected_task_names, registered_task_names)
self.assertListEqual(expected_mixture_names, registered_mixture_names)
# Verify the rates.
self.assertListEqual(expected_task_rates, task_rates)
def get_seqio_tasks_catalog_lines(seqio_task_names, seqio_mixture_names):
"""Generates a BIG-bench SeqIO Task catalog."""
bb_task_names = sorted(bigbench_json_paths.get_task_names(),
key=lambda x: x.lower())
# Group tasks with and without subtasks.
bb_task_names = sorted(bb_task_names, key=bigbench_json_paths.has_subtasks)
lines = []
lines.append(f"{utils.get_file_header()}\n\n")
# BIG-bench Lite Mixtures.
lines.append("## BIG-bench Lite SeqIO Mixtures\n\n")
lines.append("SeqIO Mixture | BIG-bench Tasks")
lines.append("----|----")
for bb_lite_name, bb_lite_task_names in bb_lite.BIGBENCH_LITE_TASK_NAMES.items(
):
bb_lite_seqio_mixtures = sorted([
m for m in seqio.MixtureRegistry.names() if f":{bb_lite_name}" in m
])
bb_lite_seqio_mixtures = [f"`{m}`" for m in bb_lite_seqio_mixtures]
bb_lite_seqio_mixtures = "<br>".join(bb_lite_seqio_mixtures)
bb_lite_task_names = sorted(bb_lite_task_names,
key=lambda x: x.lower())
bb_lite_task_urls = [
utils.get_bb_task_url(bb_task_name)
for bb_task_name in bb_lite_task_names
]
bb_lite_task_names = [
f"[{name}]({url})"
for name, url in zip(bb_lite_task_names, bb_lite_task_urls)
]
bb_lite_task_names = "<br>".join(bb_lite_task_names)
lines.append(f"{bb_lite_seqio_mixtures} | {bb_lite_task_names}")
lines.append("\n\n")
# All BIG-bench Tasks.
lines.append("## All BIG-bench SeqIO Tasks\n\n")
lines.append(
"BIG-bench Task | SeqIO Tasks | Subtasks | Description | Type")
lines.append("----|----|----|----|----")
for bb_task_name in bb_task_names:
# BB Task
bb_task_url = utils.get_bb_task_url(bb_task_name)
row = f"[{bb_task_name}]({bb_task_url})"
# SeqIO Tasks/Mixtures
formatted_bb_task_name = bbseqio_utils.format_name_for_seqio(
bb_task_name)
seqio_tasks = sorted([
f"`{t}`" for t in seqio_task_names
if f":{formatted_bb_task_name}." in t
])
seqio_mixtures = sorted([
f"`{m}`" for m in seqio_mixture_names
if f":{formatted_bb_task_name}." in m and ".mix." in m
])
to_show = seqio_mixtures if seqio_mixtures else seqio_tasks
heading = "Mixtures" if seqio_mixtures else "Tasks"
column = make_collapsible(heading, "<br>".join(to_show))
row = f"{row} | {column} "
# Subtasks
subtask_names = "-"
if bigbench_json_paths.has_subtasks(bb_task_name):
subtask_names = sorted([
bbseqio_utils.format_name_for_seqio(
subtask_name.split(":")[-1]) for subtask_name in
bigbench_json_paths.get_subtask_names(bb_task_name)
])
subtask_names = make_collapsible("Subtasks",
"<br>".join(subtask_names))
row = f"{row} | {subtask_names}"
# Description
description = bigbench_json_paths.get_task_description(bb_task_name)
column = make_collapsible("Description", description)
row = f"{row} | {column}"
# Types
task_types = []
if bigbench_json_paths.has_subtasks(bb_task_name):
for subtask_name in bigbench_json_paths.get_subtask_names(
bb_task_name):
task_types.extend(
bigbench_json_paths.get_task_types(subtask_name))
else:
task_types = bigbench_json_paths.get_task_types(bb_task_name)
task_types = set(task_types)
task_types = sorted([
bb.BigBenchTaskType.to_full_str(task_type)
for task_type in task_types
])
task_types = ", ".join(task_types)
row = f"{row} | {task_types}"
# Append row to table
lines.append(row)
return lines