def test_get_task_names(self): self.assertListEqual( bigbench_json_paths.get_task_names(bb_json_paths=TEST_JSON_PATHS), [ 'combination_composite_task', 'combination_simple_task', 'composite_gen_task', 'composite_mul_task', 'simple_gen_task', 'simple_mul_task', 'task_with_alternate_path' ])
def test_simple_combination(self, shots, vocab, max_examples): test_json_paths = { 'test_simple_combination': '/foo3', } test_json_metadata = { 'test_simple_combination': { 'subtask_names': [], 'num_generate_text': 100, 'num_multiple_choice': 100, }, } test_json_subtask_names = {} self.mock_bb_json_paths_utils(test_json_paths, test_json_metadata, test_json_subtask_names) # Register BIG-bench tasks and mixtures, similar to tasks.py. task_names = bb_json_paths.get_task_names() task_rates = {} for task_name in task_names: _, rates = task_api.register_composite_task( task_name=task_name, shots=shots, vocab=vocab, max_examples=max_examples, **TEST_PREPROCESSING_SETTINGS) task_rates.update(rates) task_rates = sorted(task_rates.items()) # Verify registered tasks and mixtures. registered_task_names = sorted( [n for n in seqio.TaskRegistry.names() if 'bigbench:test_' in n]) registered_mixture_names = sorted([ n for n in seqio.MixtureRegistry.names() if 'bigbench:test_' in n ]) expected_task_rates = sorted([ self.get_seqio_name_and_rate('test_simple_combination', GEN, None, shots, vocab, max_examples), self.get_seqio_name_and_rate('test_simple_combination', MUL, None, shots, vocab, max_examples), self.get_seqio_name_and_rate('test_simple_combination', SCR, None, shots, vocab, max_examples), ]) expected_mixture_rates = [ self.get_seqio_name_and_rate('test_simple_combination', MIX, None, shots, vocab, max_examples), ] expected_task_names = sorted( [name_and_rate[0] for name_and_rate in expected_task_rates]) expected_mixture_names = sorted( [name_and_rate[0] for name_and_rate in expected_mixture_rates]) self.assertListEqual(expected_task_names, registered_task_names) self.assertListEqual(expected_mixture_names, registered_mixture_names) # Verify the rates. self.assertListEqual(expected_task_rates, task_rates)
def get_seqio_tasks_catalog_lines(seqio_task_names, seqio_mixture_names): """Generates a BIG-bench SeqIO Task catalog.""" bb_task_names = sorted(bigbench_json_paths.get_task_names(), key=lambda x: x.lower()) # Group tasks with and without subtasks. bb_task_names = sorted(bb_task_names, key=bigbench_json_paths.has_subtasks) lines = [] lines.append(f"{utils.get_file_header()}\n\n") # BIG-bench Lite Mixtures. lines.append("## BIG-bench Lite SeqIO Mixtures\n\n") lines.append("SeqIO Mixture | BIG-bench Tasks") lines.append("----|----") for bb_lite_name, bb_lite_task_names in bb_lite.BIGBENCH_LITE_TASK_NAMES.items( ): bb_lite_seqio_mixtures = sorted([ m for m in seqio.MixtureRegistry.names() if f":{bb_lite_name}" in m ]) bb_lite_seqio_mixtures = [f"`{m}`" for m in bb_lite_seqio_mixtures] bb_lite_seqio_mixtures = "<br>".join(bb_lite_seqio_mixtures) bb_lite_task_names = sorted(bb_lite_task_names, key=lambda x: x.lower()) bb_lite_task_urls = [ utils.get_bb_task_url(bb_task_name) for bb_task_name in bb_lite_task_names ] bb_lite_task_names = [ f"[{name}]({url})" for name, url in zip(bb_lite_task_names, bb_lite_task_urls) ] bb_lite_task_names = "<br>".join(bb_lite_task_names) lines.append(f"{bb_lite_seqio_mixtures} | {bb_lite_task_names}") lines.append("\n\n") # All BIG-bench Tasks. lines.append("## All BIG-bench SeqIO Tasks\n\n") lines.append( "BIG-bench Task | SeqIO Tasks | Subtasks | Description | Type") lines.append("----|----|----|----|----") for bb_task_name in bb_task_names: # BB Task bb_task_url = utils.get_bb_task_url(bb_task_name) row = f"[{bb_task_name}]({bb_task_url})" # SeqIO Tasks/Mixtures formatted_bb_task_name = bbseqio_utils.format_name_for_seqio( bb_task_name) seqio_tasks = sorted([ f"`{t}`" for t in seqio_task_names if f":{formatted_bb_task_name}." in t ]) seqio_mixtures = sorted([ f"`{m}`" for m in seqio_mixture_names if f":{formatted_bb_task_name}." in m and ".mix." in m ]) to_show = seqio_mixtures if seqio_mixtures else seqio_tasks heading = "Mixtures" if seqio_mixtures else "Tasks" column = make_collapsible(heading, "<br>".join(to_show)) row = f"{row} | {column} " # Subtasks subtask_names = "-" if bigbench_json_paths.has_subtasks(bb_task_name): subtask_names = sorted([ bbseqio_utils.format_name_for_seqio( subtask_name.split(":")[-1]) for subtask_name in bigbench_json_paths.get_subtask_names(bb_task_name) ]) subtask_names = make_collapsible("Subtasks", "<br>".join(subtask_names)) row = f"{row} | {subtask_names}" # Description description = bigbench_json_paths.get_task_description(bb_task_name) column = make_collapsible("Description", description) row = f"{row} | {column}" # Types task_types = [] if bigbench_json_paths.has_subtasks(bb_task_name): for subtask_name in bigbench_json_paths.get_subtask_names( bb_task_name): task_types.extend( bigbench_json_paths.get_task_types(subtask_name)) else: task_types = bigbench_json_paths.get_task_types(bb_task_name) task_types = set(task_types) task_types = sorted([ bb.BigBenchTaskType.to_full_str(task_type) for task_type in task_types ]) task_types = ", ".join(task_types) row = f"{row} | {task_types}" # Append row to table lines.append(row) return lines