def verify_keywords(task_info, task_name):
valid_keywords, _ = util.get_keywords()
for keyword in task_info.keywords:
assert keyword in valid_keywords, (
f'{task_name}: Keyword "{keyword}" does not appear in keywords.md. '
f"Either add it as a new keyword in keywords.md, or modify it to match a pre-existing keyword."
)
def generate_keyword_table(metadata):
key2task = keywords_to_task(metadata)
stats = []
detailed_view = build_heading("Detailed view", level=1)
for keyword, description in zip(*util.get_keywords(include_headers=True)):
if description is None:
stats.append([f"**{keyword}**", "", ""])
tasks = None
else:
tasks = [task_dir_link(task_dir) for task_dir in key2task[keyword]]
count_indicator = keyword_link(keyword, text=str(len(tasks)))
if len(tasks) == 0:
tasks = ["None."]
count_indicator += " 🚧" # to indicate that we seek such tasks
stats.append([f"  {keyword}", count_indicator, description])
detailed_view += build_keyword_entry(keyword, description, tasks)
intro = build_keywords_intro()
summary = build_heading("Summary table", level=1)
summary += generate_table(["Keyword", "Number of tasks", "Description"],
stats)
with open(
os.path.join("bigbench", "benchmark_tasks",
"keywords_to_tasks.md"), "w") as md_file:
md_file.write(preamble + intro + summary + detailed_view)
def generate_keyword_table():
key2task = keywords_to_task()
with open("bigbench/benchmark_tasks/keywords_to_tasks.md", "w") as md_file:
md_file.write(preamble + r"""
This file lists all the tasks to which each keyword applies. For a short list of keywords and descriptions see [keywords.md](../../keywords.md). To add new keywords, directly edit [keywords.md](../../keywords.md) rather than this file.
""")
for keyword, description in zip(*util.get_keywords()):
tasks = [task_dir_link(task_dir) for task_dir in key2task[keyword]]
if len(tasks) == 0:
tasks = ["None."]
md_file.write(f"""
### {keyword}
*{description}*
**Tasks:** {', '.join(tasks)}
""")
def generate_keyword_table():
key2task = keywords_to_task()
stats = []
detailed_view = build_heading("Detailed view", level=2)
for keyword, description in zip(*util.get_keywords()):
tasks = [task_dir_link(task_dir) for task_dir in key2task[keyword]]
count_indicator = keyword_link(keyword, text=str(len(tasks)))
if len(tasks) == 0:
tasks = ["None."]
count_indicator += " 🚧" # to indicate that we seek such tasks
stats.append([keyword, count_indicator, description])
detailed_view += build_keyword_entry(keyword, description, tasks)
intro = build_keywords_intro()
summary = build_heading("Summary table", level=2)
summary += generate_table(["Keyword", "Number of tasks", "Description"],
stats)
with open("bigbench/benchmark_tasks/keywords_to_tasks.md", "w") as md_file:
md_file.write(preamble + intro + summary + detailed_view)