def generate_menu(newspaperlist):
#Create .md file
filename = ".\\files\\md\\" + MENU_FILE_NAME
file = open(filename + ".md", "w+", encoding="utf8")
#Create title
nlwrite(file, ms.header("Noticias Resumidas - Menú", 1))
nlwrite(file, ms.horizontal_rule(3, "_"))
#List all newspapers with corresponding files
for newspaper in newspaperlist:
news_path = ".\\files\\" + TODAY.strftime(newspaper.name + DATE_FMT +
".html")
nlwrite(file, header((link(ms.bold(newspaper.name), news_path)), 2))
#Finish editing the .md file
file.close()
#Open both the .md and .html file to finish processing into HTML
file = open(filename + ".md", "r", encoding="utf8")
html = open(MENU_FILE_NAME + ".html", "w+", encoding="utf8")
html.write(md.markdown(file.read(), output_format="html5",
encoding="utf8"))
#Finish HTML processing
html.close()
def category_section_main(blg, stats):
"""Generates the main section of the category README.md file."""
value_percentage = float(
(
(int(stats["unprocessed"]) - int(stats["processed"]))
/ int(stats["unprocessed"])
)
* 100
)
link_filter = markdown_strings.link(
"Download",
f"{blg.info.home}/filters/{blg.category}.txt",
)
main_title = (
markdown_strings.header(f"{blg.data_json[blg.j_key.title]}", 1)
+ "\n"
+ "**"
+ link_filter
+ "**"
)
main_desc = markdown_strings.bold(f"{fill(blg.data_json[blg.j_key.desc])}")
info_list = [
f"Sources: {len(blg.data_json[blg.j_key.sources])}",
f"""Rules before processing: {stats["unprocessed"]}""",
f"""Rules after processing: {stats["processed"]}""",
]
info_add = markdown_strings.unordered_list(info_list)
string_bold = (
f"aBL - {blg.data_json[blg.j_key.title]} is {value_percentage:.2f}% lighter"
)
sub_desc = f"The {markdown_strings.bold(string_bold)} than its combined sources"
return [main_title, main_desc, info_add, sub_desc]
def generate_markdown(newspaper):
#Creating markdown file with today's date
filename = ".\\files\\md\\" + TODAY.strftime(newspaper.name + " %Y-%m-%d")
file = open(filename + ".md", "w+", encoding="utf8")
#Write down the header of the document
nlwrite(file, ms.header(f"Resumen de {newspaper.name}", 1))
nlwrite(
file,
header(TODAY.strftime("Resumen generado el día %Y-%m-%d a las %H:%M"),
4))
#Add a back button to return to the HTML menu
nlwrite(file,
link(ms.bold("BACK"), "..\\" + MENU_FILE_NAME + ".html") + " ")
nlwrite(file, ms.horizontal_rule(3, "_"))
for cat in newspaper.get_categories():
nlwrite(file, header(ms.italics(cat), 3))
for news in newspaper.get_articles(cat):
nlwrite(file,
unorlist(link(ms.bold(news.headline), news.link) + " "))
if news.has_pub_day() and news.has_author():
parsed_date = dp.parse(news.pub_day)
final_date = parsed_date.strftime("%A %d.%m.%Y").title()
nlwrite(file, f"{final_date} | Escrito por {news.author}")
elif news.has_pub_day():
parsed_date = dp.parse(news.pub_day)
final_date = parsed_date.strftime("%A %d.%m.%Y").title()
nlwrite(file, final_date)
elif news.has_author():
nlwrite(file, f"Escrito por {news.author}")
nlwrite(file)
#Save changes to markdown document.
file.close()
def unranked_subheader(ctx):
"""Build a subheader for an unranked beatmap."""
tokens = []
if ctx.mode is not None:
tokens.append(consts.mode2str[ctx.mode])
max_combo = scrape.max_combo(ctx)
if max_combo is not None:
tokens.append("%sx max combo" % sep(max_combo))
tokens.append("Unranked")
return md.bold(" || ".join(tokens))
def approved_subheader(ctx):
"""Build a subheader for a ranked/qualified/loved beatmap."""
tokens = []
rank_one = map_rank_one(ctx)
if rank_one is not None:
tokens.append(rank_one)
max_combo = scrape.max_combo(ctx)
if max_combo is not None:
tokens.append("%sx max combo" % sep(max_combo))
status = consts.int2status[ctx.beatmap.approved.value]
if ctx.beatmap.approved_date is not None and status != "Qualified":
status += " (%d)" % ctx.beatmap.approved_date.year
tokens.append(status)
if ctx.beatmap.playcount:
tokens.append("%s plays" % sep(ctx.beatmap.playcount))
return md.bold(" || ".join(tokens))
def test_bold():
assert ms.bold('bold') == '**bold**'
def gen_diagnostics(self):
insight_timespan_threshold = 10 * 60 # 10 min
if self.job_timespan < insight_timespan_threshold:
msg = "Insight will be available when more metric samples are " \
"collected.\n"
self.diagnostics += msg
return
# Check idleness
self.diagnostics += md.header("GPU Idleness", 2) + "\n"
if len(self.idle_gpus) == self.num_gpus:
msg = md.bold("All of %s GPU(s) in the job are idle. " %
len(self.idle_gpus))
msg += "Please consider killing the job if you no longer need it.\n"
self.diagnostics += msg
return
elif len(self.idle_gpus) > 0:
msg = md.bold("There are %s idle GPU(s) in the job.\n" %
len(self.idle_gpus))
c1 = "If you are running a job on all GPUs, please check if the process(es) on the idle GPU(s) have died/hung"
c2 = "If you do not need all GPUs in the job, please consider killing the job and request a new job with fewer GPUs."
msg += md.unordered_list([c1, c2]) + "\n"
self.diagnostics += msg
else:
self.diagnostics += md.bold("All GPU(s) are active.") + "\n"
self.diagnostics += "\n"
# Check Resource usage for active GPUs
self.diagnostics += md.header("Active GPU Utilization", 2) + "\n"
good_gpu_util_threshold = 90
good_gpu_mem_util_threshold = 50
if self.active_gpu_util >= good_gpu_util_threshold:
msg = "Average active GPU utilization over time is good at " \
"%.2f%%.\n" % self.active_gpu_util
self.diagnostics += msg
else:
msg = "Average active GPU utilization over time is " \
"%.2f%% < %s%%. You can try below suggestions to boost " \
"GPU utilization:\n" % \
(self.active_gpu_util, good_gpu_util_threshold)
suggestions = []
if self.active_gpu_memory_util < good_gpu_mem_util_threshold:
suggestions.append(
"Average active GPU memory utilization over time is below "
"%s%%. Try increasing batch size to put more data "
"onto GPU memory to boost GPU utilization. For a "
"distributed job, if the model has strict "
"requirement on the global effective batch size "
"for convergence, you can consider using a job "
"with fewer GPUs and bigger batch size per GPU." %
good_gpu_mem_util_threshold)
if self.max_cpu_per_gpu is not None and \
self.cpu_per_active_gpu < self.max_cpu_per_gpu:
suggestions.append(
"The job uses %.2f CPU cores per active GPU on average"
"over time. The maximum CPU cores per GPU you can "
"use without interfering with other GPUs in this "
"cluster is %.2f. You can use more CPU cores to "
"perform data preprocessing to keep GPUs from "
"starvation. Please consider using/increasing "
"parallel preprocessing on your input data." %
(self.cpu_per_active_gpu, self.max_cpu_per_gpu))
if self.max_memory_per_gpu is not None and \
self.memory_per_active_gpu < self.max_memory_per_gpu:
suggestions.append(
"The job uses %.2fG memory per active GPU on average"
"over time. The maximum memory per GPU you can "
"use without interfering with other GPUs in this "
"cluster is %.2fG. You can preload more input "
"data into memory to make sure your data pipeline "
"is never waiting on data loading from "
"disk/remote." % (self.memory_per_active_gpu / G,
self.max_memory_per_gpu / G))
suggestions.append(
"Please check if your program is waiting on NFS I/O. "
"If so, please consider using scalable storage, e.g. "
"Azure blob.")
suggestions.append(
"Suggestions above are purely based on average usage over a "
"time window. Please take a closer look at METRICS tab to "
"better understand the utilization pattern of GPU, GPU "
"memory, CPU and memory over time for further optimization.")
msg += md.unordered_list(suggestions) + "\n"
self.diagnostics += msg + "\n"
def test_gen_insights(self):
since = 1588630427
end = 1588634027
node_spec = test_node_spec()
task_gpu_percent = test_task_gpu_percent()
task_gpu_mem_percent = test_task_gpu_mem_percent()
task_cpu_percent = test_task_cpu_percent()
task_mem_usage_byte = test_task_mem_usage_byte()
running_job_ids = test_running_job_ids()
insights = gen_insights(task_gpu_percent, task_gpu_mem_percent,
task_cpu_percent, task_mem_usage_byte, since,
end, node_spec, running_job_ids)
self.assertEqual(len(insights), 1)
insight = insights[0]
expected_diagnostics = md.header("GPU Idleness", 2) + "\n"
expected_diagnostics += md.bold("All GPU(s) are active.") + "\n\n"
expected_diagnostics += md.header("Active GPU Utilization", 2) + "\n"
expected_diagnostics += "Average active GPU utilization over time is 30.00% < 90%. You can try below suggestions to boost GPU utilization:\n"
suggestions = []
suggestions.append(
"Average active GPU memory utilization over time is below "
"50%. Try increasing batch size to put more data "
"onto GPU memory to boost GPU utilization. For a "
"distributed job, if the model has strict "
"requirement on the global effective batch size "
"for convergence, you can consider using a job "
"with fewer GPUs and bigger batch size per GPU.")
suggestions.append(
"The job uses 1.00 CPU cores per active GPU on average"
"over time. The maximum CPU cores per GPU you can "
"use without interfering with other GPUs in this "
"cluster is 4.00. You can use more CPU cores to "
"perform data preprocessing to keep GPUs from "
"starvation. Please consider using/increasing "
"parallel preprocessing on your input data.")
suggestions.append(
"The job uses 10.00G memory per active GPU on average"
"over time. The maximum memory per GPU you can "
"use without interfering with other GPUs in this "
"cluster is 100.00G. You can preload more input "
"data into memory to make sure your data pipeline "
"is never waiting on data loading from "
"disk/remote.")
suggestions.append(
"Please check if your program is waiting on NFS I/O. "
"If so, please consider using scalable storage, e.g. "
"Azure blob.")
suggestions.append(
"Suggestions above are purely based on average usage over a "
"time window. Please take a closer look at METRICS tab to "
"better understand the utilization pattern of GPU, GPU "
"memory, CPU and memory over time for further optimization.")
expected_diagnostics += md.unordered_list(suggestions) + "\n"
expected_diagnostics += "\n"
expected_insight = {
"job_id": "job0",
"since": since,
"end": end,
"diagnostics": expected_diagnostics,
}
self.assertEqual(expected_insight, insight)
def bold(self, text):
return markdown_strings.bold(text)